I want to give the option to run data in the pipel...
# ask-community
I want to give the option to run data in the pipeline as a whole set or by choosing a particular partition_name, which can be given via a config
Copy code
def transformed_data() -> pd.DataFrame:
  df = get_data_from_s3(partition_name=val_in_config)
  df = do_some_stuff(df)
  return df

data_transform_job = define_asset_job(
how would I complete this approach? (get_data_from_s3 will get all data if partition_name==none)
Hi Brittany - you could do something like this:
Copy code
from dagster import Field

@asset(config_schema={"partition_name": Field(str, is_required=False)})
def transformed_data(context) -> pd.DataFrame:
  partition_name = context.op_config.get("partition_name")
  df = get_data_from_s3(partition_name=partition_name)
  df = do_some_stuff(df)
  return df