Jeremy
09/07/2022, 10:11 PM@graph(tags={"notify": "slack"})
def splunk_augmentation_graph():
xxx = start_splunk_run()
return metadata_logic(
RUN_ID,
new_ips_df,
get_ssl_cert_info(xxx,
get_clearbit_reveal(xxx),
get_registry_info(xxx),
get_nslookup_info(xxx),
)
splunk_metadata_dev = AssetsDefinition.from_graph(
splunk_augmentation_graph,
partitions_def=splunk_dev_schedule,
keys_by_output_name={
"result": AssetKey(["augmentation_data", "splunk_metadata"])
},
)
splunk_metadata_group_dev = with_resources(
[splunk_metadata_dev],
resource_defs=RESOURCES,
)
splunk_metadata_dev_job = define_asset_job(
name="splunk_metadata_dev_job",
selection=AssetSelection.assets(splunk_metadata_dev),
partitions_def=splunk_dev_schedule,
)
@op(
out={
"splunk_metadata": Out(
asset_key=AssetKey(["augmentation_data", "splunk_metadata"]),
metadata={"partition_expr": "created_on"},
io_manager_key="snowflake_io_manager",
)
}
)
def metadata_logic(
yuhan
09/08/2022, 1:02 AMwith_resources
through resource_defs
. Here’s an example: https://docs.dagster.io/concepts/io-management/io-managers#applying-io-managers-to-assetsJeremy
09/08/2022, 1:15 AM