Jamie Lee
01/17/2023, 9:32 AMRESOURCE_INIT_FAILURE
: Initialization of resources [fivetran, io_manager, slack, s3] failed.
error on a new pipeline we built. The Ops and Jobs are set up exactly the same as other pipelines, so really confused as to why this is happening. Op code in thread 🧵intercom_sync = fivetran_sync_op.configured(
{
"connector_id": "handling_caries",
},
name="intercom_sync",
)
@op(
name="fivetran_asset",
ins={
"salesforce": In(dagster_type=FivetranOutput),
"scv_domain_map": In(dagster_type=FivetranOutput),
"braintree": In(dagster_type=FivetranOutput),
"intercom": In(dagster_type=FivetranOutput),
},
out=Out(dagster_type=Nothing),
tags=annotations,
)
def create_fivetran_asset(salesforce, scv_domain_map, braintree, intercom):
yield AssetMaterialization(
asset_key=AssetKey("fivetran_sync"),
metadata={
"Description": "Return an asset materialisation for when the fivetran ops finish"
},
)
@slack_data_on_failure
@job(
name="fivetran_sync_pipeline",
resource_defs=prod_resources,
executor_def=job_executor_def,
tags=annotations,
config=k8s_job_executor_executor_conf,
)
def fivetran_sync_pipeline():
create_fivetran_asset(
salesforce_sync(),
scv_domain_map_sync(),
braintree_sync(),
intercom_sync(),
)
chris
01/17/2023, 10:02 PMJamie Lee
01/18/2023, 9:07 AMAn exception was thrown during step execution that is likely a framework error, rather than an error in user code.
dagster._core.errors.DagsterResourceFunctionError: Error executing resource_fn on ResourceDefinition io_manager
File "/usr/local/lib/python3.8/site-packages/dagster/_cli/api.py", line 441, in _execute_step_command_body
yield from execute_plan_iterator(
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/api.py", line 1185, in __iter__
yield from self.execution_context_manager.prepare_context()
File "/usr/local/lib/python3.8/site-packages/dagster/_utils/__init__.py", line 522, in generate_setup_events
obj = next(self.generator)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/context_creation_pipeline.py", line 260, in execution_context_event_generator
yield from resources_manager.generate_setup_events()
File "/usr/local/lib/python3.8/site-packages/dagster/_utils/__init__.py", line 522, in generate_setup_events
obj = next(self.generator)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/resources_init.py", line 257, in resource_initialization_event_generator
yield from _core_resource_initialization_event_generator(
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/resources_init.py", line 220, in _core_resource_initialization_event_generator
raise dagster_user_error
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/resources_init.py", line 185, in _core_resource_initialization_event_generator
for event in manager.generate_setup_events():
File "/usr/local/lib/python3.8/site-packages/dagster/_utils/__init__.py", line 522, in generate_setup_events
obj = next(self.generator)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/resources_init.py", line 349, in single_resource_event_generator
raise dagster_user_error
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/resources_init.py", line 342, in single_resource_event_generator
check.failed(
File "/usr/local/lib/python3.8/contextlib.py", line 131, in __exit__
self.gen.throw(type, value, traceback)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/errors.py", line 206, in user_code_error_boundary
raise error_cls(
The above exception was caused by the following exception:
botocore.exceptions.ClientError: An error occurred (AccessDenied) when calling the ListObjects operation: Access Denied
File "/usr/local/lib/python3.8/site-packages/dagster/_core/errors.py", line 199, in user_code_error_boundary
yield
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/resources_init.py", line 325, in single_resource_event_generator
resource_def.resource_fn(context)
File "/opt/dagster/app/src/data_pipeline/resources/io_managers/s3.py", line 107, in s3_io_manager_with_path_entry
io_manager_with_path = S3IOManagerWithPathEntry(
File "/usr/local/lib/python3.8/site-packages/dagster_aws/s3/io_manager.py", line 28, in __init__
self.s3.list_objects(Bucket=self.bucket, Prefix=self.s3_prefix, MaxKeys=1)
File "/usr/local/lib/python3.8/site-packages/botocore/client.py", line 508, in _api_call
return self._make_api_call(operation_name, kwargs)
File "/usr/local/lib/python3.8/site-packages/botocore/client.py", line 915, in _make_api_call
raise error_class(parsed_response, operation_name)