Hebo Yang
01/04/2023, 7:46 PMMultiprocess executor: child process for step run_forecast_xp_cng_conversion_rate_nc unexpectedly exited with code -9
dagster._core.executor.child_process_executor.ChildProcessCrashException
Stack Trace:
File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/multiprocess.py", line 210, in execute
event_or_none = next(step_iter)
, File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/multiprocess.py", line 324, in execute_step_out_of_process
for ret in execute_child_process_command(multiproc_ctx, command):
, File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/child_process_executor.py", line 163, in execute_child_process_command
raise ChildProcessCrashException(exit_code=process.exitcode)
dagster._core.executor.child_process_executor.ChildProcessCrashException
Stack Trace:
File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/multiprocess.py", line 210, in execute
event_or_none = next(step_iter)
, File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/multiprocess.py", line 324, in execute_step_out_of_process
for ret in execute_child_process_command(multiproc_ctx, command):
, File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/child_process_executor.py", line 163, in execute_child_process_command
raise ChildProcessCrashException(exit_code=process.exitcode)
dagster._check.CheckError: Invariant failed. Description: Attempted to mark step run_forecast_xp_cng_conversion_rate_nc as complete that was not known to be in flight
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/api.py", line 1033, in pipeline_execution_iterator
for event in pipeline_context.executor.execute(pipeline_context, execution_plan):
File "/usr/local/lib/python3.8/site-packages/dagster/_core/executor/multiprocess.py", line 240, in execute
active_execution.handle_event(step_failure_event)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/plan/active.py", line 400, in handle_event
self.mark_failed(step_key)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/plan/active.py", line 342, in mark_failed
self._mark_complete(step_key)
File "/usr/local/lib/python3.8/site-packages/dagster/_core/execution/plan/active.py", line 387, in _mark_complete
check.invariant(
File "/usr/local/lib/python3.8/site-packages/dagster/_check/__init__.py", line 1470, in invariant
raise CheckError(f"Invariant failed. Description: {desc}")
alex
01/04/2023, 7:54 PM-9
means the process was killed via the SIGKILL
signal, likely by the oom_killer
which happens when the process tries to use too much memory.Hebo Yang
01/04/2023, 8:14 PM