celery_k8s_job_executor

modes

ModuleNotFoundError: No module named 'dagster_celery.executor_k8s'

dagit==0.9.1
dagster==0.9.1
dagster-celery==0.9.1
dagster-celery-k8s==0.9.1
dagster-cron==0.9.1
dagster-graphql==0.9.1
dagster-k8s==0.9.1

pyenv virtualenv

File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/api/utils.py", line 11, in execute_command_in_subprocess
    subprocess.check_output(parts, stderr=subprocess.STDOUT)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/lib/python3.7/subprocess.py", line 411, in check_output
    **kwargs).stdout
  File "/Users/peterszalai/.pyenv/versions/3.7.5/lib/python3.7/subprocess.py", line 512, in run
    output=stdout, stderr=stderr)
subprocess.CalledProcessError: Command '['/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/bin/python', '-m', 'dagster', 'api', 'list_repositories', '/var/folders/0l/sclnn8b52j79pvyr2pplp6p00000gp/T/tmpvz50mzut', '/var/folders/0l/sclnn8b52j79pvyr2pplp6p00000gp/T/tmp1rx65oxm']' returned non-zero exit status 1.

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/peterszalai/.pyenv/versions/dagst/bin/dagit", line 10, in <module>
    sys.exit(main())
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagit/cli.py", line 200, in main
    cli(auto_envvar_prefix='DAGIT')  # pylint:disable=E1120
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagit/cli.py", line 91, in ui
    host_dagit_ui(host, port, path_prefix, storage_fallback, port_lookup, **kwargs)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagit/cli.py", line 100, in host_dagit_ui
    workspace = get_workspace_from_kwargs(kwargs, instance)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/cli_target.py", line 201, in get_workspace_from_kwargs
    return workspace_from_load_target(created_workspace_load_target(kwargs), instance)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/cli_target.py", line 163, in workspace_from_load_target
    return load_workspace_from_yaml_paths(load_target.paths, python_user_process_api)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 25, in load_workspace_from_yaml_paths
    for workspace_config, yaml_path in zip(workspace_configs, yaml_paths)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 25, in <listcomp>
    for workspace_config, yaml_path in zip(workspace_configs, yaml_paths)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 62, in load_workspace_from_config
    location_config, yaml_path, python_user_process_api
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 333, in _location_handle_from_location_config
    location_config, yaml_path, python_user_process_api
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 364, in _location_handle_from_target_config
    target_config['python_file'], yaml_path, user_process_api
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 203, in _location_handle_from_python_file_config
    executable_path,
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/load.py", line 253, in location_handle_from_python_file
    attribute=attribute,
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/api/list_repositories.py", line 17, in sync_list_repositories
    attribute=attribute,
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/api/utils.py", line 34, in execute_unary_api_cli_command
    execute_command_in_subprocess(parts)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/api/utils.py", line 15, in execute_command_in_subprocess
    cmd=e.cmd, output=e.output.decode('utf-8')
dagster.serdes.ipc.DagsterIPCProtocolError: Error when executing API command ['/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/bin/python', '-m', 'dagster', 'api', 'list_repositories', '/var/folders/0l/sclnn8b52j79pvyr2pplp6p00000gp/T/tmpvz50mzut', '/var/folders/0l/sclnn8b52j79pvyr2pplp6p00000gp/T/tmp1rx65oxm']: Traceback (most recent call last):
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/core/code_pointer.py", line 87, in load_python_file
    return import_module_from_path(module_name, python_file)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/seven/__init__.py", line 115, in import_module_from_path
    spec.loader.exec_module(module)
  File "<frozen importlib._bootstrap_external>", line 728, in exec_module
  File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
  File "/Users/peterszalai/git/data-pipeline/src/repositories.py", line 7, in <module>
    from pipelines import (
  File "/Users/peterszalai/git/data-pipeline/src/pipelines.py", line 7, in <module>
    from dagster_celery.executor_k8s import celery_k8s_job_executor
ModuleNotFoundError: No module named 'dagster_celery.executor_k8s'

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "/Users/peterszalai/.pyenv/versions/3.7.5/lib/python3.7/runpy.py", line 193, in _run_module_as_main
    "__main__", mod_spec)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/lib/python3.7/runpy.py", line 85, in _run_code
    exec(code, run_globals)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/__main__.py", line 3, in <module>
    main()
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/__init__.py", line 38, in main
    cli(obj={})  # pylint:disable=E1123
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 829, in __call__
    return self.main(*args, **kwargs)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 782, in main
    rv = self.invoke(ctx)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 1259, in invoke
    return _process_result(sub_ctx.command.invoke(sub_ctx))
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 1066, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/click/core.py", line 610, in invoke
    return callback(*args, **kwargs)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/api.py", line 116, in command
    output = check.inst(fn(args), output_cls)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/api.py", line 141, in list_repositories_command
    loadable_targets = get_loadable_targets(python_file, module_name, working_directory, attribute)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/grpc/utils.py", line 20, in get_loadable_targets
    else loadable_targets_from_python_file(python_file, working_directory)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/cli/workspace/autodiscovery.py", line 11, in loadable_targets_from_python_file
    loaded_module = load_python_file(python_file, working_directory)
  File "/Users/peterszalai/.pyenv/versions/3.7.5/envs/dagst/lib/python3.7/site-packages/dagster/core/code_pointer.py", line 121, in load_python_file
    ie,
  File "<string>", line 3, in raise_from
dagster.core.errors.DagsterImportError: Encountered ImportError: `No module named 'dagster_celery.executor_k8s'` while importing module repositories from file /Users/peterszalai/git/data-pipeline/src/repositories.py. Local modules were resolved using the working directory `/Users/peterszalai/git/data-pipeline`. If another working directory should be used, please explicitly specify the appropriate path using the `-d` or `--working-directory` for CLI based targets or the `working_directory` configuration option for `python_file`-based workspace.yaml targets.

from dagster_celery_k8s.executor import celery_k8s_job_executor