Hi everyone. Sometimes I get a Java connection ref...
# ask-community
d
Hi everyone. Sometimes I get a Java connection refused error:
ERROR:root:Exception while sending command.
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1200, in send_command
answer = smart_decode(self.stream.readline()[:-1])
File "/usr/local/lib/python3.8/socket.py", line 669, in readinto
return self._sock.recv_into(b)
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1033, in send_command
response = connection.send_command(command)
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1211, in send_command
raise Py4JNetworkError(
py4j.protocol.Py4JNetworkError: Error while receiving
ERROR:root:Exception while sending command.
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1207, in send_command
raise Py4JNetworkError("Answer from Java side is empty")
py4j.protocol.Py4JNetworkError: Answer from Java side is empty
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1033, in send_command
response = connection.send_command(command)
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1211, in send_command
raise Py4JNetworkError(
py4j.protocol.Py4JNetworkError: Error while receiving
ERROR:py4j.java_gateway:An error occurred while trying to connect to the Java server (127.0.0.1:41979)
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 977, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1115, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 977, in _get_connection
connection = self.deque.pop()
IndexError: pop from an empty deque
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1115, in start
self.socket.connect((self.address, self.port))
ConnectionRefusedError: [Errno 111] Connection refused
plus1 1
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/tmp/tmpeznsler9.py", line 127, in main
list(run_step_from_ref(step_run_ref, instance))
File "/databricks/python/lib/python3.8/site-packages/dagster/core/execution/plan/external_step.py", line 279, in run_step_from_ref
step_context = step_run_ref_to_step_context(step_run_ref, instance)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/execution/plan/external_step.py", line 238, in step_run_ref_to_step_context
pipeline = step_run_ref.recon_pipeline.subset_for_execution_from_existing_pipeline(
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/reconstruct.py", line 230, in subset_for_execution_from_existing_pipeline
return self._subset_for_execution(solids_to_execute=solids_to_execute, solid_selection=None)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/reconstruct.py", line 183, in _subset_for_execution
pipeline_def = self.get_definition()
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/reconstruct.py", line 152, in get_definition
defn = self.repository.get_definition().get_pipeline(self.pipeline_name)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/reconstruct.py", line 74, in get_definition
return repository_def_from_pointer(self.pointer)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/reconstruct.py", line 628, in repository_def_from_pointer
target = def_from_pointer(pointer)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/reconstruct.py", line 549, in def_from_pointer
target = pointer.load_target()
File "/databricks/python/lib/python3.8/site-packages/dagster/core/code_pointer.py", line 201, in load_target
module = load_python_module(self.module, self.working_directory)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/code_pointer.py", line 125, in load_python_module
return importlib.import_module(module_name)
File "/usr/local/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1014, in _gcd_import
File "<frozen importlib._bootstrap>", line 991, in _find_and_load
File "<frozen importlib._bootstrap>", line 975, in _find_and_load_unlocked
File "<frozen importlib._bootstrap>", line 671, in _load_unlocked
File "<frozen importlib._bootstrap_external>", line 783, in exec_module
File "<frozen importlib._bootstrap>", line 219, in _call_with_frames_removed
File "/tmp/tmp9_012ifp/cashback_spark_tests/utilities/results_assertion.py", line 1, in <module>
from cashback_spark.utilities.pyspark_dataframes import flatten_df
File "/databricks/python_shell/dbruntime/PythonPackageImportsInstrumentation/__init__.py", line 158, in import_patch
original_result = python_builtin_import(name, globals, locals, fromlist, level)
File "/tmp/tmp9_012ifp/cashback_spark/__init__.py", line 1, in <module>
from .repository import cashback_spark
File "/databricks/python_shell/dbruntime/PythonPackageImportsInstrumentation/__init__.py", line 158, in import_patch
original_result = python_builtin_import(name, globals, locals, fromlist, level)
File "/tmp/tmp9_012ifp/cashback_spark/repository.py", line 40, in <module>
def cashback_spark():
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/decorators/repository_decorator.py", line 238, in repository
return _Repository()(name)
File "/databricks/python/lib/python3.8/site-packages/dagster/core/definitions/decorators/repository_decorator.py", line 33, in __call__
repository_definitions = fn()
File "/tmp/tmp9_012ifp/cashback_spark/repository.py", line 41, in cashback_spark
jobs = load_jobs()
File "/tmp/tmp9_012ifp/cashback_spark/repository.py", line 27, in load_jobs
return loader(folder="jobs")
File "/tmp/tmp9_012ifp/cashback_spark/repository.py", line 20, in loader
module = import_module(module_name)
File "/usr/local/lib/python3.8/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "/tmp/tmp9_012ifp/cashback_spark/jobs/addresses_job.py", line 1, in <module>
from cashback_spark.ops.addresses import (add_region, get_coordinates,
File "/databricks/python_shell/dbruntime/PythonPackageImportsInstrumentation/__init__.py", line 158, in import_patch
original_result = python_builtin_import(name, globals, locals, fromlist, level)
File "/tmp/tmp9_012ifp/cashback_spark/ops/addresses.py", line 131, in <module>
def compare2continent(lat, lng):
File "/databricks/spark/python/pyspark/databricks/sql/functions.py", line 72, in _create_edge_udf
session.conf.get(
File "/databricks/spark/python/pyspark/sql/conf.py", line 45, in get
return self._jconf.get(key)
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1303, in __call__
answer = self.gateway_client.send_command(command)
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1031, in send_command
connection = self._get_connection()
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 979, in _get_connection
connection = self._create_connection()
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 985, in _create_connection
connection.start()
File "/databricks/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py", line 1127, in start
raise Py4JNetworkError(msg, e)
py4j.protocol.Py4JNetworkError: An error occurred while trying to connect to the Java server (127.0.0.1:41979)
2022-09-08 12:22:10 +0100 - dagster - ERROR - assert_wrapper - c769ccf6-3823-4846-ac98-b72de0feaf23 - 7605 - assert_tables - STEP_FAILURE - Execution of step "assert_tables" failed.
dagster_databricks.databricks.DatabricksError: Run 1747310 failed with result state: DatabricksRunResultState.Failed. Message:
Stack Trace:
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 232, in dagster_event_sequence_for_step
for step_event in check.generator(step_events):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks_pyspark_step_launcher.py", line 181, in launch_step
yield from self.step_events_iterator(step_context, step_key, databricks_run_id)
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks_pyspark_step_launcher.py", line 223, in step_events_iterator
done = poll_run_state(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks.py", line 272, in poll_run_state
raise DatabricksError(error_message)
2022-09-08 12:22:10 +0100 - dagster - ERROR - assert_wrapper - c769ccf6-3823-4846-ac98-b72de0feaf23 - 7605 - RUN_FAILURE - Execution of run for "assert_wrapper" failed. An exception was thrown during execution.
dagster_databricks.databricks.DatabricksError: Run 1747310 failed with result state: DatabricksRunResultState.Failed. Message:
Stack Trace:
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 785, in pipeline_execution_iterator
for event in pipeline_context.executor.execute(pipeline_context, execution_plan):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/executor/in_process.py", line 38, in execute
yield from iter(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 879, in __iter__
yield from self.iterator(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 87, in inner_plan_execution_iterator
for step_event in check.generator(dagster_event_sequence_for_step(step_context)):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 336, in dagster_event_sequence_for_step
raise unexpected_exception
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 232, in dagster_event_sequence_for_step
for step_event in check.generator(step_events):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks_pyspark_step_launcher.py", line 181, in launch_step
yield from self.step_events_iterator(step_context, step_key, databricks_run_id)
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks_pyspark_step_launcher.py", line 223, in step_events_iterator
done = poll_run_state(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks.py", line 272, in poll_run_state
raise DatabricksError(error_message)
Traceback (most recent call last):
File "/mnt/c/Users/david/Repos/D4DZarco/cashback_spark/cashback_spark_tests/test_jobs/subpayments_job.py", line 45, in <module>
execute_pipeline(reconstructable(assert_wrapper), run_config=config, mode="prod_adls2")
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 383, in execute_pipeline
return _logged_execute_pipeline(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/telemetry.py", line 110, in wrap
result = f(*args, **kwargs)
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 434, in _logged_execute_pipeline
return execute_run(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 236, in execute_run
event_list = list(_execute_run_iterable)
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 879, in __iter__
yield from self.iterator(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 785, in pipeline_execution_iterator
for event in pipeline_context.executor.execute(pipeline_context, execution_plan):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/executor/in_process.py", line 38, in execute
yield from iter(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/api.py", line 879, in __iter__
yield from self.iterator(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 87, in inner_plan_execution_iterator
for step_event in check.generator(dagster_event_sequence_for_step(step_context)):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 336, in dagster_event_sequence_for_step
raise unexpected_exception
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster/core/execution/plan/execute_plan.py", line 232, in dagster_event_sequence_for_step
for step_event in check.generator(step_events):
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks_pyspark_step_launcher.py", line 181, in launch_step
yield from self.step_events_iterator(step_context, step_key, databricks_run_id)
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks_pyspark_step_launcher.py", line 223, in step_events_iterator
done = poll_run_state(
File "/home/dan/anaconda3/envs/dagster_0_14_6/lib/python3.9/site-packages/dagster_databricks/databricks.py", line 272, in poll_run_state
raise DatabricksError(error_message)
dagster_databricks.databricks.DatabricksError: Run 1747310 failed with result state: DatabricksRunResultState.Failed. Message:
Any hint?
y
the error seems to be surfaced from databricks side. cc @owen do you have any hint for troubleshooting?
o
hard to tell from the stack trace -- have you been able to successfully use the step launcher before, and this error just started showing up?
also, do you have access to the databricks console? if so, the driver logs might have more detailed stdout / stderr info
d
Thanks. I'll be waiting for next time it occurs to check the databricks logs, as it only shows the last few hours.