Why would I be getting

Exceeded max_retries of 1

for an Op with

retry_policy=RetryPolicy(max_retries=9, delay=30)

after 1 retry?

Hi Charlie, would you mind sharing a code snippet of the op?

yup!

@op(
    ins={
        "table": In(dagster_type=Any),
        "projection": In(dagster_type=Optional[String]),
        "query": In(dagster_type=Optional[String]),
        "count": In(dagster_type=Int),
        "n_pages": In(dagster_type=Int),
    },
    out={"gcs_file_handles": Out(dagster_type=List)},
    required_resource_keys={"gcs_fm"},
    retry_policy=RetryPolicy(max_retries=9, delay=30),
    config_schema={"query_timeout": Field(Int, is_required=False, default_value=30)},
    tags={"dagster/priority": 6},
)
def get_data(context, table, projection, query, count, n_pages):
    file_ext = "json.gz"
    file_stem = "_".join(filter(None, [table.name, str(query or "")]))

    gcs_file_handles = []
    for p in range(n_pages):
        file_key = f"{table.name}/{file_stem}_p_{p}.{file_ext}"

        if context.retry_number > 0 and context.resources.gcs_fm._has_object(
            key=file_key
        ):
            context.log.debug("File already exists from previous try. Skipping.")
        else:
            context.log.debug(f"page:\t{(p + 1)}/{n_pages}")

            try:
                data = time_limit_query(
                    context=context,
                    table=table,
                    query=query,
                    projection=projection,
                    page=(p + 1),
                )
            except Exception as e:
                raise RetryRequested(
                    max_retries=context.op_def.retry_policy.max_retries,
                    seconds_to_wait=context.op_def.retry_policy.delay,
                ) from e

            jsongz_obj = gzip.compress(json.dumps(data).encode("utf-8"))

            gcs_file_handles.append(
                context.resources.gcs_fm.upload_data(
                    context=context, obj=jsongz_obj, file_key=file_key
                )
            )

So on looking into this, what you’re seeing is the intended behavior, but we think it’s a bit confusing and are open to changing it. Moving further discussion to Github.

Question in the thread has been surfaced to GitHub Discussions for future discoverability: https://github.com/dagster-io/dagster/discussions/8061

cool thanks!