Daniel Mosesson
02/28/2023, 8:59 AMsandy
02/28/2023, 4:47 PMDaniel Gafni
02/28/2023, 4:49 PMDaniel Mosesson
02/28/2023, 4:52 PMDaniel Gafni
02/28/2023, 5:02 PMget_downstream_partitions_for_partition_range
(I never had to use this one, so the implementation can be just anything) and get_upstream_partitions_for_partition_range
(this one is important and actually does the mapping). Here is my example which is mapping N upstream days indo a single downstream day.
class NDaysPartitionMapping(PartitionMapping):
def __init__(self, days: int, offset: int = 0):
self.days = days
self.offset = offset
def get_downstream_partitions_for_partition_range(
self,
upstream_partition_key_range: PartitionKeyRange,
downstream_partitions_def: PartitionsDefinition | None,
upstream_partitions_def: PartitionsDefinition,
) -> PartitionKeyRange:
assert isinstance(upstream_partitions_def, DailyPartitionsDefinition)
assert downstream_partitions_def is not None # mypy
return upstream_partition_key_range
def get_upstream_partitions_for_partition_range(
self,
downstream_partition_key_range: PartitionKeyRange | None,
downstream_partitions_def: PartitionsDefinition | None,
upstream_partitions_def: PartitionsDefinition,
) -> PartitionKeyRange:
assert isinstance(downstream_partitions_def, TimeWindowPartitionsDefinition)
assert isinstance(upstream_partitions_def, TimeWindowPartitionsDefinition)
assert downstream_partition_key_range is not None # mypy
assert downstream_partitions_def is not None # mypy
mapped_range = PartitionKeyRange(
start=(
datetime.strptime(downstream_partition_key_range.start, downstream_partitions_def.fmt)
- timedelta(days=self.days)
- timedelta(days=self.offset)
).strftime(upstream_partitions_def.fmt),
end=(
datetime.strptime(downstream_partition_key_range.end, downstream_partitions_def.fmt)
- timedelta(days=self.offset)
).strftime(upstream_partitions_def.fmt),
)
return mapped_range
Daniel Mosesson
02/28/2023, 5:14 PMins
+ AssetIn(partition_mapping
to make it work?Daniel Gafni
02/28/2023, 5:15 PMDaniel Mosesson
02/28/2023, 5:15 PM