esztermarton
12/07/2020, 12:26 PMFROM bitnami/python:3.8
COPY ./*.txt /app
RUN apt-get update && apt-get install -yqq cron \
&& pip install -r requirements.txt
COPY . /app
RUN chmod +x /app/entrypoint.sh && mkdir -p /app/data
ENV PYTHONPATH=$PYTHONPATH:/app
ENV DAGSTER_HOME=/app
EXPOSE 3000
ENTRYPOINT ["/app/entrypoint.sh"]
#!/bin/sh
export DAGSTER_HOME=/app
# This block may be omitted if not packaging a repository with cron schedules:
####################################################################################################
# see: <https://unix.stackexchange.com/a/453053> - fixes inflated hard link count
touch /etc/crontab /etc/cron.*/*
service cron start
# Add all schedules defined by the user
dagster schedule up
####################################################################################################
# Launch Dagit as a service
DAGSTER_HOME=/app dagit -h 0.0.0.0 -p 3003
scheduler:
module: dagster_cron.cron_scheduler
class: SystemCronScheduler
dagit<1.0.0
dagster<1.0.0
dagster-cron<1.0.0
tweepy==3.9.0
pandas==1.1.4
jupyter==1.0.0
pytest==6.1.2
nest_asyncio
pylint==2.6.0
rope==0.18.0
# This file defines the dagster workspace.
load_from:
- python_file:
relative_path: my_module/repo.py
location_name: my_module
from datetime import datetime
from dagster import repository, daily_schedule
from my_module.config import TIMESTAMP_FORMAT
from my_module.pipelines import my_pipeline
@daily_schedule(pipeline_name="my_pipeline", start_date=datetime(2020, 11, 26))
def my_daily_schedule(date):
return {
"solids": {
"solid_1": {
"config": {"timestamp": date.strftime(TIMESTAMP_FORMAT)}
},
"solid_2": {
"config": {"timestamp": date.strftime(TIMESTAMP_FORMAT)}
},
}
}
@repository(name="my_module")
def repo():
return [my_pipeline, my_daily_schedule]
daniel
12/07/2020, 1:42 PMesztermarton
12/07/2020, 1:43 PMdaniel
12/07/2020, 1:47 PMesztermarton
12/07/2020, 3:03 PMdagster schedule debug
. It's like nothing happens 🤔daniel
12/07/2020, 3:27 PMesztermarton
12/07/2020, 3:42 PM