diff --git a/Dockerfile b/Dockerfile index 95139ed..4aeaead 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ LABEL org.opencontainers.image.title="AlertFlow" LABEL org.opencontainers.image.authors="InfoDengue Team" LABEL org.opencontainers.image.source="https://github.com/AlertaDengue/AlertFlow" LABEL org.opencontainers.image.version="latest" -LABEL org.opencontainers.image.description="Airflow containers system for AlertaDengue" +LABEL org.opencontainers.image.description="AlertFlow containers system for AlertaDengue" USER root @@ -34,13 +34,32 @@ RUN apt-get update \ wget \ gettext \ build-essential \ + libffi-dev \ + libreadline-dev \ + sqlite3 \ + libsqlite3-dev \ + zlib1g-dev \ && rm -rf /var/lib/apt/lists/* +RUN cd /tmp \ + && wget https://www.python.org/ftp/python/3.12.3/Python-3.12.3.tgz \ + && tar -xf Python-3.12.3.tgz \ + && cd Python-3.12.3 \ + && ./configure --enable-optimizations --prefix=/opt/pysus \ + && make -j$(nproc) \ + && make altinstall \ + && rm -rf /tmp/Python-3.12.3* + +RUN /opt/pysus/bin/python3.12 -m ensurepip --default-pip \ + && /opt/pysus/bin/pip3.12 install --upgrade pip setuptools wheel \ + && /opt/pysus/bin/pip3.12 install pendulum apache-airflow-task-sdk \ + && /opt/pysus/bin/pip3.12 install "pysus==2.0.4" + RUN addgroup --gid ${HOST_GID} airflow \ && usermod -u ${HOST_UID} -g ${HOST_GID} -d /home/airflow -s /bin/bash airflow \ && echo "airflow ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/airflow \ && chmod 0440 /etc/sudoers.d/airflow \ - && chown -R ${HOST_UID}:${HOST_GID} ${AIRFLOW_HOME}/ /opt/airflow/ + && chown -R ${HOST_UID}:${HOST_GID} ${AIRFLOW_HOME}/ /opt/airflow/ /opt/pysus/ ENV PATH "$PATH:/home/airflow/.local/bin" ENV PATH "$PATH:/usr/bin/dirname" @@ -56,6 +75,6 @@ RUN curl -sSL https://install.python-poetry.org | python3 WORKDIR ${AIRFLOW_HOME} RUN poetry config virtualenvs.create false \ - && poetry install --no-root --only main --no-root + && poetry install --no-root --only main ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/alertflow/dags/pysus.py b/alertflow/dags/pysus.py new file mode 100644 index 0000000..5c9c31c --- /dev/null +++ b/alertflow/dags/pysus.py @@ -0,0 +1,36 @@ +import pendulum +from airflow import DAG +from airflow.decorators import task +from airflow.sdk import Variable + +with DAG( + dag_id="PYSUS", + schedule=None, + start_date=pendulum.datetime(2026, 1, 1), + catchup=False, +) as dag: + ACCESS = Variable.get("pysus_s3_access_key", deserialize_json=True) + SECRET = Variable.get("pysus_s3_secret_key", deserialize_json=True) + DADOSGOV = Variable.get("pysus_dadosgov_token", deserialize_json=True) + + @task.external_python( + task_id="pysus_update_s3_files", + python="/opt/pysus/bin/python3.12", + ) + def callable_external_python(access_key: str, secret_key: str): + import sys + import time + + print(access_key) + print(secret_key) + print(f"Running task via {sys.executable}") + print("Sleeping") + for _ in range(4): + print("Please wait...", flush=True) + time.sleep(1) + print("Finished") + + external_python_task = callable_external_python( + ACCESS["PYSUS_S3_ACCESS_KEY"], + SECRET["PYSUS_S3_SECRET_KEY"], + ) diff --git a/docker-compose.yaml b/docker-compose.yaml index 2c26b50..2180934 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,6 +33,9 @@ x-airflow-common: AIRFLOW_HOME: ${AIRFLOW_HOME:-/opt/airflow} AIRFLOW_VAR_PSQL_MAIN_URI: '{"PSQL_MAIN_URI":"${PSQL_URI_MAIN}"}' AIRFLOW_VAR_CDSAPI_KEY: '{"CDSAPI_KEY":"${CDSAPI_KEY}"}' + AIRFLOW_VAR_PYSUS_S3_ACCESS_KEY: '{"PYSUS_S3_ACCESS_KEY":"${PYSUS_S3_ACCESS_KEY}"}' + AIRFLOW_VAR_PYSUS_S3_SECRET_KEY: '{"PYSUS_S3_SECRET_KEY":"${PYSUS_S3_SECRET_KEY}"}' + AIRFLOW_VAR_PYSUS_DADOSGOV_TOKEN: '{"PYSUS_DADOSGOV_TOKEN":"${PYSUS_DADOSGOV_TOKEN}"}' # -- AIRFLOW__SMTP__SMTP_HOST: ${AIRFLOW__SMTP__SMTP_HOST:-''} AIRFLOW__SMTP__SMTP_STARTTLS: ${AIRFLOW__SMTP__SMTP_STARTTLS:-true}