diff --git a/.dockerignore b/.dockerignore index df08c066ce3b4..401e8fff9fb40 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,6 +39,7 @@ !task-sdk/ !airflow-ctl/ !go-sdk/ +!java-sdk/ # Add all "test" distributions !tests diff --git a/.github/workflows/ci-amd.yml b/.github/workflows/ci-amd.yml index 7c4f563007222..21eb17e81267f 100644 --- a/.github/workflows/ci-amd.yml +++ b/.github/workflows/ci-amd.yml @@ -54,6 +54,12 @@ env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} + # Keep these in sync: + # - jvmTarget, languageVersion, and sourceCompatibility in java-sdk/build.gradle.kts + # - TEMURIN_VERSION in scripts/docker/install_jdk.sh + # - JAVA_VERSION in .github/workflows/ci-amd.yml and .github/workflows/ci-arm.yml + # - java-version in .github/workflows/codeql-analysis.yml + JAVA_VERSION: '11' SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} VERBOSE: "true" @@ -974,18 +980,49 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - # keep this in sync with the jvmTarget in java-sdk/build.gradle.kts - name: Setup Java uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 with: distribution: 'temurin' - java-version: '11' + java-version: ${{ env.JAVA_VERSION }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: Run Java SDK tests working-directory: ./java-sdk run: ./gradlew test + build-java-sdk-docs: + name: "Java SDK docs" + needs: [build-info] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + timeout-minutes: 30 + permissions: + contents: read + packages: read + if: needs.build-info.outputs.run-java-sdk-tests == 'true' + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: "Build Java SDK Javadoc" + run: breeze build-docs --sdk-docs-only --sdk=java + - name: "Upload Javadoc artifact" + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: java-sdk-docs + path: generated/_build/docs/java-sdk/ + retention-days: 7 + if-no-files-found: error + overwrite: 'true' + tests-airflow-ctl: name: "Airflow CTL tests" uses: ./.github/workflows/airflow-distributions-tests.yml @@ -1040,6 +1077,7 @@ jobs: - tests-airflow-ctl - tests-go-sdk - tests-java-sdk + - build-java-sdk-docs - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml index b750c4a949c14..9250c092b8e3d 100644 --- a/.github/workflows/ci-arm.yml +++ b/.github/workflows/ci-arm.yml @@ -44,6 +44,12 @@ env: GITHUB_REPOSITORY: ${{ github.repository }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} GITHUB_USERNAME: ${{ github.actor }} + # Keep these in sync: + # - jvmTarget, languageVersion, and sourceCompatibility in java-sdk/build.gradle.kts + # - TEMURIN_VERSION in scripts/docker/install_jdk.sh + # - JAVA_VERSION in .github/workflows/ci-amd.yml and .github/workflows/ci-arm.yml + # - java-version in .github/workflows/codeql-analysis.yml + JAVA_VERSION: '11' SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} VERBOSE: "true" @@ -964,18 +970,49 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - # keep this in sync with the jvmTarget in java-sdk/build.gradle.kts - name: Setup Java uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 with: distribution: 'temurin' - java-version: '11' + java-version: ${{ env.JAVA_VERSION }} - name: "Cleanup dist files" run: rm -fv ./dist/* - name: Run Java SDK tests working-directory: ./java-sdk run: ./gradlew test + build-java-sdk-docs: + name: "Java SDK docs" + needs: [build-info] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + timeout-minutes: 30 + permissions: + contents: read + packages: read + if: needs.build-info.outputs.run-java-sdk-tests == 'true' + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: "Build Java SDK Javadoc" + run: breeze build-docs --sdk-docs-only --sdk=java + - name: "Upload Javadoc artifact" + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: java-sdk-docs + path: generated/_build/docs/java-sdk/ + retention-days: 7 + if-no-files-found: error + overwrite: 'true' + tests-airflow-ctl: name: "Airflow CTL tests" uses: ./.github/workflows/airflow-distributions-tests.yml @@ -1030,6 +1067,7 @@ jobs: - tests-airflow-ctl - tests-go-sdk - tests-java-sdk + - build-java-sdk-docs - tests-with-lowest-direct-resolution-core - tests-with-lowest-direct-resolution-providers uses: ./.github/workflows/finalize-tests.yml diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index ce46330319a65..8ef9ddbe36097 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -96,7 +96,11 @@ jobs: with: persist-credentials: false - # keep this in sync with the jvmTarget in java-sdk/build.gradle.kts + # Keep these in sync: + # - jvmTarget, languageVersion, and sourceCompatibility in java-sdk/build.gradle.kts + # - TEMURIN_VERSION in scripts/docker/install_jdk.sh + # - JAVA_VERSION in .github/workflows/ci-amd.yml and .github/workflows/ci-arm.yml + # - java-version in .github/workflows/codeql-analysis.yml - name: Setup Java if: matrix.language == 'java' uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 diff --git a/.github/workflows/publish-docs-to-s3.yml b/.github/workflows/publish-docs-to-s3.yml index 8c5d152931a94..1192283a081e4 100644 --- a/.github/workflows/publish-docs-to-s3.yml +++ b/.github/workflows/publish-docs-to-s3.yml @@ -345,8 +345,56 @@ jobs: if-no-files-found: 'error' overwrite: 'true' + build-java-sdk-docs: + needs: [build-info] + timeout-minutes: 30 + name: "Build Java SDK Javadoc" + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + outputs: + built: ${{ steps.sdk-present.outputs.exists }} + steps: + - name: "Checkout ${{ inputs.ref }}" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + ref: ${{ inputs.ref }} + - name: "Check whether the Java SDK exists at this ref" + id: sdk-present + env: + REF: ${{ inputs.ref }} + run: | + if [[ -f "java-sdk/gradlew" ]]; then + echo "exists=true" >> "${GITHUB_OUTPUT}" + else + echo "exists=false" >> "${GITHUB_OUTPUT}" + echo "java-sdk/ not found at ref ${REF} — skipping Javadoc build." + fi + - name: "Install Breeze" + if: steps.sdk-present.outputs.exists == 'true' + uses: ./.github/actions/breeze + - name: "Build Java SDK Javadoc" + if: steps.sdk-present.outputs.exists == 'true' + run: breeze build-docs --sdk-docs-only --sdk=java + - name: "Upload Java SDK docs" + if: steps.sdk-present.outputs.exists == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: java-sdk-docs + path: generated/_build/docs/java-sdk/ + retention-days: '7' + if-no-files-found: 'error' + overwrite: 'true' + publish-docs-to-s3: - needs: [build-docs, build-info] + needs: [build-docs, build-java-sdk-docs, build-info] name: "Publish documentation to S3" permissions: id-token: write @@ -385,6 +433,12 @@ jobs: path: /mnt/_build - name: "Move docs to generated folder" run: mv /mnt/_build generated/_build + - name: "Download Java SDK docs artifact" + if: needs.build-java-sdk-docs.outputs.built == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: java-sdk-docs + path: generated/_build/docs/java-sdk - name: "Make sure SBOM dir exists and has the right permissions" run: | sudo mkdir -vp ./files/sbom @@ -416,8 +470,10 @@ jobs: - name: "Publish docs to /mnt/airflow-site directory using ${{ inputs.ref }} reference breeze" env: INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} + JAVA_SDK_DOCS: ${{ needs.build-java-sdk-docs.outputs.built == 'true' && 'java-sdk' || '' }} run: > - breeze release-management publish-docs --override-versioned --run-in-parallel ${INCLUDE_DOCS} + breeze release-management publish-docs --override-versioned --run-in-parallel + ${INCLUDE_DOCS} ${JAVA_SDK_DOCS} - name: Check disk space available run: df -H - name: "Update watermarks" diff --git a/Dockerfile.ci b/Dockerfile.ci index 2379df5129777..144b118ad69f8 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1222,6 +1222,10 @@ function environment_initialization() { export AIRFLOW__SCHEDULER__GO_WORKER=True fi + if [[ ${INSTALL_SDK_JAVA=} == "true" ]]; then + "${AIRFLOW_SOURCES}/scripts/docker/install_jdk.sh" + fi + RUN_TESTS=${RUN_TESTS:="false"} CI=${CI:="false"} diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index f35a9a4f07244..8600627f06283 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -1,4 +1,4 @@ - +