From 8f4d7b9b21fde9742468314892825b7e2998bfaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 22 Oct 2025 22:16:34 +0100 Subject: [PATCH 01/21] pom: Upgrade avro version from 1.7.7 to 1.11.4 #TASK-7809 #TASK-7962 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index eb71ba27..75fc8969 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.14.3 3.14.0 1.7.36 - 1.7.7 + 1.11.4 4.11.5 8.8.2 1.69 From 353ca563bf09f98f4149e814cd4244fd00c67fa9 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 27 Oct 2025 16:14:49 +0100 Subject: [PATCH 02/21] cicd: change hdp3.1 for hdi5.1 #TASK-7809 --- .github/workflows/test-xetabase-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 0b060f58..5ab67da5 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -101,7 +101,7 @@ jobs: run: | cd opencga-enterprise ln -s ../opencga opencga-home - ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -c localhost:27018 -H hdp3.1 + ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -c localhost:27018 -H hdi5.1 - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: From da5fd537f8970ec1d9b8c970a7a5b5dae05bfbb0 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 28 Oct 2025 13:07:01 +0100 Subject: [PATCH 03/21] Added prepare hadoop --- .github/workflows/test-xetabase-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 4dc74452..0dc4b54c 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -90,7 +90,7 @@ jobs: run: | cd opencga-enterprise ln -s ../opencga opencga-home - ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} -H hdi5.1 + ./build.sh --test --test-level runShortTests --prepare-branches --test-save-reports --task ${{ inputs.task }} --storage-hadoop ${{ vars.HADOOP_FLAVOUR }} --prepare-hadoop - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: From 852609b7f456c385c993c6a532e7dc41905c4505 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 3 Nov 2025 12:18:33 +0100 Subject: [PATCH 04/21] Prepare Hadoop profile #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index c4e836b5..71897bc8 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -10,6 +10,14 @@ on: type: string required: false default: "build-folder" + needs_hadoop_preparation: + type: boolean + required: false + default: false + hadoop_flavour: + type: string + required: false + default: "" outputs: version: description: "Project version" @@ -37,6 +45,13 @@ jobs: chmod +x ./.github/workflows/scripts/get_same_branch.sh ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} fi + - name: Prepare Hadoop profile + if: ${{ inputs.needs_hadoop_preparation == true }} + run: | + chmod +x ./.github/workflows/scripts/prepare_hadoop.sh + ./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour ${{ inputs.hadoop }} + env: + THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }} - name: Maven Build (skip tests) run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress - uses: actions/upload-artifact@v4 From a713a78042441df6b52b70b4d5c1311740af566c Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 3 Nov 2025 12:48:09 +0100 Subject: [PATCH 05/21] Prepare Hadoop profile #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 71897bc8..99f09abe 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -17,7 +17,7 @@ on: hadoop_flavour: type: string required: false - default: "" + default: "hdi5.1" outputs: version: description: "Project version" @@ -49,9 +49,10 @@ jobs: if: ${{ inputs.needs_hadoop_preparation == true }} run: | chmod +x ./.github/workflows/scripts/prepare_hadoop.sh - ./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour ${{ inputs.hadoop }} + ./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}" env: THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }} + - name: Maven Build (skip tests) run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress - uses: actions/upload-artifact@v4 From 46d1bedea4c25c0d7db9c10af76803a0be72b7e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Mon, 3 Nov 2025 16:43:54 +0000 Subject: [PATCH 06/21] cicd: Add param to control upload-artifact. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 99f09abe..1a78d7e4 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -10,6 +10,10 @@ on: type: string required: false default: "build-folder" + upload-artifact: + type: boolean + required: false + default: true needs_hadoop_preparation: type: boolean required: false @@ -56,6 +60,7 @@ jobs: - name: Maven Build (skip tests) run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress - uses: actions/upload-artifact@v4 + if: ${{ inputs.upload-artifact == true }} with: name: ${{ inputs.build_folder }} path: build From 6539d73116026aef3b3e0c579c21ee6fdaa30376 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 3 Nov 2025 18:06:04 +0100 Subject: [PATCH 07/21] Default value should be empty --- .github/workflows/build-java-app-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 99f09abe..17ad0830 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -17,7 +17,7 @@ on: hadoop_flavour: type: string required: false - default: "hdi5.1" + default: "" outputs: version: description: "Project version" From 49d540e33102bd84aabab73461fd635bb97ba671 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 4 Nov 2025 11:59:22 +0100 Subject: [PATCH 08/21] Added Cache local Maven repository step #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 59c657fc..1dbc8c86 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -43,6 +43,14 @@ jobs: distribution: 'temurin' java-version: '8' cache: 'maven' + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ github.ref }}-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-maven-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/*.jar') }} + restore-keys: | + ${{ github.ref }}-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-maven- + save-always: true - name: Install dependencies branches run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then From 34f6f28b51246abd4fa1f126f51647fe05536699 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Tue, 4 Nov 2025 12:29:16 +0100 Subject: [PATCH 09/21] cicd:Fix Input 'save-always' has been deprecated #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 1dbc8c86..2f953841 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -50,7 +50,6 @@ jobs: key: ${{ github.ref }}-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-maven-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/*.jar') }} restore-keys: | ${{ github.ref }}-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-maven- - save-always: true - name: Install dependencies branches run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then From 344d2b5287392adef44ff12bf1a8c64b6843d0a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 4 Nov 2025 13:09:08 +0000 Subject: [PATCH 10/21] cicd: Add output cache-key #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 2f953841..ac903081 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -47,9 +47,9 @@ jobs: uses: actions/cache@v4 with: path: ~/.m2/repository - key: ${{ github.ref }}-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-maven-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/*.jar') }} + key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ github.sha }} restore-keys: | - ${{ github.ref }}-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-maven- + ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- - name: Install dependencies branches run: | if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then @@ -77,3 +77,9 @@ jobs: echo "version=`mvn help:evaluate -q -Dexpression=project.version -DforceStdout`" >> $GITHUB_OUTPUT - name: test-version-from-check run: echo "Project version is " ${{ steps.get_project_version.outputs.version }} + + +outputs: + cache-key: + description: "The key of the maven cache created" + value: ${{ jobs.build-workflow.steps.cache-local-maven-repository.outputs.cache-hit-key }} \ No newline at end of file From 31100ad1dddb2a3b3b66d8075a8f9f9886129719 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 4 Nov 2025 13:16:39 +0000 Subject: [PATCH 11/21] cicd: Fix cache-key output. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index ac903081..ed1a528b 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -33,6 +33,7 @@ jobs: runs-on: ${{ vars.UBUNTU_VERSION }} outputs: version: ${{ steps.get_project_version.outputs.version }} + cache-key: ${{ steps.cache.outputs.cache-hit-key }} steps: - uses: actions/checkout@v4 with: @@ -44,6 +45,7 @@ jobs: java-version: '8' cache: 'maven' - name: Cache local Maven repository + id: cache uses: actions/cache@v4 with: path: ~/.m2/repository @@ -78,8 +80,3 @@ jobs: - name: test-version-from-check run: echo "Project version is " ${{ steps.get_project_version.outputs.version }} - -outputs: - cache-key: - description: "The key of the maven cache created" - value: ${{ jobs.build-workflow.steps.cache-local-maven-repository.outputs.cache-hit-key }} \ No newline at end of file From ebfc19c5dd6cdd1dc15fd7d36d83586a9cf6bbba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 4 Nov 2025 13:38:56 +0000 Subject: [PATCH 12/21] cicd: Add missing output cache-key. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index ed1a528b..9f4dcd83 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -26,6 +26,9 @@ on: version: description: "Project version" value: ${{ jobs.build-workflow.outputs.version }} + cache-key: + description: "Cache key used for Maven repository" + value: ${{ jobs.build-workflow.outputs.cache-key }} jobs: build-workflow: From ba1ec8fcb21e506458ac3f82baf67716602e48fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Tue, 4 Nov 2025 13:51:00 +0000 Subject: [PATCH 13/21] cicd: Ensure correct cache-key is used. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 9f4dcd83..7b923c8f 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ vars.UBUNTU_VERSION }} outputs: version: ${{ steps.get_project_version.outputs.version }} - cache-key: ${{ steps.cache.outputs.cache-hit-key }} + cache-key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ github.sha }} steps: - uses: actions/checkout@v4 with: From c4af62fd53c3b746f7a28bd9e250324dbffc7730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 5 Nov 2025 10:03:36 +0000 Subject: [PATCH 14/21] cicd: Improve cache-key name to include dependencies sha. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 7b923c8f..42b57ba7 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -36,7 +36,7 @@ jobs: runs-on: ${{ vars.UBUNTU_VERSION }} outputs: version: ${{ steps.get_project_version.outputs.version }} - cache-key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ github.sha }} + cache-key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} steps: - uses: actions/checkout@v4 with: @@ -47,19 +47,27 @@ jobs: distribution: 'temurin' java-version: '8' cache: 'maven' + - name: Clone dependencies + id: clone_dependencies + run: | + if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/get_same_branch.sh + export DEPENDENCIES_SHA=${{ github.sha }} + ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + fi - name: Cache local Maven repository id: cache uses: actions/cache@v4 with: path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ github.sha }} + key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} restore-keys: | ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- - - name: Install dependencies branches + - name: Compile dependencies run: | - if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then - chmod +x ./.github/workflows/scripts/get_same_branch.sh - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + if [ -f "./.github/workflows/scripts/compile_same_branch.sh" ]; then + chmod +x ./.github/workflows/scripts/compile_same_branch.sh + ./.github/workflows/scripts/compile_same_branch.sh fi - name: Prepare Hadoop profile if: ${{ inputs.needs_hadoop_preparation == true }} From 5fa4e891bd14320adc3052457f49186a53c8ec84 Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 6 Nov 2025 10:06:19 +0100 Subject: [PATCH 15/21] cicd: Added Show which cache was used step for logs #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 42b57ba7..cdcf0132 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -63,6 +63,11 @@ jobs: key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} restore-keys: | ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- + - name: Show which cache was used + run: | + echo "hit=${{ steps.maven-cache.outputs['cache-hit'] }}" + echo "primary=${{ steps.maven-cache.outputs['cache-primary-key'] }}" + echo "matched=${{ steps.maven-cache.outputs['cache-matched-key'] }}" - name: Compile dependencies run: | if [ -f "./.github/workflows/scripts/compile_same_branch.sh" ]; then From dd7aedbbf05b05659f7d04c9b25d84bf07385afc Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Thu, 6 Nov 2025 10:08:38 +0100 Subject: [PATCH 16/21] cicd: Added Show which cache was used step for logs #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index cdcf0132..fa7eafc7 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -65,9 +65,9 @@ jobs: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- - name: Show which cache was used run: | - echo "hit=${{ steps.maven-cache.outputs['cache-hit'] }}" - echo "primary=${{ steps.maven-cache.outputs['cache-primary-key'] }}" - echo "matched=${{ steps.maven-cache.outputs['cache-matched-key'] }}" + echo "hit=${{ steps.cache.outputs['cache-hit'] }}" + echo "primary=${{ steps.cache.outputs['cache-primary-key'] }}" + echo "matched=${{ steps.cache.outputs['cache-matched-key'] }}" - name: Compile dependencies run: | if [ -f "./.github/workflows/scripts/compile_same_branch.sh" ]; then From e9a09d2721038d73085aea5cc31b35dedcc15f1e Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Fri, 7 Nov 2025 12:22:22 +0100 Subject: [PATCH 17/21] Added THIRDPARTY_READ_TOKEN #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 5 ----- .github/workflows/test-xetabase-workflow.yml | 2 ++ 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index fa7eafc7..42b57ba7 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -63,11 +63,6 @@ jobs: key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} restore-keys: | ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- - - name: Show which cache was used - run: | - echo "hit=${{ steps.cache.outputs['cache-hit'] }}" - echo "primary=${{ steps.cache.outputs['cache-primary-key'] }}" - echo "matched=${{ steps.cache.outputs['cache-matched-key'] }}" - name: Compile dependencies run: | if [ -f "./.github/workflows/scripts/compile_same_branch.sh" ]; then diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 0dc4b54c..b7429557 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -91,6 +91,8 @@ jobs: cd opencga-enterprise ln -s ../opencga opencga-home ./build.sh --test --test-level runShortTests --prepare-branches --test-save-reports --task ${{ inputs.task }} --storage-hadoop ${{ vars.HADOOP_FLAVOUR }} --prepare-hadoop + env: + THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }} - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: From 06cd49472d2db574ff72622cafc8867ca4ce56c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 7 Nov 2025 11:53:57 +0000 Subject: [PATCH 18/21] cicd: Centralize "get_same_branch.sh" and "compile_same_branch.sh". #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 39 ++++++++++++++---- .../workflows/scripts/compile_same_branch.sh | 23 +++++++++++ .github/workflows/scripts/get_same_branch.sh | 40 +++++++++++++++++++ 3 files changed, 95 insertions(+), 7 deletions(-) create mode 100755 .github/workflows/scripts/compile_same_branch.sh create mode 100755 .github/workflows/scripts/get_same_branch.sh diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index fa7eafc7..d04e2776 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -10,10 +10,15 @@ on: type: string required: false default: "build-folder" - upload-artifact: + upload_artifact: type: boolean required: false default: true + dependency_repos: + # Comma-separated list of dependency repositories to clone and build before building the main project. + type: string + required: false + default: "" needs_hadoop_preparation: type: boolean required: false @@ -22,6 +27,10 @@ on: type: string required: false default: "" + java_commons_libs_branch: + type: string + required: false + default: "develop" outputs: version: description: "Project version" @@ -39,8 +48,22 @@ jobs: cache-key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} steps: - uses: actions/checkout@v4 + id: "checkout-main" + ## This checkout pulls the code of the repository where this workflow is being used with: fetch-depth: '10' + - uses: actions/checkout@v4 + if: ${{ inputs.dependency_repos != "" }} + id: "checkout-java-common-libs" + ## This checkout pulls the code of the java-common-libs repository to get the scripts + ## Checkout to "java-common-libs" folder. + ## Filter by ".github" folder + with: + repository: opencb/java-common-libs + ref: ${{ inputs.java_commons_libs_branch }} + path: java-common-libs + filter: ".github" + fetch-depth: '1' - name: Set up JDK 8 uses: actions/setup-java@v4 with: @@ -49,11 +72,12 @@ jobs: cache: 'maven' - name: Clone dependencies id: clone_dependencies + if: ${{ inputs.dependency_repos != "" }} run: | - if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then - chmod +x ./.github/workflows/scripts/get_same_branch.sh + if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh export DEPENDENCIES_SHA=${{ github.sha }} - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + java-common-libs/.github/workflows/scripts/get_same_branch.sh "${{ github.ref_name }}" "${{ inputs.dependency_repos }}" fi - name: Cache local Maven repository id: cache @@ -69,10 +93,11 @@ jobs: echo "primary=${{ steps.cache.outputs['cache-primary-key'] }}" echo "matched=${{ steps.cache.outputs['cache-matched-key'] }}" - name: Compile dependencies + if: ${{ inputs.dependency_repos != "" }} run: | - if [ -f "./.github/workflows/scripts/compile_same_branch.sh" ]; then - chmod +x ./.github/workflows/scripts/compile_same_branch.sh - ./.github/workflows/scripts/compile_same_branch.sh + if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then + chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh + java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}" fi - name: Prepare Hadoop profile if: ${{ inputs.needs_hadoop_preparation == true }} diff --git a/.github/workflows/scripts/compile_same_branch.sh b/.github/workflows/scripts/compile_same_branch.sh new file mode 100755 index 00000000..3ac1c303 --- /dev/null +++ b/.github/workflows/scripts/compile_same_branch.sh @@ -0,0 +1,23 @@ + +WORKSPACE=${WORKSPACE:-/home/runner/work/} + + +function compile() { + local REPO=$1 + if [ ! -d "${WORKSPACE}/$REPO" ]; then + echo "Directory ${WORKSPACE}/$REPO does not exist. Skip compile" + return 0; + fi + echo "::group::Compiling '$REPO' project from branch $BRANCH_NAME" + cd "${WORKSPACE}/$REPO" || exit 2 + mvn clean install -DskipTests --no-transfer-progress + echo "::endgroup::" +} + +## Comma separated list of repos to compile +REPOS=$1 + +IFS=',' read -ra REPO_ARRAY <<<"$REPOS" +for REPO in "${REPO_ARRAY[@]}"; do + compile "$REPO" +done \ No newline at end of file diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh new file mode 100755 index 00000000..37b056fb --- /dev/null +++ b/.github/workflows/scripts/get_same_branch.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +BRANCH_NAME=$1 +## Comma separated list of repos to checkout +REPOS=$2 +DEPENDENCIES_SHA=${DEPENDENCIES_SHA:-""} +WORKSPACE=${WORKSPACE:-/home/runner/work/} + +if [[ -z "$BRANCH_NAME" ]]; then + echo "The first parameter is mandatory and must be a valid branch name." + exit 1 +fi + +function checkout(){ + local REPO=$1 + echo "::group::Installing '$REPO' project from branch $BRANCH_NAME" + cd "${WORKSPACE}" || exit 2 + git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME" + if [ -d "./$REPO" ]; then + cd "$REPO" || exit 2 + DEPENDENCIES_SHA=${DEPENDENCIES_SHA}:$(git rev-parse HEAD) + echo "Branch name $BRANCH_NAME already exists." + else + echo "Branch name $BRANCH_NAME does not exist in $REPO repository. Skipping installation." + fi + echo "::endgroup::" +} + +IFS=',' read -ra REPO_ARRAY <<<"$REPOS" +for REPO in "${REPO_ARRAY[@]}"; do + checkout "$REPO" +done + +## Apply sha1 to DEPENDENCIES_SHA if contains `:` +if [[ "$DEPENDENCIES_SHA" == *":"* ]]; then + DEPENDENCIES_SHA=$(echo -n "$DEPENDENCIES_SHA" | sha1sum | awk '{print $1}') +fi + +## Export DEPENDENCIES_SHA as github output +echo "dependencies_sha=$DEPENDENCIES_SHA" >> "$GITHUB_OUTPUT" \ No newline at end of file From 52f3008b5421540e8850e864c7d05c72337a0242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 7 Nov 2025 12:10:25 +0000 Subject: [PATCH 19/21] cicd: Use cache_key at maven-deploy. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 6 +++--- .../workflows/deploy-maven-repository-workflow.yml | 11 +++++++++++ .github/workflows/develop.yml | 2 ++ .github/workflows/release.yml | 2 ++ .github/workflows/test-analysis.yml | 12 ++++++++++++ .github/workflows/test-xetabase-workflow.yml | 8 ++++++++ 6 files changed, 38 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 8bd14554..254317f6 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -35,9 +35,9 @@ on: version: description: "Project version" value: ${{ jobs.build-workflow.outputs.version }} - cache-key: + cache_key: description: "Cache key used for Maven repository" - value: ${{ jobs.build-workflow.outputs.cache-key }} + value: ${{ jobs.build-workflow.outputs.cache_key }} jobs: build-workflow: @@ -45,7 +45,7 @@ jobs: runs-on: ${{ vars.UBUNTU_VERSION }} outputs: version: ${{ steps.get_project_version.outputs.version }} - cache-key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} + cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} steps: - uses: actions/checkout@v4 id: "checkout-main" diff --git a/.github/workflows/deploy-maven-repository-workflow.yml b/.github/workflows/deploy-maven-repository-workflow.yml index e34012dc..aca09c51 100644 --- a/.github/workflows/deploy-maven-repository-workflow.yml +++ b/.github/workflows/deploy-maven-repository-workflow.yml @@ -6,6 +6,9 @@ on: maven_opts: type: string required: false + cache_key: + type: string + required: false secrets: MAVEN_GPG_PASSPHRASE: required: true @@ -32,6 +35,14 @@ jobs: server-password: MAVEN_NEXUS_PASSWORD # env variable for token in deploy gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} # Value of the GPG private key to import gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + - name: Cache local Maven repository + uses: actions/cache/restore@v4 + if: ${{ inputs.cache_key != '' }} + with: + path: ~/.m2/repository + key: ${{ inputs.cache_key }} + ## Force cache hit to avoid analyzing with incomplete dependencies + fail-on-cache-miss: true - name: Deploy to Maven Central repository run: mvn clean deploy -DskipTests -Pdeploy-maven ${{ inputs.maven_opts }} --no-transfer-progress env: diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 3029b0fd..0cc9c687 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -14,3 +14,5 @@ jobs: uses: ./.github/workflows/deploy-maven-repository-workflow.yml needs: build secrets: inherit + with: + cache_key: ${{ needs.build.outputs.cache_key }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5ff2bde..37681d7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,8 @@ jobs: uses: ./.github/workflows/deploy-maven-repository-workflow.yml needs: build secrets: inherit + with: + cache_key: ${{ needs.build.outputs.cache_key }} release: uses: ./.github/workflows/release-github-workflow.yml diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 3af76372..cc2efef7 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -2,6 +2,10 @@ name: Build and test the project on: workflow_call: + inputs: + cache_key: + type: string + required: false secrets: SONAR_TOKEN: required: true @@ -20,6 +24,14 @@ jobs: distribution: 'temurin' java-version: '11' cache: 'maven' + - name: Cache local Maven repository + uses: actions/cache/restore@v4 + if: ${{ inputs.cache_key != '' }} + with: + path: ~/.m2/repository + key: ${{ inputs.cache_key }} + ## Force cache hit to avoid analyzing with incomplete dependencies + fail-on-cache-miss: true - name: Start MongoDB v6.0 uses: supercharge/mongodb-github-action@1.8.0 with: diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index b7429557..0d18858b 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -86,6 +86,14 @@ jobs: env: SSH_HOST: ${{ env.SSH_HOST }} SSH_PORT: ${{ env.SSH_PORT }} + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}- - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, opencga and opencga-enterprise run: | cd opencga-enterprise From 588f8050e4ce3d923111a96d0658398876e39455 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 7 Nov 2025 12:20:41 +0000 Subject: [PATCH 20/21] cicd: Fix double to single quote. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 254317f6..14d472dc 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -53,7 +53,7 @@ jobs: with: fetch-depth: '10' - uses: actions/checkout@v4 - if: ${{ inputs.dependency_repos != "" }} + if: ${{ inputs.dependency_repos != '' }} id: "checkout-java-common-libs" ## This checkout pulls the code of the java-common-libs repository to get the scripts ## Checkout to "java-common-libs" folder. @@ -72,7 +72,7 @@ jobs: cache: 'maven' - name: Clone dependencies id: clone_dependencies - if: ${{ inputs.dependency_repos != "" }} + if: ${{ inputs.dependency_repos != '' }} run: | if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh @@ -88,7 +88,7 @@ jobs: restore-keys: | ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- - name: Compile dependencies - if: ${{ inputs.dependency_repos != "" }} + if: ${{ inputs.dependency_repos != '' }} run: | if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh From db0aa0ead0c31a5ebaa900fce70ead7fcddb0355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Fri, 7 Nov 2025 12:27:20 +0000 Subject: [PATCH 21/21] cicd: Remove checkout filter. #TASK-7809 --- .github/workflows/build-java-app-workflow.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index 14d472dc..aaa7cbf1 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -48,12 +48,14 @@ jobs: cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} steps: - uses: actions/checkout@v4 + name: Checkout main repository id: "checkout-main" ## This checkout pulls the code of the repository where this workflow is being used with: fetch-depth: '10' - uses: actions/checkout@v4 if: ${{ inputs.dependency_repos != '' }} + name: Checkout java-common-libs repository id: "checkout-java-common-libs" ## This checkout pulls the code of the java-common-libs repository to get the scripts ## Checkout to "java-common-libs" folder. @@ -62,7 +64,6 @@ jobs: repository: opencb/java-common-libs ref: ${{ inputs.java_commons_libs_branch }} path: java-common-libs - filter: ".github" fetch-depth: '1' - name: Set up JDK 8 uses: actions/setup-java@v4