diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml
index c4e836b5..aaa7cbf1 100644
--- a/.github/workflows/build-java-app-workflow.yml
+++ b/.github/workflows/build-java-app-workflow.yml
@@ -10,10 +10,34 @@ on:
type: string
required: false
default: "build-folder"
+ upload_artifact:
+ type: boolean
+ required: false
+ default: true
+ dependency_repos:
+ # Comma-separated list of dependency repositories to clone and build before building the main project.
+ type: string
+ required: false
+ default: ""
+ needs_hadoop_preparation:
+ type: boolean
+ required: false
+ default: false
+ hadoop_flavour:
+ type: string
+ required: false
+ default: ""
+ java_commons_libs_branch:
+ type: string
+ required: false
+ default: "develop"
outputs:
version:
description: "Project version"
value: ${{ jobs.build-workflow.outputs.version }}
+ cache_key:
+ description: "Cache key used for Maven repository"
+ value: ${{ jobs.build-workflow.outputs.cache_key }}
jobs:
build-workflow:
@@ -21,25 +45,68 @@ jobs:
runs-on: ${{ vars.UBUNTU_VERSION }}
outputs:
version: ${{ steps.get_project_version.outputs.version }}
+ cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
steps:
- uses: actions/checkout@v4
+ name: Checkout main repository
+ id: "checkout-main"
+ ## This checkout pulls the code of the repository where this workflow is being used
with:
fetch-depth: '10'
+ - uses: actions/checkout@v4
+ if: ${{ inputs.dependency_repos != '' }}
+ name: Checkout java-common-libs repository
+ id: "checkout-java-common-libs"
+ ## This checkout pulls the code of the java-common-libs repository to get the scripts
+ ## Checkout to "java-common-libs" folder.
+ ## Filter by ".github" folder
+ with:
+ repository: opencb/java-common-libs
+ ref: ${{ inputs.java_commons_libs_branch }}
+ path: java-common-libs
+ fetch-depth: '1'
- name: Set up JDK 8
uses: actions/setup-java@v4
with:
distribution: 'temurin'
java-version: '8'
cache: 'maven'
- - name: Install dependencies branches
+ - name: Clone dependencies
+ id: clone_dependencies
+ if: ${{ inputs.dependency_repos != '' }}
+ run: |
+ if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then
+ chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh
+ export DEPENDENCIES_SHA=${{ github.sha }}
+ java-common-libs/.github/workflows/scripts/get_same_branch.sh "${{ github.ref_name }}" "${{ inputs.dependency_repos }}"
+ fi
+ - name: Cache local Maven repository
+ id: cache
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
+ restore-keys: |
+ ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-
+ - name: Compile dependencies
+ if: ${{ inputs.dependency_repos != '' }}
run: |
- if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
- chmod +x ./.github/workflows/scripts/get_same_branch.sh
- ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+ if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then
+ chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh
+ java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}"
fi
+ - name: Prepare Hadoop profile
+ if: ${{ inputs.needs_hadoop_preparation == true }}
+ run: |
+ chmod +x ./.github/workflows/scripts/prepare_hadoop.sh
+ ./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}"
+ env:
+ THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
+
- name: Maven Build (skip tests)
run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress
- uses: actions/upload-artifact@v4
+ if: ${{ inputs.upload-artifact == true }}
with:
name: ${{ inputs.build_folder }}
path: build
@@ -49,3 +116,4 @@ jobs:
echo "version=`mvn help:evaluate -q -Dexpression=project.version -DforceStdout`" >> $GITHUB_OUTPUT
- name: test-version-from-check
run: echo "Project version is " ${{ steps.get_project_version.outputs.version }}
+
diff --git a/.github/workflows/deploy-maven-repository-workflow.yml b/.github/workflows/deploy-maven-repository-workflow.yml
index e34012dc..aca09c51 100644
--- a/.github/workflows/deploy-maven-repository-workflow.yml
+++ b/.github/workflows/deploy-maven-repository-workflow.yml
@@ -6,6 +6,9 @@ on:
maven_opts:
type: string
required: false
+ cache_key:
+ type: string
+ required: false
secrets:
MAVEN_GPG_PASSPHRASE:
required: true
@@ -32,6 +35,14 @@ jobs:
server-password: MAVEN_NEXUS_PASSWORD # env variable for token in deploy
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} # Value of the GPG private key to import
gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase
+ - name: Cache local Maven repository
+ uses: actions/cache/restore@v4
+ if: ${{ inputs.cache_key != '' }}
+ with:
+ path: ~/.m2/repository
+ key: ${{ inputs.cache_key }}
+ ## Force cache hit to avoid analyzing with incomplete dependencies
+ fail-on-cache-miss: true
- name: Deploy to Maven Central repository
run: mvn clean deploy -DskipTests -Pdeploy-maven ${{ inputs.maven_opts }} --no-transfer-progress
env:
diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml
index 3029b0fd..0cc9c687 100644
--- a/.github/workflows/develop.yml
+++ b/.github/workflows/develop.yml
@@ -14,3 +14,5 @@ jobs:
uses: ./.github/workflows/deploy-maven-repository-workflow.yml
needs: build
secrets: inherit
+ with:
+ cache_key: ${{ needs.build.outputs.cache_key }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c5ff2bde..37681d7c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -13,6 +13,8 @@ jobs:
uses: ./.github/workflows/deploy-maven-repository-workflow.yml
needs: build
secrets: inherit
+ with:
+ cache_key: ${{ needs.build.outputs.cache_key }}
release:
uses: ./.github/workflows/release-github-workflow.yml
diff --git a/.github/workflows/scripts/compile_same_branch.sh b/.github/workflows/scripts/compile_same_branch.sh
new file mode 100755
index 00000000..3ac1c303
--- /dev/null
+++ b/.github/workflows/scripts/compile_same_branch.sh
@@ -0,0 +1,23 @@
+
+WORKSPACE=${WORKSPACE:-/home/runner/work/}
+
+
+function compile() {
+ local REPO=$1
+ if [ ! -d "${WORKSPACE}/$REPO" ]; then
+ echo "Directory ${WORKSPACE}/$REPO does not exist. Skip compile"
+ return 0;
+ fi
+ echo "::group::Compiling '$REPO' project from branch $BRANCH_NAME"
+ cd "${WORKSPACE}/$REPO" || exit 2
+ mvn clean install -DskipTests --no-transfer-progress
+ echo "::endgroup::"
+}
+
+## Comma separated list of repos to compile
+REPOS=$1
+
+IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
+for REPO in "${REPO_ARRAY[@]}"; do
+ compile "$REPO"
+done
\ No newline at end of file
diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh
new file mode 100755
index 00000000..37b056fb
--- /dev/null
+++ b/.github/workflows/scripts/get_same_branch.sh
@@ -0,0 +1,40 @@
+#!/bin/bash
+
+BRANCH_NAME=$1
+## Comma separated list of repos to checkout
+REPOS=$2
+DEPENDENCIES_SHA=${DEPENDENCIES_SHA:-""}
+WORKSPACE=${WORKSPACE:-/home/runner/work/}
+
+if [[ -z "$BRANCH_NAME" ]]; then
+ echo "The first parameter is mandatory and must be a valid branch name."
+ exit 1
+fi
+
+function checkout(){
+ local REPO=$1
+ echo "::group::Installing '$REPO' project from branch $BRANCH_NAME"
+ cd "${WORKSPACE}" || exit 2
+ git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME"
+ if [ -d "./$REPO" ]; then
+ cd "$REPO" || exit 2
+ DEPENDENCIES_SHA=${DEPENDENCIES_SHA}:$(git rev-parse HEAD)
+ echo "Branch name $BRANCH_NAME already exists."
+ else
+ echo "Branch name $BRANCH_NAME does not exist in $REPO repository. Skipping installation."
+ fi
+ echo "::endgroup::"
+}
+
+IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
+for REPO in "${REPO_ARRAY[@]}"; do
+ checkout "$REPO"
+done
+
+## Apply sha1 to DEPENDENCIES_SHA if contains `:`
+if [[ "$DEPENDENCIES_SHA" == *":"* ]]; then
+ DEPENDENCIES_SHA=$(echo -n "$DEPENDENCIES_SHA" | sha1sum | awk '{print $1}')
+fi
+
+## Export DEPENDENCIES_SHA as github output
+echo "dependencies_sha=$DEPENDENCIES_SHA" >> "$GITHUB_OUTPUT"
\ No newline at end of file
diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml
index 3af76372..cc2efef7 100644
--- a/.github/workflows/test-analysis.yml
+++ b/.github/workflows/test-analysis.yml
@@ -2,6 +2,10 @@ name: Build and test the project
on:
workflow_call:
+ inputs:
+ cache_key:
+ type: string
+ required: false
secrets:
SONAR_TOKEN:
required: true
@@ -20,6 +24,14 @@ jobs:
distribution: 'temurin'
java-version: '11'
cache: 'maven'
+ - name: Cache local Maven repository
+ uses: actions/cache/restore@v4
+ if: ${{ inputs.cache_key != '' }}
+ with:
+ path: ~/.m2/repository
+ key: ${{ inputs.cache_key }}
+ ## Force cache hit to avoid analyzing with incomplete dependencies
+ fail-on-cache-miss: true
- name: Start MongoDB v6.0
uses: supercharge/mongodb-github-action@1.8.0
with:
diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml
index 3720d93c..0d18858b 100644
--- a/.github/workflows/test-xetabase-workflow.yml
+++ b/.github/workflows/test-xetabase-workflow.yml
@@ -86,11 +86,21 @@ jobs:
env:
SSH_HOST: ${{ env.SSH_HOST }}
SSH_PORT: ${{ env.SSH_PORT }}
+ - name: Cache local Maven repository
+ id: cache
+ uses: actions/cache@v4
+ with:
+ path: ~/.m2/repository
+ key: ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-${{ github.sha }}
+ restore-keys: |
+ ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-
- name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, opencga and opencga-enterprise
run: |
cd opencga-enterprise
ln -s ../opencga opencga-home
- ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }}
+ ./build.sh --test --test-level runShortTests --prepare-branches --test-save-reports --task ${{ inputs.task }} --storage-hadoop ${{ vars.HADOOP_FLAVOUR }} --prepare-hadoop
+ env:
+ THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
- name: Upload reports results to Github
uses: actions/upload-artifact@v4
with:
diff --git a/pom.xml b/pom.xml
index 0d49cb80..842d3159 100644
--- a/pom.xml
+++ b/pom.xml
@@ -22,7 +22,7 @@
2.14.3
3.14.0
1.7.36
- 1.7.7
+ 1.11.4
4.11.5
8.8.2
1.69