diff --git a/.github/workflows/build-java-app-workflow.yml b/.github/workflows/build-java-app-workflow.yml index c4e836b5..aaa7cbf1 100644 --- a/.github/workflows/build-java-app-workflow.yml +++ b/.github/workflows/build-java-app-workflow.yml @@ -10,10 +10,34 @@ on: type: string required: false default: "build-folder" + upload_artifact: + type: boolean + required: false + default: true + dependency_repos: + # Comma-separated list of dependency repositories to clone and build before building the main project. + type: string + required: false + default: "" + needs_hadoop_preparation: + type: boolean + required: false + default: false + hadoop_flavour: + type: string + required: false + default: "" + java_commons_libs_branch: + type: string + required: false + default: "develop" outputs: version: description: "Project version" value: ${{ jobs.build-workflow.outputs.version }} + cache_key: + description: "Cache key used for Maven repository" + value: ${{ jobs.build-workflow.outputs.cache_key }} jobs: build-workflow: @@ -21,25 +45,68 @@ jobs: runs-on: ${{ vars.UBUNTU_VERSION }} outputs: version: ${{ steps.get_project_version.outputs.version }} + cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} steps: - uses: actions/checkout@v4 + name: Checkout main repository + id: "checkout-main" + ## This checkout pulls the code of the repository where this workflow is being used with: fetch-depth: '10' + - uses: actions/checkout@v4 + if: ${{ inputs.dependency_repos != '' }} + name: Checkout java-common-libs repository + id: "checkout-java-common-libs" + ## This checkout pulls the code of the java-common-libs repository to get the scripts + ## Checkout to "java-common-libs" folder. + ## Filter by ".github" folder + with: + repository: opencb/java-common-libs + ref: ${{ inputs.java_commons_libs_branch }} + path: java-common-libs + fetch-depth: '1' - name: Set up JDK 8 uses: actions/setup-java@v4 with: distribution: 'temurin' java-version: '8' cache: 'maven' - - name: Install dependencies branches + - name: Clone dependencies + id: clone_dependencies + if: ${{ inputs.dependency_repos != '' }} + run: | + if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then + chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh + export DEPENDENCIES_SHA=${{ github.sha }} + java-common-libs/.github/workflows/scripts/get_same_branch.sh "${{ github.ref_name }}" "${{ inputs.dependency_repos }}" + fi + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }} + restore-keys: | + ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}- + - name: Compile dependencies + if: ${{ inputs.dependency_repos != '' }} run: | - if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then - chmod +x ./.github/workflows/scripts/get_same_branch.sh - ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} + if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then + chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh + java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}" fi + - name: Prepare Hadoop profile + if: ${{ inputs.needs_hadoop_preparation == true }} + run: | + chmod +x ./.github/workflows/scripts/prepare_hadoop.sh + ./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}" + env: + THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }} + - name: Maven Build (skip tests) run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress - uses: actions/upload-artifact@v4 + if: ${{ inputs.upload-artifact == true }} with: name: ${{ inputs.build_folder }} path: build @@ -49,3 +116,4 @@ jobs: echo "version=`mvn help:evaluate -q -Dexpression=project.version -DforceStdout`" >> $GITHUB_OUTPUT - name: test-version-from-check run: echo "Project version is " ${{ steps.get_project_version.outputs.version }} + diff --git a/.github/workflows/deploy-maven-repository-workflow.yml b/.github/workflows/deploy-maven-repository-workflow.yml index e34012dc..aca09c51 100644 --- a/.github/workflows/deploy-maven-repository-workflow.yml +++ b/.github/workflows/deploy-maven-repository-workflow.yml @@ -6,6 +6,9 @@ on: maven_opts: type: string required: false + cache_key: + type: string + required: false secrets: MAVEN_GPG_PASSPHRASE: required: true @@ -32,6 +35,14 @@ jobs: server-password: MAVEN_NEXUS_PASSWORD # env variable for token in deploy gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} # Value of the GPG private key to import gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase + - name: Cache local Maven repository + uses: actions/cache/restore@v4 + if: ${{ inputs.cache_key != '' }} + with: + path: ~/.m2/repository + key: ${{ inputs.cache_key }} + ## Force cache hit to avoid analyzing with incomplete dependencies + fail-on-cache-miss: true - name: Deploy to Maven Central repository run: mvn clean deploy -DskipTests -Pdeploy-maven ${{ inputs.maven_opts }} --no-transfer-progress env: diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 3029b0fd..0cc9c687 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -14,3 +14,5 @@ jobs: uses: ./.github/workflows/deploy-maven-repository-workflow.yml needs: build secrets: inherit + with: + cache_key: ${{ needs.build.outputs.cache_key }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c5ff2bde..37681d7c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,8 @@ jobs: uses: ./.github/workflows/deploy-maven-repository-workflow.yml needs: build secrets: inherit + with: + cache_key: ${{ needs.build.outputs.cache_key }} release: uses: ./.github/workflows/release-github-workflow.yml diff --git a/.github/workflows/scripts/compile_same_branch.sh b/.github/workflows/scripts/compile_same_branch.sh new file mode 100755 index 00000000..3ac1c303 --- /dev/null +++ b/.github/workflows/scripts/compile_same_branch.sh @@ -0,0 +1,23 @@ + +WORKSPACE=${WORKSPACE:-/home/runner/work/} + + +function compile() { + local REPO=$1 + if [ ! -d "${WORKSPACE}/$REPO" ]; then + echo "Directory ${WORKSPACE}/$REPO does not exist. Skip compile" + return 0; + fi + echo "::group::Compiling '$REPO' project from branch $BRANCH_NAME" + cd "${WORKSPACE}/$REPO" || exit 2 + mvn clean install -DskipTests --no-transfer-progress + echo "::endgroup::" +} + +## Comma separated list of repos to compile +REPOS=$1 + +IFS=',' read -ra REPO_ARRAY <<<"$REPOS" +for REPO in "${REPO_ARRAY[@]}"; do + compile "$REPO" +done \ No newline at end of file diff --git a/.github/workflows/scripts/get_same_branch.sh b/.github/workflows/scripts/get_same_branch.sh new file mode 100755 index 00000000..37b056fb --- /dev/null +++ b/.github/workflows/scripts/get_same_branch.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +BRANCH_NAME=$1 +## Comma separated list of repos to checkout +REPOS=$2 +DEPENDENCIES_SHA=${DEPENDENCIES_SHA:-""} +WORKSPACE=${WORKSPACE:-/home/runner/work/} + +if [[ -z "$BRANCH_NAME" ]]; then + echo "The first parameter is mandatory and must be a valid branch name." + exit 1 +fi + +function checkout(){ + local REPO=$1 + echo "::group::Installing '$REPO' project from branch $BRANCH_NAME" + cd "${WORKSPACE}" || exit 2 + git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME" + if [ -d "./$REPO" ]; then + cd "$REPO" || exit 2 + DEPENDENCIES_SHA=${DEPENDENCIES_SHA}:$(git rev-parse HEAD) + echo "Branch name $BRANCH_NAME already exists." + else + echo "Branch name $BRANCH_NAME does not exist in $REPO repository. Skipping installation." + fi + echo "::endgroup::" +} + +IFS=',' read -ra REPO_ARRAY <<<"$REPOS" +for REPO in "${REPO_ARRAY[@]}"; do + checkout "$REPO" +done + +## Apply sha1 to DEPENDENCIES_SHA if contains `:` +if [[ "$DEPENDENCIES_SHA" == *":"* ]]; then + DEPENDENCIES_SHA=$(echo -n "$DEPENDENCIES_SHA" | sha1sum | awk '{print $1}') +fi + +## Export DEPENDENCIES_SHA as github output +echo "dependencies_sha=$DEPENDENCIES_SHA" >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index 3af76372..cc2efef7 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -2,6 +2,10 @@ name: Build and test the project on: workflow_call: + inputs: + cache_key: + type: string + required: false secrets: SONAR_TOKEN: required: true @@ -20,6 +24,14 @@ jobs: distribution: 'temurin' java-version: '11' cache: 'maven' + - name: Cache local Maven repository + uses: actions/cache/restore@v4 + if: ${{ inputs.cache_key != '' }} + with: + path: ~/.m2/repository + key: ${{ inputs.cache_key }} + ## Force cache hit to avoid analyzing with incomplete dependencies + fail-on-cache-miss: true - name: Start MongoDB v6.0 uses: supercharge/mongodb-github-action@1.8.0 with: diff --git a/.github/workflows/test-xetabase-workflow.yml b/.github/workflows/test-xetabase-workflow.yml index 3720d93c..0d18858b 100644 --- a/.github/workflows/test-xetabase-workflow.yml +++ b/.github/workflows/test-xetabase-workflow.yml @@ -86,11 +86,21 @@ jobs: env: SSH_HOST: ${{ env.SSH_HOST }} SSH_PORT: ${{ env.SSH_PORT }} + - name: Cache local Maven repository + id: cache + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}- - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, opencga and opencga-enterprise run: | cd opencga-enterprise ln -s ../opencga opencga-home - ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }} + ./build.sh --test --test-level runShortTests --prepare-branches --test-save-reports --task ${{ inputs.task }} --storage-hadoop ${{ vars.HADOOP_FLAVOUR }} --prepare-hadoop + env: + THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }} - name: Upload reports results to Github uses: actions/upload-artifact@v4 with: diff --git a/pom.xml b/pom.xml index 0d49cb80..842d3159 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 2.14.3 3.14.0 1.7.36 - 1.7.7 + 1.11.4 4.11.5 8.8.2 1.69