TASK-7809 - Upgrade avro version from 1.7.7 to 1.11.4

.github/workflows/build-java-app-workflow.yml

@@ -10,36 +10,103 @@
         type: string
         required: false
         default: "build-folder"
+      upload_artifact:
+        type: boolean
+        required: false
+        default: true
+      dependency_repos:
+        # Comma-separated list of dependency repositories to clone and build before building the main project.
+        type: string
+        required: false
+        default: ""
+      needs_hadoop_preparation:
+        type: boolean
+        required: false
+        default: false
+      hadoop_flavour:
+        type: string
+        required: false
+        default: ""
+      java_commons_libs_branch:
+        type: string
+        required: false
+        default: "develop"
     outputs:
       version:
         description: "Project version"
         value: ${{ jobs.build-workflow.outputs.version }}
+      cache_key:
+        description: "Cache key used for Maven repository"
+        value: ${{ jobs.build-workflow.outputs.cache_key }}
 
 jobs:
   build-workflow:
     name: Build Java app
     runs-on: ${{ vars.UBUNTU_VERSION }}
     outputs:
       version: ${{ steps.get_project_version.outputs.version }}
+      cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
     steps:
       - uses: actions/checkout@v4
+        name: Checkout main repository
+        id: "checkout-main"
+        ## This checkout pulls the code of the repository where this workflow is being used
         with:
           fetch-depth: '10'
+      - uses: actions/checkout@v4
+        if: ${{ inputs.dependency_repos != '' }}
+        name: Checkout java-common-libs repository
+        id: "checkout-java-common-libs"
+        ## This checkout pulls the code of the java-common-libs repository to get the scripts
+        ## Checkout to "java-common-libs" folder.
+        ## Filter by ".github" folder
+        with:
+          repository: opencb/java-common-libs
+          ref: ${{ inputs.java_commons_libs_branch }}
+          path: java-common-libs
+          fetch-depth: '1'
       - name: Set up JDK 8
         uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '8'
           cache: 'maven'
-      - name: Install dependencies branches
+      - name: Clone dependencies
+        id: clone_dependencies
+        if: ${{ inputs.dependency_repos != '' }}
+        run: |
+          if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then
+            chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh
+            export DEPENDENCIES_SHA=${{ github.sha }}
+            java-common-libs/.github/workflows/scripts/get_same_branch.sh "${{ github.ref_name }}" "${{ inputs.dependency_repos }}"
+          fi
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-
+      - name: Compile dependencies
+        if: ${{ inputs.dependency_repos != '' }}
         run: |
-          if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
-            chmod +x ./.github/workflows/scripts/get_same_branch.sh
-            ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
+          if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then
+            chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh
+            java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}"
           fi
+      - name: Prepare Hadoop profile
+        if: ${{ inputs.needs_hadoop_preparation == true }}
+        run: |
+          chmod +x ./.github/workflows/scripts/prepare_hadoop.sh
+          ./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}"
+        env:
+          THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
+
       - name: Maven Build (skip tests)
         run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress
       - uses: actions/upload-artifact@v4
+        if: ${{ inputs.upload-artifact == true }}
         with:
           name: ${{ inputs.build_folder }}
           path: build
@@ -49,3 +116,4 @@
           echo "version=`mvn help:evaluate -q -Dexpression=project.version -DforceStdout`" >> $GITHUB_OUTPUT
       - name: test-version-from-check
         run: echo "Project version is " ${{ steps.get_project_version.outputs.version }}
+

.github/workflows/deploy-maven-repository-workflow.yml

@@ -6,6 +6,9 @@ on:
       maven_opts:
         type: string
         required: false
+      cache_key:
+        type: string
+        required: false
     secrets:
       MAVEN_GPG_PASSPHRASE:
         required: true
@@ -32,6 +35,14 @@ jobs:
           server-password: MAVEN_NEXUS_PASSWORD                   # env variable for token in deploy
           gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}   # Value of the GPG private key to import
           gpg-passphrase: MAVEN_GPG_PASSPHRASE                    # env variable for GPG private key passphrase
+      - name: Cache local Maven repository
+        uses: actions/cache/restore@v4
+        if: ${{ inputs.cache_key != '' }}
+        with:
+          path: ~/.m2/repository
+          key: ${{ inputs.cache_key }}
+          ## Force cache hit to avoid analyzing with incomplete dependencies
+          fail-on-cache-miss: true
       - name: Deploy to Maven Central repository
         run: mvn clean deploy -DskipTests -Pdeploy-maven ${{ inputs.maven_opts }} --no-transfer-progress
         env:

.github/workflows/develop.yml

@@ -14,3 +14,5 @@ jobs:
     uses: ./.github/workflows/deploy-maven-repository-workflow.yml
     needs: build
     secrets: inherit
+    with:
+      cache_key: ${{ needs.build.outputs.cache_key }}

.github/workflows/release.yml

@@ -13,6 +13,8 @@ jobs:
     uses: ./.github/workflows/deploy-maven-repository-workflow.yml
     needs: build
     secrets: inherit
+    with:
+      cache_key: ${{ needs.build.outputs.cache_key }}
 
   release:
     uses: ./.github/workflows/release-github-workflow.yml

.github/workflows/scripts/compile_same_branch.sh

@@ -0,0 +1,23 @@
+
+WORKSPACE=${WORKSPACE:-/home/runner/work/}
+
+
+function compile() {
+  local REPO=$1
+  if [ ! -d "${WORKSPACE}/$REPO" ]; then
+    echo "Directory ${WORKSPACE}/$REPO does not exist. Skip compile"
+    return 0;
+  fi
+  echo "::group::Compiling '$REPO' project from branch $BRANCH_NAME"
+  cd "${WORKSPACE}/$REPO" || exit 2
+  mvn clean install -DskipTests --no-transfer-progress
+  echo "::endgroup::"
+}
+
+## Comma separated list of repos to compile
+REPOS=$1
+
+IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
+for REPO in "${REPO_ARRAY[@]}"; do
+  compile "$REPO"
+done

.github/workflows/scripts/get_same_branch.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+BRANCH_NAME=$1
+## Comma separated list of repos to checkout
+REPOS=$2
+DEPENDENCIES_SHA=${DEPENDENCIES_SHA:-""}
+WORKSPACE=${WORKSPACE:-/home/runner/work/}
+
+if [[ -z "$BRANCH_NAME"  ]]; then
+  echo "The first parameter is mandatory and must be a valid branch name."
+  exit 1
+fi
+
+function checkout(){
+  local REPO=$1
+  echo "::group::Installing '$REPO' project from branch $BRANCH_NAME"
+  cd "${WORKSPACE}" || exit 2
+  git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME"
+  if [ -d "./$REPO" ]; then
+    cd "$REPO" || exit 2
+    DEPENDENCIES_SHA=${DEPENDENCIES_SHA}:$(git rev-parse HEAD)
+    echo "Branch name $BRANCH_NAME already exists."
+  else
+    echo "Branch name $BRANCH_NAME does not exist in $REPO repository. Skipping installation."
+  fi
+  echo "::endgroup::"
+}
+
+IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
+for REPO in "${REPO_ARRAY[@]}"; do
+  checkout "$REPO"
+done
+
+## Apply sha1 to DEPENDENCIES_SHA if contains `:`
+if [[ "$DEPENDENCIES_SHA" == *":"* ]]; then
+  DEPENDENCIES_SHA=$(echo -n "$DEPENDENCIES_SHA" | sha1sum | awk '{print $1}')
+fi
+
+## Export DEPENDENCIES_SHA as github output
+echo "dependencies_sha=$DEPENDENCIES_SHA" >> "$GITHUB_OUTPUT"

.github/workflows/test-analysis.yml

@@ -2,6 +2,10 @@ name: Build and test the project
 
 on:
   workflow_call:
+    inputs:
+      cache_key:
+        type: string
+        required: false
     secrets:
       SONAR_TOKEN:
         required: true
@@ -20,6 +24,14 @@ jobs:
           distribution: 'temurin'
           java-version: '11'
           cache: 'maven'
+      - name: Cache local Maven repository
+        uses: actions/cache/restore@v4
+        if: ${{ inputs.cache_key != '' }}
+        with:
+          path: ~/.m2/repository
+          key: ${{ inputs.cache_key }}
+          ## Force cache hit to avoid analyzing with incomplete dependencies
+          fail-on-cache-miss: true
       - name: Start MongoDB v6.0
         uses: supercharge/mongodb-github-action@1.8.0
         with:

.github/workflows/test-xetabase-workflow.yml

@@ -86,11 +86,21 @@
         env:
           SSH_HOST: ${{ env.SSH_HOST }}
           SSH_PORT: ${{ env.SSH_PORT }}
+      - name: Cache local Maven repository
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-
       - name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, opencga and opencga-enterprise
         run: |
           cd opencga-enterprise
           ln -s ../opencga opencga-home
-          ./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }}
+          ./build.sh --test --test-level runShortTests --prepare-branches --test-save-reports --task ${{ inputs.task }} --storage-hadoop ${{ vars.HADOOP_FLAVOUR }} --prepare-hadoop
+        env:
+          THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
       - name: Upload reports results to Github
         uses: actions/upload-artifact@v4
         with:

pom.xml

@@ -22,7 +22,7 @@
         <jackson.version>2.14.3</jackson.version>
         <apache.commons.lang3.version>3.14.0</apache.commons.lang3.version>
         <slf4j.version>1.7.36</slf4j.version>
-        <avro.version>1.7.7</avro.version>
+        <avro.version>1.11.4</avro.version>
         <mongodb.version>4.11.5</mongodb.version>
         <solr.version>8.8.2</solr.version>
         <jcommander.version>1.69</jcommander.version>