Noble testing

.github/workflows/ubuntu.yml

@@ -17,15 +17,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [ "Build Jammy", "Build Jammy OMP", "Build Noble" ]
+        name: [ "Build Noble" ]
         include:
 
-          - name: "Build Jammy"
-            release: jammy
-
-          - name: "Build Jammy OMP"
-            release: jammy-omp
-
           - name: "Build Noble"
             release: noble
 
@@ -59,21 +53,21 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        name: [ "Unit Jammy", "Short Jammy", "Medium Jammy", "Unit Noble" ]
+        name: [ "Unit Noble", "Short Noble", "Medium Noble" ]
         include:
 
-          - name: "Unit Jammy"
-            release: jammy
+          - name: "Unit Noble"
+            release: noble
             command: "make unittest"
             output: "test_results_unittests.xml"
 
-          - name: "Short Jammy"
-            release: jammy
+          - name: "Short Noble"
+            release: noble
             command: "make THREADS=2 test"
             output: "test_results.xml"
 
-          - name: "Medium Jammy"
-            release: jammy
+          - name: "Medium Noble"
+            release: noble
             command: "make THREADS=2 mediumtest"
             output: "test_results_medium.xml"
 
@@ -220,8 +214,8 @@ jobs:
       - name: ${{ matrix.name }} Longtesting
         run: |
          chmod 777 .
-         docker pull fluidity/actions:jammy-${{ github.sha }}
-         docker run -v $PWD:/host fluidity/actions:jammy-${{ github.sha }} /bin/bash -c "git clone https://github.com/fluidityproject/longtests && bin/testharness -x test_results_${{ matrix.name }}.xml -f ${{ matrix.name }}.xml && cp -v test_results_${{ matrix.name }}.xml /host"
+         docker pull fluidity/actions:noble-${{ github.sha }}
+         docker run -v $PWD:/host fluidity/actions:noble-${{ github.sha }} /bin/bash -c "git clone https://github.com/fluidityproject/longtests && bin/testharness -x test_results_${{ matrix.name }}.xml -f ${{ matrix.name }}.xml && cp -v test_results_${{ matrix.name }}.xml /host"
 
       - uses: actions/upload-artifact@v4
         with:

assemble/Full_Projection.F90

@@ -432,7 +432,7 @@ subroutine petsc_solve_setup_full_projection(y,A,b,ksp,petsc_numbering_p,name,so
 
       ! Set up RHS and Solution vectors (note these are loaded later):
       b = PetscNumberingCreateVec(petsc_numbering_p)
-      call VecDuplicate(b,y,ierr) ! Duplicate vector b and form vector y
+      call FixedVecDuplicate(b,y,ierr) ! Duplicate vector b and form vector y
 
       ! Schur complement objects now fully set up. The next stage is to setup
       ! KSP and PC for the solution of Ay = b, where A = G_t*M^-1*G.

assemble/tests/test_matrix_free.F90

@@ -298,7 +298,7 @@ subroutine petsc_solve_matrix_free(x, matrix, rhs)
 
   call VecCreateSeq(MPI_COMM_SELF,size(rhs%val), b, ierr)
 
-  call VecDuplicate(b, y, ierr)
+  call FixedVecDuplicate(b, y, ierr)
 
   ! copy array into PETSc vecs
   call VecSetValues(b, size(rhs%val), &

docker/Dockerfile.noble

@@ -19,14 +19,18 @@ RUN apt-get update && \
      apt-get update && \
      echo "Europe/London" > /etc/timezone && \
      apt-get -y install fluidity-dev texlive-pstricks texlive texlive-latex-extra texlive-science python3-pip python3-junit.xml && \
+     apt-get -y install sudo flex bison && \
      apt-get clean
 
+# Use old version of gmsh in CI to avoid mesh-sensitive tests failing
 WORKDIR /usr/local
-RUN curl -fsL https://gmsh.info/bin/Linux/gmsh-4.14.0-Linux64.tgz | tar --strip-components=1 -zxf -
+RUN curl -fsL https://gmsh.info/bin/Linux/gmsh-2.16.0-Linux64.tgz | tar --strip-components=1 -zxf -
 
 ENV OMPI_MCA_btl_vader_single_copy_mechanism=none
 ENV OMPI_MCA_rmaps_base_oversubscribe=1
 
+# Build vtk 9.4 instead of noble's vtk 9.2 as we need the fix for https://gitlab.kitware.com/vtk/vtk/-/issues/19258
+# (failing reading of vtk files with newer versions of expat)
 WORKDIR /root
 RUN curl -fsL https://vtk.org/files/release/9.4/VTK-9.4.0.tar.gz | tar -xzf - && \
     cd VTK-9.4.0 && \
@@ -38,6 +42,22 @@ RUN curl -fsL https://vtk.org/files/release/9.4/VTK-9.4.0.tar.gz | tar -xzf - &&
     cd ../.. && \
     rm -rf VTK-9.4.0
 
+# Build petsc 3.24 because noble's petsc version has an intermediate gamg with known issues
+WORKDIR /tmp
+RUN curl -O https://web.cels.anl.gov/projects/petsc/download/release-snapshots/petsc-3.24.2.tar.gz
+RUN tar -xzf petsc-3.24.2.tar.gz && mv petsc-3.24.2 /opt/petsc && rm petsc-3.24.2.tar.gz
+
+WORKDIR /opt/petsc
+RUN ./configure --with-debugging=1 --with-fortran-bindings=1 --with-shared-libraries=1 --with-strict-petscerrorcode --with-bison --with-fftw --with-hdf5 --with-hwloc --with-metis --with-mumps --download-netcdf --download-pnetcdf --download-ptscotch --with-scalapack-lib=-lscalapack-openmpi --with-suitesparse --download-superlu_dist --with-zlib --download-hypre -download-hdf5
+RUN make PETSC_DIR=/opt/petsc PETSC_ARCH=arch-linux-c-debug all
+
+# Environment for building fluidity with this vtk and petsc
+ENV VTK_INSTALL_PREFIX=/opt/vtk-9.4.0
+ENV PETSC_DIR=/opt/petsc
+ENV PETSC_ARCH=arch-linux-c-debug
+ENV PATH="$PETSC_DIR/$PETSC_ARCH/bin:$PATH"
+ENV PYTHONPATH="/opt/vtk-9.4.0/lib/python3.12/site-packages"
+
 # Add a Fluidity user who will be the default user for this container
 # Make sure the user has a userid matching the host system
 # -- pass this as an argument at build time

docker/actions/Dockerfile.actions.noble

@@ -4,7 +4,6 @@ USER root
 
 RUN apt-get -y update && \
       apt-get -y dist-upgrade && \
-      apt-get -y install sudo && \
       rm -rf /var/cache/apt/archives && \
       rm -rf /var/lib/apt/lists
 
@@ -15,11 +14,21 @@ COPY . /home/fluidity
 RUN chown -R fluidity /home/fluidity
 
 USER fluidity
+WORKDIR /home/fluidity
 
-ENV VTK_INSTALL_PREFIX=/opt/vtk-9.4.0
 RUN ./configure --enable-2d-adaptivity
+# NOTE: running "make makefiles" is only needed for
+# developers that have changed any of the module use statements
+# For all other users, this step is NOT needed.
+# "make makefiles" regenerates the */Makefile.dependencies
+# which should already be committed to the repo. We do this here
+# only to check whether the regenerated files have changed from
+# the committed version, in which case the "test -z" and thus the
+# build fails. If that happens, developers should run "make makefiles"
+# locally and commit the changes.
 RUN make makefiles
 RUN test -z "$(git status --porcelain */Makefile.dependencies)"
+
 RUN make
 RUN make fltools
 RUN make manual

examples/rotating_channel/channel.flml

@@ -207,9 +207,7 @@
               <integer_value rank="0">40</integer_value>
             </restart>
           </iterative_method>
-          <preconditioner name="hypre">
-            <hypre_type name="euclid"/>
-          </preconditioner>
+          <preconditioner name="sor"/>
           <relative_error>
             <real_value rank="0">1.0e-10</real_value>
           </relative_error>

examples/water_collapse/water_collapse.xml

@@ -31,7 +31,7 @@ nodesend= stat("water_collapse.stat")["CoordinateMesh"]["nodes"][-1]</variable>
   <pass_tests>
     <test name="finish_time" language="python">assert finishtime &gt; 2.4</test>
     <test name="water_conservation" language="python">print('mass loss = ', abs(vfracintend-vfracintstart)/vfracintstart)
-assert abs(vfracintend-vfracintstart)/vfracintstart &lt; 1.e-5
+assert abs(vfracintend-vfracintstart)/vfracintstart &lt; 5.e-5
 </test>
     <test name="divergence" language="python">print('divergence = ', max(maxdivergence, mindivergence))
 assert max(maxdivergence, mindivergence) &lt; 1.E-5

femtools/Diagnostic_variables.F90

@@ -1184,10 +1184,10 @@ subroutine initialise_constant_diagnostics(unit, binary_format)
 
     call get_environment_variable(name="HOSTNAME", value=value_buffer, status=stat)
     if (stat /= 0) then
-      ewrite(-1, *) "GET_ENVIRONMENT_VARIABLE('HOSTNAME') returned no-zero status: ", stat
+      ewrite(0, *) "GET_ENVIRONMENT_VARIABLE('HOSTNAME') returned no-zero status: ", stat
+      buffer=constant_tag(name="HostName", type="string", value=trim(value_buffer))
+      write(unit, '(a)') trim(buffer)
     end if
-    buffer=constant_tag(name="HostName", type="string", value=trim(value_buffer))
-    write(unit, '(a)') trim(buffer)
 
     ! Constant values
     if(present_and_true(binary_format)) then

femtools/Multigrid.F90

@@ -746,7 +746,7 @@ function Prolongator(A, epsilon, omega, maxclustersize, cluster) result (P)
   call VecCopy(diag, sqrt_diag, ierr)
   call VecSqrtAbs(sqrt_diag, ierr)
   !
-  call VecDuplicate(sqrt_diag, inv_sqrt_diag, ierr)
+  call FixedVecDuplicate(sqrt_diag, inv_sqrt_diag, ierr)
   call VecCopy(sqrt_diag, inv_sqrt_diag, ierr)
   call VecReciprocal(inv_sqrt_diag, ierr)
   call MatDiagonalScale(A, inv_sqrt_diag, inv_sqrt_diag, ierr)
@@ -802,7 +802,7 @@ function Prolongator(A, epsilon, omega, maxclustersize, cluster) result (P)
   ! now restore the original matrix
   ! unfortunately MatDiagonalScale is broken for one-sided scaling, i.e.
   ! supplying PETSC_NULL(_OBJECT) for one the vectors
-  call VecDuplicate(diag, one, ierr)
+  call FixedVecDuplicate(diag, one, ierr)
   call VecSet(one, 1.0_PetscReal_kind, ierr)
   call MatDiagonalScale(A, diag, one, ierr)
 

femtools/Petsc_Tools.F90

@@ -118,6 +118,7 @@ module Petsc_Tools
   logical, public, save :: petsc_test_error_handler_called = .false.
   public petsc_test_error_handler
   public IsNullMatNullSpace
+  public FixedVecDuplicate
 
 contains
 
@@ -1030,6 +1031,7 @@ function block_csr2petsc(A, petsc_numbering, column_petsc_numbering, &
                if (A%diagonal .and. bh/=bv) cycle
                ! row number in PETSc land:
                rows(1)=row_numbering%gnn2unn(i, bv)
+               if (rows(1)<0) cycle
                vals => row_val_ptr(A, bv, bh, i)
 #ifdef DOUBLEP
                call MatSetValues(M, 1, rows, len, colidx(1:len), vals, &
@@ -1539,5 +1541,18 @@ function IsNullMatNullSpace(nullsp)
 
 end function IsNullMatNullSpace
 
+subroutine FixedVecDuplicate(v, newv, ierr)
+  ! It appears that for VecSeq, petsc does not copy the options
+  ! of the duplicated vector. Ensure it does.
+  Vec, intent(in) :: v
+  Vec, intent(out) :: newv
+  PetscErrorCode :: ierr
+
+    call VecDuplicate(v, newv, ierr)
+    call VecSetOption(newv, VEC_IGNORE_NEGATIVE_INDICES, PETSC_TRUE, ierr)
+    call VecSetOption(newv, VEC_IGNORE_OFF_PROC_ENTRIES, PETSC_TRUE, ierr)
+
+end subroutine FixedVecDuplicate
+
 #include "Reference_count_petsc_numbering_type.F90"
 end module Petsc_Tools

femtools/Solvers.F90

@@ -680,7 +680,7 @@ subroutine petsc_solve_setup(y, A, b, ksp, petsc_numbering, &
   ! one of the PETSc supplied orderings see
   ! http://www-unix.mcs.anl.gov/petsc/petsc-as/snapshots/petsc-current/docs/manualpages/MatOrderings/MatGetOrdering.html
   MatOrderingType:: ordering_type
-  logical:: use_reordering
+  PetscBool:: use_reordering
   real time1, time2
   integer ierr
   logical:: parallel, timing, have_cache
@@ -761,7 +761,7 @@ subroutine petsc_solve_setup(y, A, b, ksp, petsc_numbering, &
   ! Note the explicitly-described options rcm, 1wd and natural are now not
   ! listed explicitly in the schema (but can still be used by adding the
   ! appropriate string in the solver reordering node).
-  call PetscOptionsGetString(PETSC_NULL_OPTIONS, "", "-ordering_type", ordering_type, use_reordering, ierr)
+  call PetscOptionsGetString(PETSC_NULL_OPTIONS, PETSC_NULL_CHARACTER, "-ordering_type", ordering_type, use_reordering, ierr)
   if (.not. use_reordering) then
     call get_option(trim(solver_option_path)//'/reordering[0]/name', &
       ordering_type, stat=ierr)
@@ -903,7 +903,7 @@ subroutine petsc_solve_setup(y, A, b, ksp, petsc_numbering, &
   end if
 
   b=PetscNumberingCreateVec(petsc_numbering)
-  call VecDuplicate(b, y, ierr)
+  call FixedVecDuplicate(b, y, ierr)
 
   if (timing) then
     call cpu_time(time2)
@@ -1032,7 +1032,7 @@ subroutine petsc_solve_setup_petsc_csr(y, b, &
   end if
 
   b=PetscNumberingCreateVec(matrix%column_numbering)
-  call VecDuplicate(b, y, ierr)
+  call FixedVecDuplicate(b, y, ierr)
 
   if (timing) then
     call cpu_time(time2)
@@ -1833,6 +1833,7 @@ recursive subroutine setup_pc_from_options(pc, pmat, option_path, &
     MatSolverType:: matsolvertype
     PetscErrorCode:: ierr
     integer :: n_local, first_local
+    integer :: i, nolevels
 
     call get_option(trim(option_path)//'/name', pctype)
 
@@ -1981,6 +1982,18 @@ recursive subroutine setup_pc_from_options(pc, pmat, option_path, &
         ! PC setup seems to be required so that the Coarse Eq Lim option is used.
         call PCSetup(pc,ierr)
 
+        call PCMGGetLevels(pc, nolevels, ierr)
+        if (nolevels<1) then
+          FLAbort("Something went wrong in gamg preconditioner setup")
+        end if
+        ! set up and downsmoother to SOR (instead of new default Jacobi)
+        ! not changing coarse level (i=0)
+        do i=1, nolevels-1
+          call PCMGGetSmoother(pc, i, subksp, ierr)
+          call KSPGetPC(subksp, subpc, ierr)
+          call PCSetType(subpc, PCSOR, ierr)
+        end do
+
         call MatGetNullSpace(pmat, nullsp, ierr)
         if (ierr==0 .and. .not. IsNullMatNullSpace(nullsp)) then
           ! if the preconditioner matrix has a nullspace, this may still be present
@@ -2023,25 +2036,30 @@ recursive subroutine setup_fieldsplit_preconditioner(pc, option_path, &
     PetscErrorCode:: ierr
     integer:: i, n
 
-    call PCSetType(pc, "fieldsplit", ierr)
+    call PCSetType(pc, PCFIELDSPLIT, ierr)
+#if (PETSC_VERSION_MAJOR==3 && PETSC_VERSION_MINOR==23)
+    FLAbort("Fortran fieldsplit interface is broken in petsc 3.23")
+#endif
 
     call PCFieldSplitGetSubKSP(pc, n, subksps, ierr)
     if (n==0) then
       ! first time this pc set to type fieldplit: it's the first time we set it up,
       ! or it was previously set to a different type - in this case, PCSetType will
       ! have called PCCreate_FieldSplit which will have set n/o splits to zero
-      do i=1, size(subksps)
+      do i=1, size(petsc_numbering%gnn2unn, 2)
         index_set = petsc_numbering_create_is(petsc_numbering, dim=i)
         call PCFieldSplitSetIS(pc, PETSC_NULL_CHARACTER, index_set, ierr)
         call ISDestroy(index_set, ierr)
       end do
 
+#if (PETSC_VERSION_MAJOR==3 && PETSC_VERSION_MINOR<23)
     elseif (n/=size(subksps)) then
 
       ! if this pc is reused (and we've previously already set it up with fieldsplit)
       ! we need to check the n/o fieldsplits is the same
 
       FLAbort("PC being reused with different number of fieldsplits")
+#endif
 
     end if
 
@@ -2058,7 +2076,8 @@ recursive subroutine setup_fieldsplit_preconditioner(pc, option_path, &
       FLAbort("Unknown fieldsplit_type")
     end select
 
-    call pcfieldsplitgetsubksp(pc, n, subksps, ierr)
+    call PCSetup(pc, ierr)
+    call PCFieldSplitGetSubKSP(pc, n, subksps, ierr)
 
     assert(n==size(subksps))
 
@@ -2269,7 +2288,7 @@ subroutine petsc_monitor_setup(petsc_numbering, max_its)
 
   if (petsc_monitor_has_exact) then
 
-    call VecDuplicate(petsc_monitor_x, petsc_monitor_exact, ierr)
+    call FixedVecDuplicate(petsc_monitor_x, petsc_monitor_exact, ierr)
 
     if (ncomponents==1) then
       call field2petsc(petsc_monitor_exact_sfield, petsc_numbering, petsc_monitor_exact)
@@ -2431,7 +2450,7 @@ subroutine MyKSPMonitor(ksp,n,rnorm,dummy,ierr)
     ! then (re)compute the (true) residual
     call KSPGetRhs(ksp, rhs, ierr)
     call KSPGetOperators(ksp, Amat, Pmat, ierr)
-    call VecDuplicate(petsc_monitor_x, r, ierr)
+    call FixedVecDuplicate(petsc_monitor_x, r, ierr)
     call MatMult(Amat, petsc_monitor_x, r, ierr)
     call VecAXPY(r, real(-1.0, kind = PetscScalar_kind), rhs, ierr)
     if (size(petsc_monitor_numbering%gnn2unn,2)==1) then
@@ -2602,7 +2621,7 @@ function create_null_space_from_options_vector(mat, null_space_option_path, &
    assert(i==nnulls)
 
    if (present(rotation_matrix) .and. nnulls>0) then
-     call VecDuplicate(null_space_array(1), aux_vec, ierr)
+     call FixedVecDuplicate(null_space_array(1), aux_vec, ierr)
      do i=1, nnulls
        ! rotate the null vector and store it in aux_vec
        call MatMultTranspose(rotation_matrix, null_space_array(i), aux_vec, ierr)