From fda067c73019db49583d8eeddda12c396d3028a0 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Thu, 27 Feb 2025 16:38:43 +0100
Subject: [PATCH 01/25] add new Dockerfile and auto-deployment config

---
 .github/workflows/deploy-image.yml |  15 ++-
 Dockerfile                         | 146 +++++++++++++++++++----------
 2 files changed, 102 insertions(+), 59 deletions(-)

diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index 09b7a790..b4bd7f0e 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -2,7 +2,6 @@ name: Create and publish a Docker image
 
 on:
   push:
-    branches: ['master', 'develop']
 
 env:
   REGISTRY: ghcr.io
@@ -10,17 +9,16 @@ env:
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-latest
+    runs-on: 
+      - ubuntu-latest
+      # - macos-latest
     permissions:
       contents: read
       packages: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
@@ -28,14 +26,13 @@ jobs:
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+        uses: docker/build-push-action@v6
         with:
-          context: .
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
diff --git a/Dockerfile b/Dockerfile
index 7d2785ab..0747a8ef 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,57 +1,103 @@
-FROM continuumio/miniconda3
+FROM python:3.9-bookworm
 
-LABEL Description="ShapePipe Docker Image"
-ENV SHELL /bin/bash
+LABEL Description="Conda-Free ShapePipe Docker Image"
+ENV SHELL=/bin/bash
 
-ARG CC=gcc-9
-ARG CXX=g++-9
+# Install system dependencies
+RUN apt-get update -y --quiet --fix-missing && \
+    apt-get dist-upgrade -y --quiet --fix-missing && \
+    apt-cache policy autconf && \ 
+    apt-get install -y --quiet \
+    apt-utils \
+    autoconf \
+    automake \
+    build-essential \
+    cmake \
+    curl \
+    ffmpeg \
+    g++ \
+    gcc  \
+    gfortran \
+    git-lfs \
+    libatlas-base-dev \
+    libblas-dev \
+    liblapack-dev \
+    libcfitsio-dev \
+    libfftw3-bin \
+    libfftw3-dev \
+    libgl1-mesa-glx \
+    libtool \
+    libtool-bin \
+    libtool-doc \
+    locales \
+    locate \
+    make \
+    openmpi-bin \
+    libopenmpi-dev \
+    pkg-config \
+    protobuf-compiler \
+    psfex=3.21.1-1 \
+    source-extractor=2.25.0+ds-3 \
+    weightwatcher=1.12+dfsg-3 \
+    vim \
+    xterm && \
+    apt-get clean -y && \
+    apt-get autoremove --purge --quiet -y && \
+    rm -rf /var/lib/apt/lists/* /var/tmp/*
 
-# gcc < 10 is required to compile ww
-ENV CC=gcc-9
-ENV CXX=g++-9
+# Install CDS client by hand
+RUN cd /tmp && \
+    curl -O http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz && \
+    tar xvfz cdsclient.tar.gz \ 
+    && cd cdsclient-* \
+    && ./configure && make && make install \
+    && rm -rf /tmp/*
 
-RUN apt-get update --allow-releaseinfo-change && \
-    apt-get update && \
-    apt-get upgrade -y && \
-    apt-get install apt-utils -y && \
-    apt-get install make -y && \
-    apt-get install automake -y && \
-    apt-get install autoconf -y && \
-    apt-get install gcc-9 g++-9 -y && \
-    apt-get install gfortran -y && \
-    apt-get install locales -y && \
-    apt-get install libgl1-mesa-glx -y && \
-    apt-get install xterm -y && \
-    apt-get install cmake protobuf-compiler -y && \
-    apt-get install libtool libtool-bin libtool-doc -y && \
-    apt-get install libfftw3-bin libfftw3-dev -y && \
-    apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
-    apt-get install vim -y && \
-    apt-get install locate -y && \
-    apt-get install curl -y && \
-    apt-get install acl -y && \
-    apt-get install sssd -y && \
-    apt-get clean
+# Install python dependencies
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir \
+    astropy==5.2 \
+    cs_util==0.1.0 \
+    galsim==2.2.6 \
+    ipython==8.18.1 \
+    joblib==1.1.0 \
+    jupyterlab==4.3.1 \
+    matplotlib==3.8.4 \
+    mccd==1.2.4 \
+    modopt==1.6.1 \
+    mpi4py==3.1.3 \
+    numba==0.58.1 \
+    numpy==1.26.4 \
+    numpydoc==1.2  \
+    pandas==1.4.1  \
+    PyQt5==5.15.6 \
+    pyqtgraph==0.12.4 \
+    pytest==8.3.3 \
+    pytest-cov==5.0.0 \
+    pytest-pycodestyle==2.4.1 \
+    pytest-pydocstyle==2.4.0 \
+    python-pysap==0.2.1 \
+    reproject==0.8 \
+    sf_tools==2.0.4 \
+    sip_tpv==1.1 \
+    skaha==1.4.3 \
+    sqlitedict==2.0.0 \
+    termcolor==1.1.0 \
+    tqdm==4.63.0 \
+    treecorr==4.2.6 \
+    vos==3.6.1.1 \
+    git+https://github.com/aguinot/ngmix@stable_version \
+    git+https://github.com/tobias-liaudat/Stile@v0.1
 
-ADD nsswitch.conf /etc/
 
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
-    locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+WORKDIR /app
+COPY . /app/.
 
-SHELL ["/bin/bash", "--login", "-c"]
-
-COPY ./environment.yml ./
-COPY install_shapepipe README.rst setup.py setup.cfg ./
-RUN touch ./README.md
-
-RUN conda update -n base -c defaults conda -c defaults
-RUN conda env create --file environment.yml
-
-COPY shapepipe ./shapepipe
-COPY scripts ./scripts
-
-RUN source activate shapepipe
-#RUN pip install jupyter
+# Install shapepipe and symlink scripts
+RUN pip install --no-cache-dir -e . && \ 
+    for ext in .py .sh .bash; do \
+    for script in /app/scripts/*/*$ext; do \
+    link_name=`basename $script $ext`; \
+    ln -s $script /usr/local/bin/$link_name; \
+    done; \
+    done
\ No newline at end of file

From 59e604efbf6e17d72b800cf9645d0f04cf9854e2 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Thu, 27 Feb 2025 16:39:34 +0100
Subject: [PATCH 02/25] delete obsolete conda environments and install script

---
 environment-dev.yml |  38 ---
 environment.yml     |  46 ---
 install_shapepipe   | 702 --------------------------------------------
 3 files changed, 786 deletions(-)
 delete mode 100644 environment-dev.yml
 delete mode 100644 environment.yml
 delete mode 100755 install_shapepipe

diff --git a/environment-dev.yml b/environment-dev.yml
deleted file mode 100644
index 703c1917..00000000
--- a/environment-dev.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-# ShapePipe Development Environment
-# ---------------------------------
-# This environment should only be used for development and testing of  
-# ShapePipe (i.e. nothing that could be published). Packages need only
-# specify the minimum compatible versions. This environment can updated 
-# as needed.
-name: shapepipe-dev
-channels:
-  - conda-forge
-dependencies:
-  - python>=3.9.18
-  - pip>=23.3.1
-  - astropy>=5.1.1
-  - autoconf>=2.71
-  - automake>=1.16.5
-  - cmake>=3.27.7
-  - galsim>=2.5.1
-  - joblib>=1.3.2
-  - libtool>=2.4
-  - matplotlib>=3.8.1
-  - numba>=0.58.1
-  - pandas>=2.1
-  - pyqt5-sip>=12.13
-  - pyqtgraph>=0.13.3
-  - reproject>=0.12
-  - sqlitedict>=2.1
-  - termcolor>=2.3
-  - tqdm>=4.66.1
-  - treecorr>=4.3.3
-  - pip:
-    - cs_util==0.0.5
-    - mccd==1.2.4
-    - modopt==1.6.1
-    - sip_tpv==1.1
-    - sf_tools==2.0.4
-    - git+https://github.com/CEA-COSMIC/pysap@develop
-    - git+https://github.com/aguinot/ngmix@stable_version
-    - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/environment.yml b/environment.yml
deleted file mode 100644
index ebb04d2c..00000000
--- a/environment.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-name: shapepipe_v1.4
-# ShapePipe Production Environment
-# --------------------------------
-# This environment should be used for any production runs of ShapePipe 
-# (i.e. anything that could be published). All packages are pinned to
-# specific versions to maximise the reproducibility of the results.
-# This environment should be updated for each release of ShapePipe.
-channels:
-  - conda-forge
-  - defaults
-dependencies:
-  - python=3.9
-  - pip>=21.2.4
-  - astromatic-psfex==3.21.1
-  - astromatic-source-extractor==2.25.0
-  - astropy==5.2
-  - automake
-  - autoconf
-  - cmake
-  - h5py
-  - galsim
-  - joblib
-  - libtool
-  - matplotlib==3.8.4
-  - numba==0.58.1
-  - numpy==1.26.4
-  - pandas
-  - pip:
-    - cs_util==0.1
-    - mccd==1.2.4
-    - modopt
-    - PyQt5
-    - pyqtgraph
-    - python-pysap==0.2.1
-    - reproject
-    - shear_psf_leakage
-    - skaha
-    - sip_tpv
-    - sf_tools
-    - sqlitedict
-    - termcolor
-    - tqdm
-    - treecorr
-    - vos
-    - git+https://github.com/aguinot/ngmix@stable_version
-    - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/install_shapepipe b/install_shapepipe
deleted file mode 100755
index 263c4e8c..00000000
--- a/install_shapepipe
+++ /dev/null
@@ -1,702 +0,0 @@
-#!/usr/bin/env bash
-export LC_ALL=en_US.UTF-8
-export LC_CTYPE=en_US.utf8
-
-##############################################################################
-# Install ShapePipe Executables
-#
-# This script can only be run after the ShapePipe environment has been
-# installed.
-#
-##############################################################################
-
-version="1.8"
-last_update="13/10/24"
-
-##############################################################################
-# EXTERNAL PACKAGES AND SCRIPTS
-##############################################################################
-
-# External packages
-
-# Conda package versions
-fftw_ver="3.3.10"
-libpng_ver="1.6.37"
-mpi4py_ver="4.0.1"
-openblas_ver="0.3.18"
-
-# WeightWatcher Package
-ww_url="https://github.com/astromatic/weightwatcher/archive/refs/tags/1.12.tar.gz"
-ww_tar="1.12.tar.gz"
-ww_dir="weightwatcher-1.12"
-ww_bin="ww"
-ww_version=TRUE
-
-# CDSclient Package (need Findgsc2.2)
-cdsclient_url="http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz"
-cdscleint_tar="cdsclient.tar.gz"
-cdsclient_dir="cdsclient-4.07"
-cdsclient_bin="findgsc2.2"
-cdsclient_version=FALSE
-
-# mpi4py Package
-mpi_url="https://github.com/mpi4py/mpi4py/archive/${mpi4py_ver}.tar.gz"
-mpi_tar="${mpi4py_ver}.tar.gz"
-mpi_dir="mpi4py-${mpi4py_ver}"
-mpi_bin="mpiexec"
-mpi_version=TRUE
-
-# Divider line
-line="########################################################################"
-
-# Text colours
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-CYAN='\033[0;36m'
-YELLOW='\033[1;33m'
-NC='\033[0m'
-
-# Set variable defaults
-PIPE_DIR=$PWD
-BUILD_DIR=$PIPE_DIR/build
-ENV_NAME='shapepipe'
-ENV_DEV=FALSE
-USE_CONDA_SOURCE=FALSE
-BUILD_ENV=TRUE
-DEVELOP=FALSE
-VOS=FALSE
-INSTALL_WW=TRUE
-INSTALL_CDSCLIENT=TRUE
-ATLAS_SEARCH=FALSE
-use_atlas=FALSE
-USE_MPI=TRUE
-ERROR=FALSE
-
-##############################################################################
-# SCRIPT FUNCTIONS
-##############################################################################
-
-# Function to download, and unzip and untar a package
-download_package() {
-  cd $BUILD_DIR
-  wget --no-check-certificate $1
-  tar -xzf $2
-  rm $2
-}
-
-# Function to build a standard package
-build_standard() {
-  cd $BUILD_DIR/$1
-  ./configure --prefix=$CONDA_PREFIX --quiet
-  make --quiet
-  make install
-}
-
-# Function to build mpi4pi using preinstalled MPI libraries
-build_mpi(){
-  cd $BUILD_DIR/$1
-  python setup.py build --mpicc=$MPI_ROOT
-  python setup.py install
-  ln -s $MPI_ROOT/bin/$mpi_bin $CONDA_PREFIX/bin/$mpi_bin
-}
-
-# Check the conda executable is available
-check_conda() {
-  if ! type -t "conda" > /dev/null
-  then
-    echo -ne "${RED}ERROR: Conda command not found, make sure it is installed before proceding.${NC}\n"
-    exit 1
-  fi
-  # Set Conda variables
-  CONDA_VERSION=$(conda -V | perl -pe '($_)=/([0-9]+([.][0-9]+)+)/' )
-  CONDA_VERSION_MAJOR=$(cut -d'.' -f1 <<<$CONDA_VERSION)
-  CONDA_VERSION_MINOR=$(cut -d'.' -f2 <<<$CONDA_VERSION)
-  CONDA_VERSION_PATCH=$(cut -d'.' -f3 <<<$CONDA_VERSION)
-  CONDA_SH=/etc/profile.d/conda.sh
-  # Check Conda major version
-  if [ "$CONDA_VERSION_MAJOR" -lt 4 ]
-  then
-    echo -ne "${RED}ERROR: Found Conda version $CONDA_VERSION but require 4.0.0 or greater.${NC}\n"
-    exit 1
-  fi
-  # Look for conda.sh file
-  if [ -f "/opt/conda/$CONDA_SH" ]
-  then
-    source "/opt/conda/$CONDA_SH"
-  elif [ -f "$CONDA_PREFIX$CONDA_SH" ]
-  then
-    source "$CONDA_PREFIX$CONDA_SH"
-  else
-    echo -ne "${RED}ERROR: Could not find $CONDA_SH in /opt/conda or \$CONDA_PREFIX.${NC}\n"
-    echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n"
-    exit 1
-  fi
-  # Set environment activation command
-  if [ "$USE_CONDA_SOURCE" == TRUE ]
-  then
-    CONDA_ACTIVATE="source activate"
-    CONDA_DEACTIVATE="source deactivate"
-  else
-    CONDA_ACTIVATE="conda activate"
-    CONDA_DEACTIVATE="conda deactivate"
-  fi
-
-}
-
-# Check if a binary executable is already installed in the conda environment
-check_binary() {
-  if ! type -t "$CONDA_PREFIX/bin/$1" > /dev/null
-  then
-    return 0
-  else
-    echo "Executable '$1' is already installed, skipping."
-    echo ""
-    return 1
-  fi
-}
-
-# Check if a binary executable is already installed in the conda environment
-check_binary2() {
-  if type -t "$CONDA_PREFIX/bin/$1" > /dev/null
-  then
-    echo -ne "$2: ${GREEN}INSTALLED${NC}\n"
-    if [ "$3" == TRUE ]
-    then
-      "$CONDA_PREFIX/bin/$1" --version | head -n 2
-    fi
-  else
-    if [ "$4" == TRUE ]
-    then
-      echo -ne "$2: ${RED}INSTALLATION FAILED${NC}\n"
-      ERROR=TRUE
-    else
-      echo -ne "$2: ${YELLOW}NOT INSTALLED${NC}\n"
-    fi
-  fi
-
-  echo ""
-}
-
-# Check the environment name
-check_env_name() {
-if [ "$ENV_DEV" == TRUE ]
-then
-  ENV_NAME='shapepipe-dev'
-fi
-}
-
-# Check if a python site-package is installed correctly in the conda environment
-check_site_pkg() {
-  if compgen -G "$CONDA_PREFIX/lib/python3.*/site-packages/$1*" > /dev/null
-  then
-    echo -ne "   - $1:\t \xe2\x9c\x85\n"
-  else
-    echo -ne "   - $1:\t \xe2\x9d\x8c\n"
-  fi
-}
-
-# Function to report progress
-report_progress() {
-  echo ''
-  echo $line
-  echo -e "${CYAN}Installing $1${NC}"
-  echo $line
-  echo ''
-}
-
-# Function to start the script
-start() {
-  echo ''
-  echo $line
-  echo 'ShapePipe Installation Script'
-  echo ''
-  echo 'Authors: Samuel Farrens and Martin Kilbinger'
-  echo 'Version:' $version
-  echo 'Last Update:' $last_update
-  echo $line
-  echo ''
-}
-
-# Function to report package versions
-packages() {
-  check_binary2 $ww_bin "WeightWatcher" $ww_version $INSTALL_WW
-  check_binary2 $cdsclient_bin "CDSclient" $cdsclient_version $INSTALL_CDSCLIENT
-  check_binary2 $mpi_bin "MPI" $mpi_version $USE_MPI $MPI_ROOT
-  check_binary2 "shapepipe_run" "ShapePipe_Run" TRUE TRUE
-}
-
-site-packages() {
-  check_site_pkg "shapepipe"
-  check_site_pkg "astropy"
-  check_site_pkg "galsim"
-  check_site_pkg "joblib"
-  check_site_pkg "mccd"
-  check_site_pkg "ngmix"
-  check_site_pkg "numpy"
-  check_site_pkg "pandas"
-  check_site_pkg "pysap"
-  check_site_pkg "scipy"
-  check_site_pkg "sf_tools"
-  check_site_pkg "sip_tpv"
-  check_site_pkg "sqlitedict"
-  check_site_pkg "treecorr"
-}
-
-# Function to return exit code
-exit_code() {
-  if [ "$ERROR" == TRUE ]
-  then
-    echo "ShapePipe installation failed as some packages were not properly installed."
-    exit 1
-  else
-    exit 0
-  fi
-}
-
-# Function to report package versions
-package_report() {
-
-  check_env_name
-  if [ "$CONDA_DEFAULT_ENV" != "$ENV_NAME" ]
-  then
-    check_conda
-    activate_shapepipe
-  fi
-  INSTALL_WW=FALSE
-  INSTALL_CDSCLIENT=FALSE
-  USE_MPI=FALSE
-  start
-  echo 'Package Report'
-  echo ''
-  packages
-  echo 'Python Package Check List:'
-  echo ''
-  site-packages
-  echo ''
-  exit_code
-}
-
-# Function to end the script
-finish() {
-  echo $line
-  echo ''
-  echo $line
-  echo 'Installation Complete!'
-  echo ''
-  packages
-  echo 'Python Package Check List:'
-  echo ''
-  site-packages
-  echo ''
-  echo $line
-  echo ''
-  exit_code
-}
-
-check_status () {
-  if [ "$2" == FALSE ]
-  then
-    echo -ne "$1 ${YELLOW}$2${NC}"
-  elif [ "$2" == TRUE ]
-  then
-    echo -ne "$1 ${GREEN}$2${NC}"
-  else
-    echo -ne "$1 ${CYAN}$2${NC}"
-  fi
-  echo -ne " ${CYAN}$3${NC}\n"
-}
-
-# Function to show the installation setup
-setup() {
-  check_status 'Operating System:' $SYSOS
-  check_status 'Conda Version:' $CONDA_VERSION
-  check_status 'Build Conda Environment:' $BUILD_ENV $ENV_NAME
-  check_status 'Install WeightWatcher:' $INSTALL_WW
-  check_status 'Install CDSclient:' $INSTALL_CDSCLIENT
-  check_status 'Use MPI:' $USE_MPI $MPI_ROOT
-  check_status 'ShapePipe Directory:' $PIPE_DIR
-  check_status 'Build Directory:' $BUILD_DIR
-  echo ''
-}
-
-# Function to show the astromatic setup
-astromatic_setup() {
-  export LDFLAGS="-lpthread"
-  echo 'LDFLAGS:' $LDFLAGS
-  echo 'FFTW Library Path:' $FFTW_LIB
-  echo 'FFTW Include Path:' $FFTW_INC
-  if [ "$use_atlas" == TRUE ]
-  then
-    echo 'ATLAS Library Path:' $ATLAS_LIB
-    echo 'ATLAS Include Path:' $ATLAS_INC
-  else
-    echo 'OpenBLAS Library Path:' $BLAS_LIB
-    echo 'OpenBLAS Include Path:' $BLAS_INC
-  fi
-  echo ''
-}
-
-# Function to uninstall ShapePipe and dependencies
-uninstall() {
-  check_env_name
-  read -p "Uninstall $ENV_NAME environment [y/N]: " response
-  response=${response:-n}
-  if [ $response == "y" ]
-  then
-    rm -r build dist src/*.egg-info .eggs
-    deactivate_shapepipe
-    conda remove -n $ENV_NAME --all -y
-    echo "Finished uninstalling $ENV_NAME"
-  fi
-}
-
-# Function to activate ShapePipe environment
-activate_shapepipe() {
-  # Run activation command
-  echo -ne "Running ${CYAN}$CONDA_ACTIVATE $ENV_NAME${NC}\n"
-  eval "$CONDA_ACTIVATE $ENV_NAME"
-  # Check if the activation was successful
-  if [ "$?" -eq 0 ]
-  then
-    echo -ne "Successfully activated ShapePipe environment ${CYAN}$CONDA_PREFIX${NC}\n"
-    export LD_RUN_PATH=$CONDA_PREFIX/lib
-  else
-    echo -ne "${RED}ERROR: Could not activate ShapePipe environment.${NC}\n"
-    exit 1
-  fi
-}
-
-# Function to deactivate ShapePipe environment
-deactivate_shapepipe() {
-  # Run deactivation command
-  eval $CONDA_DEACTIVATE
-}
-
-build_symlinks() {
-  echo "Creating symbolic links for scripts"
-  script_dirs=("scripts/python" "scripts/sh")
-  for dir in "${script_dirs[@]}"; do
-      for script in "$dir"/*; do
-          if [[ "$script" == *".sh" ]]; then
-              base_name=$(basename "$script" ".sh")
-          elif [[ "$script" == *".bash" ]]; then
-              base_name=$(basename "$script" ".bash")
-          elif [[ "$script" == *".py" ]]; then
-              base_name=$(basename "$script" ".py")
-          else
-              echo -ne "$2: ${YELLOW}WARNING:${NC}\n"
-              echo "Unknown script extension for $script."
-              continue
-          fi
-          # Create symbolic link inside $CONDA_PREFIX/bin without extension
-          ln -sf "$(pwd)/$script" "$CONDA_PREFIX/bin/$base_name"
-          # Make the symbolic link executable
-          chmod +x "$CONDA_PREFIX/bin/$base_name"
-      done
-  done
-}
-
-##############################################################################
-# SCRIPT OPTIONS
-##############################################################################
-
-help="$(basename "$0") [OPTIONS]\n\n
-Options:\n
-\t-h,--help\t show this help message and exit\n
-\t-v,--version\t print script version and exit\n
-\t-p,--packages\t print status of installed packages\n
-\t--uninstall\t uninstall ShapePipe\n
-\t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n
-\t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n
-\t--env-name\t set the name of the Conda environment (default is shapepipe)\n
-\t--env-dev\t use the developers Conda environment instead of the standard production environment\n
-\t--use-conda-source\t use 'conda source' instead of the default 'conda activate' to activate the shapepipe environment\n
-\t--develop\t option to install additional development packages\n
-\t--vos\t\t option to install virtual observatory software\n
-\t--no-env\t do not build Conda environment\n
-\t--no-ww\t\t do not build WeightWatcher\n
-\t--no-cds\t do not build CDSclient\n
-\t--no-exe\t do not build any system executables\n\n
-
-Executable Build Options:\n
-\t--fftw-lib\t path to FFTW library directory\n
-\t--fftw-inc\t path to FFTW include directory\n
-\t--blas-lib\t path to OpenBLAS library directory\n
-\t--blas-inc\t path to OpenBLAS include directory\n
-\t--atlas-lib\t path to ATLAS library directory\n
-\t--atlas-inc\t path to ATLAS include directory\n
-\t--atlas-search\t search system path for ATLAS directories\n\n
-
-MPI Build Options:\n
-\t--no-mpi\t do not use MPI\n
-\t--mpi-root\t path to MPI installation\n\n
-
-"
-
-# Parse command line options
-for i in "$@"
-do
-case $i in
-    -h|--help)
-    start
-    echo -ne $help
-    shift
-    exit
-    ;;
-    -v|--version)
-    echo $(basename "$0") $version
-    shift
-    exit
-    ;;
-    -p|--packages)
-    package_report
-    shift
-    exit
-    ;;
-    --uninstall)
-    uninstall
-    shift
-    exit
-    ;;
-    --pipe-dir=*)
-    PIPE_DIR="${i#*=}"
-    shift
-    ;;
-    --build-dir=*)
-    BUILD_DIR="${i#*=}"
-    shift
-    ;;
-    --env-name=*)
-    ENV_NAME="${i#*=}"
-    shift
-    ;;
-    --env-dev)
-    ENV_DEV=TRUE
-    shift
-    ;;
-    --use-conda-source)
-    USE_CONDA_SOURCE=TRUE
-    shift
-    ;;
-    --develop)
-    DEVELOP=TRUE
-    shift
-    ;;
-    --vos)
-    VOS=TRUE
-    shift
-    ;;
-    --no-env)
-    BUILD_ENV=FALSE
-    shift
-    ;;
-    --no-ww)
-    INSTALL_WW=FALSE
-    shift
-    ;;
-    --no-cds)
-    INSTALL_CDSCLIENT=FALSE
-    shift
-    ;;
-    --fftw-lib=*)
-    FFTW_LIB="${i#*=}"
-    shift
-    ;;
-    --fftw-inc=*)
-    FFTW_INC="${i#*=}"
-    shift
-    ;;
-    --blas-lib=*)
-    BLAS_LIB="${i#*=}"
-    shift
-    ;;
-    --blas-inc=*)
-    BLAS_INC="${i#*=}"
-    shift
-    ;;
-    --atlas-lib=*)
-    ATLAS_LIB="${i#*=}"
-    shift
-    ;;
-    --atlas-inc=*)
-    ATLAS_INC="${i#*=}"
-    shift
-    ;;
-    --atlas-search)
-    ATLAS_SEARCH=TRUE
-    shift
-    ;;
-    --no-mpi)
-    USE_MPI=FALSE
-    shift
-    ;;
-    --mpi-root=*)
-    MPI_ROOT="${i#*=}"
-    shift
-    ;;
-    --no-exe)
-    INSTALL_WW=FALSE
-    INSTALL_CDSCLIENT=FALSE
-    shift
-    ;;
-    *)
-    echo "Invalid option, see help!"
-    exit 1
-    ;;
-esac
-done
-
-##############################################################################
-# CREATE CONDA ENVIRONMENT
-##############################################################################
-
-# Start script
-start
-
-# Check if conda is installed
-check_conda
-
-# Check the Conda environment name
-check_env_name
-
-# Find the operating system
-case "$OSTYPE" in
-  darwin*)
-  SYSOS="macOS"
-  ;;
-  linux*)
-  SYSOS="LINUX"
-  ;;
-  *)
-  echo "unknown: $OSTYPE"
-  exit 1
-  ;;
-esac
-
-if [ ! -z "$ATLAS_LIB" ] || [ "$ATLAS_SEARCH" == TRUE ]
-then
-  use_atlas=TRUE
-fi
-
-# Create build directory if it does not already exist
-if [ ! -d "$BUILD_DIR" ]
-then
-  mkdir $BUILD_DIR
-fi
-
-# Print script set-up
-setup
-
-# Build conda environment
-if [ "$BUILD_ENV" == TRUE ]
-then
-  report_progress 'ShapePipe Environment'
-  if [ "$ENV_DEV" == TRUE ]
-  then
-    conda env create -f environment-dev.yml -n $ENV_NAME
-  else
-    conda env create -f environment.yml -n $ENV_NAME
-  fi
-fi
-
-# Activate conda environment
-activate_shapepipe
-
-if [ -z "$FFTW_LIB" ]; then FFTW_LIB=$CONDA_PREFIX/lib; fi
-if [ -z "$FFTW_INC" ]; then FFTW_INC=$CONDA_PREFIX/include; fi
-if [ -z "$BLAS_LIB" ]; then BLAS_LIB=$CONDA_PREFIX/lib; fi
-if [ -z "$BLAS_INC" ]; then BLAS_INC=$CONDA_PREFIX/include; fi
-
-##############################################################################
-# INSTALL THE VOS LIBRARY
-##############################################################################
-
-# Install the vos library
-if [ "$VOS" == TRUE ]
-then
-  report_progress 'vos'
-  python -m pip install vos
-fi
-
-##############################################################################
-# INSTALL MACOS REQUIRMENTS
-##############################################################################
-
-# Set up macOS environment
-if [ "$SYSOS" == "macOS" ]
-then
-  report_progress 'macOS Requirements'
-  echo "Setting C compiler flags"
-  export C_INCLUDE_PATH=$CONDA_PREFIX/include
-  export CFLAGS="-Wl,-rpath,$CONDA_PREFIX/lib"
-  export CPPFLAGS="-Wno-everything"
-else
-  export CPPFLAGS="-w"
-fi
-
-##############################################################################
-# INSTALL MPI REQUIRMENTS
-##############################################################################
-
-# Install mpi4py
-if [ "$USE_MPI" == TRUE ]
-then
-  report_progress 'MPI'
-  if [ -z "$MPI_ROOT" ]
-  then
-    if [ "$ENV_DEV" == TRUE ]
-    then
-      conda install -n $ENV_NAME -c conda-forge "mpi4py" -y
-    else
-      conda install -n $ENV_NAME -c conda-forge "mpi4py==${mpi4py_ver}" -y
-    fi
-  else
-    download_package $mpi_url $mpi_tar
-    build_mpi $mpi_dir
-  fi
-fi
-
-##############################################################################
-# BUILD EXTERNAL EXECUTABLES
-##############################################################################
-
-# Build WeightWatcher
-if [ "$INSTALL_WW" == TRUE ] && check_binary $ww_bin
-then
-  report_progress 'WeightWatcher'
-  download_package $ww_url $ww_tar
-  export CXXFLAGS="-fcommon"
-  build_standard $ww_dir
-fi
-
-# Build CDSclient
-if [ "$INSTALL_CDSCLIENT" == TRUE ] && check_binary $cdsclient_bin
-then
-  report_progress 'CDSclient'
-  download_package $cdsclient_url $cdscleint_tar
-  build_standard $cdsclient_dir
-fi
-
-##############################################################################
-# INSTALL SHAPEPIPE LIBRARY
-##############################################################################
-
-# Install ShapePipe package
-report_progress 'ShapePipe'
-cd $PIPE_DIR
-
-if [ "$DEVELOP" == "TRUE" ] || [ "$ENV_DEV" == "TRUE" ] 
-then
-  echo "Installing ShapePipe package with developer tools"
-  python -m pip install ".[dev]"
-else 
-  echo "Installing ShapePipe package"
-  python -m pip install .
-fi
-
-# Create symbolic links to scripts
-build_symlinks
-
-finish
-
-##############################################################################

From 68488eeab54756371af6d73b291f465acd4c3e89 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Thu, 27 Feb 2025 17:05:31 +0100
Subject: [PATCH 03/25] bump python version to 3.11

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 0747a8ef..25b1eb32 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.9-bookworm
+FROM python:3.11-bookworm
 
 LABEL Description="Conda-Free ShapePipe Docker Image"
 ENV SHELL=/bin/bash

From 4388dc7c37b091d861a48a7673c0ab29c12e3467 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Thu, 27 Feb 2025 17:07:07 +0100
Subject: [PATCH 04/25] bump package version to 1.1

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index ab87fac6..bd83613c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "shapepipe"
-version = "1.0.2"
+version = "1.1.0"
 description = "Galaxy shape measurement pipeline."
 readme = "README.rst"
 authors = [

From 5a8dff3c7adb1dfade891947023290b8a139dce1 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Thu, 27 Feb 2025 17:50:39 +0100
Subject: [PATCH 05/25] dockerfile fix

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 25b1eb32..feec7d0c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -100,4 +100,4 @@ RUN pip install --no-cache-dir -e . && \
     link_name=`basename $script $ext`; \
     ln -s $script /usr/local/bin/$link_name; \
     done; \
-    done
\ No newline at end of file
+    done

From da4df3e48905a6582112a088281e0c6495224412 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 11:07:50 +0100
Subject: [PATCH 06/25] remove pinned versions from reproject, mpi4py

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index feec7d0c..a81c528e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -65,7 +65,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
     matplotlib==3.8.4 \
     mccd==1.2.4 \
     modopt==1.6.1 \
-    mpi4py==3.1.3 \
+    mpi4py \
     numba==0.58.1 \
     numpy==1.26.4 \
     numpydoc==1.2  \
@@ -77,7 +77,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
     pytest-pycodestyle==2.4.1 \
     pytest-pydocstyle==2.4.0 \
     python-pysap==0.2.1 \
-    reproject==0.8 \
+    reproject \
     sf_tools==2.0.4 \
     sip_tpv==1.1 \
     skaha==1.4.3 \

From 2df662a7eec7c7f7fbce4d6bb848ddea5e9c62b4 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 11:35:40 +0100
Subject: [PATCH 07/25] bump reproject and mpi4py versions to be compatible
 with python 3.11

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index a81c528e..4049e5c9 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -65,7 +65,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
     matplotlib==3.8.4 \
     mccd==1.2.4 \
     modopt==1.6.1 \
-    mpi4py \
+    mpi4py==4.0.3 \
     numba==0.58.1 \
     numpy==1.26.4 \
     numpydoc==1.2  \
@@ -77,7 +77,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
     pytest-pycodestyle==2.4.1 \
     pytest-pydocstyle==2.4.0 \
     python-pysap==0.2.1 \
-    reproject \
+    reproject==0.14.1 \
     sf_tools==2.0.4 \
     sip_tpv==1.1 \
     skaha==1.4.3 \

From 62223a32ef94746efa9d09538ac1729e47b3ccec Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 11:45:08 +0100
Subject: [PATCH 08/25] add develop version of pysap

---
 Dockerfile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4049e5c9..0e7687ad 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -76,7 +76,6 @@ RUN pip install --no-cache-dir --upgrade pip && \
     pytest-cov==5.0.0 \
     pytest-pycodestyle==2.4.1 \
     pytest-pydocstyle==2.4.0 \
-    python-pysap==0.2.1 \
     reproject==0.14.1 \
     sf_tools==2.0.4 \
     sip_tpv==1.1 \
@@ -87,7 +86,8 @@ RUN pip install --no-cache-dir --upgrade pip && \
     treecorr==4.2.6 \
     vos==3.6.1.1 \
     git+https://github.com/aguinot/ngmix@stable_version \
-    git+https://github.com/tobias-liaudat/Stile@v0.1
+    git+https://github.com/tobias-liaudat/Stile@v0.1 \ 
+    git+https://github.com/CEA-COSMIC/pysap@develop
 
 
 WORKDIR /app

From bfd80d86364c764ca42136f6c98b8b4c82a821f9 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 13:32:00 +0100
Subject: [PATCH 09/25] add back conda stuff so i don't have to rewrite lots of
 documentation

---
 environment-dev.yml |  38 +++
 environment.yml     |  46 +++
 install_shapepipe   | 702 ++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 786 insertions(+)
 create mode 100644 environment-dev.yml
 create mode 100644 environment.yml
 create mode 100755 install_shapepipe

diff --git a/environment-dev.yml b/environment-dev.yml
new file mode 100644
index 00000000..703c1917
--- /dev/null
+++ b/environment-dev.yml
@@ -0,0 +1,38 @@
+# ShapePipe Development Environment
+# ---------------------------------
+# This environment should only be used for development and testing of  
+# ShapePipe (i.e. nothing that could be published). Packages need only
+# specify the minimum compatible versions. This environment can updated 
+# as needed.
+name: shapepipe-dev
+channels:
+  - conda-forge
+dependencies:
+  - python>=3.9.18
+  - pip>=23.3.1
+  - astropy>=5.1.1
+  - autoconf>=2.71
+  - automake>=1.16.5
+  - cmake>=3.27.7
+  - galsim>=2.5.1
+  - joblib>=1.3.2
+  - libtool>=2.4
+  - matplotlib>=3.8.1
+  - numba>=0.58.1
+  - pandas>=2.1
+  - pyqt5-sip>=12.13
+  - pyqtgraph>=0.13.3
+  - reproject>=0.12
+  - sqlitedict>=2.1
+  - termcolor>=2.3
+  - tqdm>=4.66.1
+  - treecorr>=4.3.3
+  - pip:
+    - cs_util==0.0.5
+    - mccd==1.2.4
+    - modopt==1.6.1
+    - sip_tpv==1.1
+    - sf_tools==2.0.4
+    - git+https://github.com/CEA-COSMIC/pysap@develop
+    - git+https://github.com/aguinot/ngmix@stable_version
+    - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/environment.yml b/environment.yml
new file mode 100644
index 00000000..ebb04d2c
--- /dev/null
+++ b/environment.yml
@@ -0,0 +1,46 @@
+name: shapepipe_v1.4
+# ShapePipe Production Environment
+# --------------------------------
+# This environment should be used for any production runs of ShapePipe 
+# (i.e. anything that could be published). All packages are pinned to
+# specific versions to maximise the reproducibility of the results.
+# This environment should be updated for each release of ShapePipe.
+channels:
+  - conda-forge
+  - defaults
+dependencies:
+  - python=3.9
+  - pip>=21.2.4
+  - astromatic-psfex==3.21.1
+  - astromatic-source-extractor==2.25.0
+  - astropy==5.2
+  - automake
+  - autoconf
+  - cmake
+  - h5py
+  - galsim
+  - joblib
+  - libtool
+  - matplotlib==3.8.4
+  - numba==0.58.1
+  - numpy==1.26.4
+  - pandas
+  - pip:
+    - cs_util==0.1
+    - mccd==1.2.4
+    - modopt
+    - PyQt5
+    - pyqtgraph
+    - python-pysap==0.2.1
+    - reproject
+    - shear_psf_leakage
+    - skaha
+    - sip_tpv
+    - sf_tools
+    - sqlitedict
+    - termcolor
+    - tqdm
+    - treecorr
+    - vos
+    - git+https://github.com/aguinot/ngmix@stable_version
+    - git+https://github.com/tobias-liaudat/Stile@v0.1
diff --git a/install_shapepipe b/install_shapepipe
new file mode 100755
index 00000000..263c4e8c
--- /dev/null
+++ b/install_shapepipe
@@ -0,0 +1,702 @@
+#!/usr/bin/env bash
+export LC_ALL=en_US.UTF-8
+export LC_CTYPE=en_US.utf8
+
+##############################################################################
+# Install ShapePipe Executables
+#
+# This script can only be run after the ShapePipe environment has been
+# installed.
+#
+##############################################################################
+
+version="1.8"
+last_update="13/10/24"
+
+##############################################################################
+# EXTERNAL PACKAGES AND SCRIPTS
+##############################################################################
+
+# External packages
+
+# Conda package versions
+fftw_ver="3.3.10"
+libpng_ver="1.6.37"
+mpi4py_ver="4.0.1"
+openblas_ver="0.3.18"
+
+# WeightWatcher Package
+ww_url="https://github.com/astromatic/weightwatcher/archive/refs/tags/1.12.tar.gz"
+ww_tar="1.12.tar.gz"
+ww_dir="weightwatcher-1.12"
+ww_bin="ww"
+ww_version=TRUE
+
+# CDSclient Package (need Findgsc2.2)
+cdsclient_url="http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz"
+cdscleint_tar="cdsclient.tar.gz"
+cdsclient_dir="cdsclient-4.07"
+cdsclient_bin="findgsc2.2"
+cdsclient_version=FALSE
+
+# mpi4py Package
+mpi_url="https://github.com/mpi4py/mpi4py/archive/${mpi4py_ver}.tar.gz"
+mpi_tar="${mpi4py_ver}.tar.gz"
+mpi_dir="mpi4py-${mpi4py_ver}"
+mpi_bin="mpiexec"
+mpi_version=TRUE
+
+# Divider line
+line="########################################################################"
+
+# Text colours
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+CYAN='\033[0;36m'
+YELLOW='\033[1;33m'
+NC='\033[0m'
+
+# Set variable defaults
+PIPE_DIR=$PWD
+BUILD_DIR=$PIPE_DIR/build
+ENV_NAME='shapepipe'
+ENV_DEV=FALSE
+USE_CONDA_SOURCE=FALSE
+BUILD_ENV=TRUE
+DEVELOP=FALSE
+VOS=FALSE
+INSTALL_WW=TRUE
+INSTALL_CDSCLIENT=TRUE
+ATLAS_SEARCH=FALSE
+use_atlas=FALSE
+USE_MPI=TRUE
+ERROR=FALSE
+
+##############################################################################
+# SCRIPT FUNCTIONS
+##############################################################################
+
+# Function to download, and unzip and untar a package
+download_package() {
+  cd $BUILD_DIR
+  wget --no-check-certificate $1
+  tar -xzf $2
+  rm $2
+}
+
+# Function to build a standard package
+build_standard() {
+  cd $BUILD_DIR/$1
+  ./configure --prefix=$CONDA_PREFIX --quiet
+  make --quiet
+  make install
+}
+
+# Function to build mpi4pi using preinstalled MPI libraries
+build_mpi(){
+  cd $BUILD_DIR/$1
+  python setup.py build --mpicc=$MPI_ROOT
+  python setup.py install
+  ln -s $MPI_ROOT/bin/$mpi_bin $CONDA_PREFIX/bin/$mpi_bin
+}
+
+# Check the conda executable is available
+check_conda() {
+  if ! type -t "conda" > /dev/null
+  then
+    echo -ne "${RED}ERROR: Conda command not found, make sure it is installed before proceding.${NC}\n"
+    exit 1
+  fi
+  # Set Conda variables
+  CONDA_VERSION=$(conda -V | perl -pe '($_)=/([0-9]+([.][0-9]+)+)/' )
+  CONDA_VERSION_MAJOR=$(cut -d'.' -f1 <<<$CONDA_VERSION)
+  CONDA_VERSION_MINOR=$(cut -d'.' -f2 <<<$CONDA_VERSION)
+  CONDA_VERSION_PATCH=$(cut -d'.' -f3 <<<$CONDA_VERSION)
+  CONDA_SH=/etc/profile.d/conda.sh
+  # Check Conda major version
+  if [ "$CONDA_VERSION_MAJOR" -lt 4 ]
+  then
+    echo -ne "${RED}ERROR: Found Conda version $CONDA_VERSION but require 4.0.0 or greater.${NC}\n"
+    exit 1
+  fi
+  # Look for conda.sh file
+  if [ -f "/opt/conda/$CONDA_SH" ]
+  then
+    source "/opt/conda/$CONDA_SH"
+  elif [ -f "$CONDA_PREFIX$CONDA_SH" ]
+  then
+    source "$CONDA_PREFIX$CONDA_SH"
+  else
+    echo -ne "${RED}ERROR: Could not find $CONDA_SH in /opt/conda or \$CONDA_PREFIX.${NC}\n"
+    echo -ne "${RED}Activate the base/root Conda environment and try again.${NC}\n"
+    exit 1
+  fi
+  # Set environment activation command
+  if [ "$USE_CONDA_SOURCE" == TRUE ]
+  then
+    CONDA_ACTIVATE="source activate"
+    CONDA_DEACTIVATE="source deactivate"
+  else
+    CONDA_ACTIVATE="conda activate"
+    CONDA_DEACTIVATE="conda deactivate"
+  fi
+
+}
+
+# Check if a binary executable is already installed in the conda environment
+check_binary() {
+  if ! type -t "$CONDA_PREFIX/bin/$1" > /dev/null
+  then
+    return 0
+  else
+    echo "Executable '$1' is already installed, skipping."
+    echo ""
+    return 1
+  fi
+}
+
+# Check if a binary executable is already installed in the conda environment
+check_binary2() {
+  if type -t "$CONDA_PREFIX/bin/$1" > /dev/null
+  then
+    echo -ne "$2: ${GREEN}INSTALLED${NC}\n"
+    if [ "$3" == TRUE ]
+    then
+      "$CONDA_PREFIX/bin/$1" --version | head -n 2
+    fi
+  else
+    if [ "$4" == TRUE ]
+    then
+      echo -ne "$2: ${RED}INSTALLATION FAILED${NC}\n"
+      ERROR=TRUE
+    else
+      echo -ne "$2: ${YELLOW}NOT INSTALLED${NC}\n"
+    fi
+  fi
+
+  echo ""
+}
+
+# Check the environment name
+check_env_name() {
+if [ "$ENV_DEV" == TRUE ]
+then
+  ENV_NAME='shapepipe-dev'
+fi
+}
+
+# Check if a python site-package is installed correctly in the conda environment
+check_site_pkg() {
+  if compgen -G "$CONDA_PREFIX/lib/python3.*/site-packages/$1*" > /dev/null
+  then
+    echo -ne "   - $1:\t \xe2\x9c\x85\n"
+  else
+    echo -ne "   - $1:\t \xe2\x9d\x8c\n"
+  fi
+}
+
+# Function to report progress
+report_progress() {
+  echo ''
+  echo $line
+  echo -e "${CYAN}Installing $1${NC}"
+  echo $line
+  echo ''
+}
+
+# Function to start the script
+start() {
+  echo ''
+  echo $line
+  echo 'ShapePipe Installation Script'
+  echo ''
+  echo 'Authors: Samuel Farrens and Martin Kilbinger'
+  echo 'Version:' $version
+  echo 'Last Update:' $last_update
+  echo $line
+  echo ''
+}
+
+# Function to report package versions
+packages() {
+  check_binary2 $ww_bin "WeightWatcher" $ww_version $INSTALL_WW
+  check_binary2 $cdsclient_bin "CDSclient" $cdsclient_version $INSTALL_CDSCLIENT
+  check_binary2 $mpi_bin "MPI" $mpi_version $USE_MPI $MPI_ROOT
+  check_binary2 "shapepipe_run" "ShapePipe_Run" TRUE TRUE
+}
+
+site-packages() {
+  check_site_pkg "shapepipe"
+  check_site_pkg "astropy"
+  check_site_pkg "galsim"
+  check_site_pkg "joblib"
+  check_site_pkg "mccd"
+  check_site_pkg "ngmix"
+  check_site_pkg "numpy"
+  check_site_pkg "pandas"
+  check_site_pkg "pysap"
+  check_site_pkg "scipy"
+  check_site_pkg "sf_tools"
+  check_site_pkg "sip_tpv"
+  check_site_pkg "sqlitedict"
+  check_site_pkg "treecorr"
+}
+
+# Function to return exit code
+exit_code() {
+  if [ "$ERROR" == TRUE ]
+  then
+    echo "ShapePipe installation failed as some packages were not properly installed."
+    exit 1
+  else
+    exit 0
+  fi
+}
+
+# Function to report package versions
+package_report() {
+
+  check_env_name
+  if [ "$CONDA_DEFAULT_ENV" != "$ENV_NAME" ]
+  then
+    check_conda
+    activate_shapepipe
+  fi
+  INSTALL_WW=FALSE
+  INSTALL_CDSCLIENT=FALSE
+  USE_MPI=FALSE
+  start
+  echo 'Package Report'
+  echo ''
+  packages
+  echo 'Python Package Check List:'
+  echo ''
+  site-packages
+  echo ''
+  exit_code
+}
+
+# Function to end the script
+finish() {
+  echo $line
+  echo ''
+  echo $line
+  echo 'Installation Complete!'
+  echo ''
+  packages
+  echo 'Python Package Check List:'
+  echo ''
+  site-packages
+  echo ''
+  echo $line
+  echo ''
+  exit_code
+}
+
+check_status () {
+  if [ "$2" == FALSE ]
+  then
+    echo -ne "$1 ${YELLOW}$2${NC}"
+  elif [ "$2" == TRUE ]
+  then
+    echo -ne "$1 ${GREEN}$2${NC}"
+  else
+    echo -ne "$1 ${CYAN}$2${NC}"
+  fi
+  echo -ne " ${CYAN}$3${NC}\n"
+}
+
+# Function to show the installation setup
+setup() {
+  check_status 'Operating System:' $SYSOS
+  check_status 'Conda Version:' $CONDA_VERSION
+  check_status 'Build Conda Environment:' $BUILD_ENV $ENV_NAME
+  check_status 'Install WeightWatcher:' $INSTALL_WW
+  check_status 'Install CDSclient:' $INSTALL_CDSCLIENT
+  check_status 'Use MPI:' $USE_MPI $MPI_ROOT
+  check_status 'ShapePipe Directory:' $PIPE_DIR
+  check_status 'Build Directory:' $BUILD_DIR
+  echo ''
+}
+
+# Function to show the astromatic setup
+astromatic_setup() {
+  export LDFLAGS="-lpthread"
+  echo 'LDFLAGS:' $LDFLAGS
+  echo 'FFTW Library Path:' $FFTW_LIB
+  echo 'FFTW Include Path:' $FFTW_INC
+  if [ "$use_atlas" == TRUE ]
+  then
+    echo 'ATLAS Library Path:' $ATLAS_LIB
+    echo 'ATLAS Include Path:' $ATLAS_INC
+  else
+    echo 'OpenBLAS Library Path:' $BLAS_LIB
+    echo 'OpenBLAS Include Path:' $BLAS_INC
+  fi
+  echo ''
+}
+
+# Function to uninstall ShapePipe and dependencies
+uninstall() {
+  check_env_name
+  read -p "Uninstall $ENV_NAME environment [y/N]: " response
+  response=${response:-n}
+  if [ $response == "y" ]
+  then
+    rm -r build dist src/*.egg-info .eggs
+    deactivate_shapepipe
+    conda remove -n $ENV_NAME --all -y
+    echo "Finished uninstalling $ENV_NAME"
+  fi
+}
+
+# Function to activate ShapePipe environment
+activate_shapepipe() {
+  # Run activation command
+  echo -ne "Running ${CYAN}$CONDA_ACTIVATE $ENV_NAME${NC}\n"
+  eval "$CONDA_ACTIVATE $ENV_NAME"
+  # Check if the activation was successful
+  if [ "$?" -eq 0 ]
+  then
+    echo -ne "Successfully activated ShapePipe environment ${CYAN}$CONDA_PREFIX${NC}\n"
+    export LD_RUN_PATH=$CONDA_PREFIX/lib
+  else
+    echo -ne "${RED}ERROR: Could not activate ShapePipe environment.${NC}\n"
+    exit 1
+  fi
+}
+
+# Function to deactivate ShapePipe environment
+deactivate_shapepipe() {
+  # Run deactivation command
+  eval $CONDA_DEACTIVATE
+}
+
+build_symlinks() {
+  echo "Creating symbolic links for scripts"
+  script_dirs=("scripts/python" "scripts/sh")
+  for dir in "${script_dirs[@]}"; do
+      for script in "$dir"/*; do
+          if [[ "$script" == *".sh" ]]; then
+              base_name=$(basename "$script" ".sh")
+          elif [[ "$script" == *".bash" ]]; then
+              base_name=$(basename "$script" ".bash")
+          elif [[ "$script" == *".py" ]]; then
+              base_name=$(basename "$script" ".py")
+          else
+              echo -ne "$2: ${YELLOW}WARNING:${NC}\n"
+              echo "Unknown script extension for $script."
+              continue
+          fi
+          # Create symbolic link inside $CONDA_PREFIX/bin without extension
+          ln -sf "$(pwd)/$script" "$CONDA_PREFIX/bin/$base_name"
+          # Make the symbolic link executable
+          chmod +x "$CONDA_PREFIX/bin/$base_name"
+      done
+  done
+}
+
+##############################################################################
+# SCRIPT OPTIONS
+##############################################################################
+
+help="$(basename "$0") [OPTIONS]\n\n
+Options:\n
+\t-h,--help\t show this help message and exit\n
+\t-v,--version\t print script version and exit\n
+\t-p,--packages\t print status of installed packages\n
+\t--uninstall\t uninstall ShapePipe\n
+\t--pipe-dir\t set the path to the ShapePipe base directory (default is \$PWD)\n
+\t--build-dir\t set the path to the ShapePipe build (default is \$PWD/build)\n
+\t--env-name\t set the name of the Conda environment (default is shapepipe)\n
+\t--env-dev\t use the developers Conda environment instead of the standard production environment\n
+\t--use-conda-source\t use 'conda source' instead of the default 'conda activate' to activate the shapepipe environment\n
+\t--develop\t option to install additional development packages\n
+\t--vos\t\t option to install virtual observatory software\n
+\t--no-env\t do not build Conda environment\n
+\t--no-ww\t\t do not build WeightWatcher\n
+\t--no-cds\t do not build CDSclient\n
+\t--no-exe\t do not build any system executables\n\n
+
+Executable Build Options:\n
+\t--fftw-lib\t path to FFTW library directory\n
+\t--fftw-inc\t path to FFTW include directory\n
+\t--blas-lib\t path to OpenBLAS library directory\n
+\t--blas-inc\t path to OpenBLAS include directory\n
+\t--atlas-lib\t path to ATLAS library directory\n
+\t--atlas-inc\t path to ATLAS include directory\n
+\t--atlas-search\t search system path for ATLAS directories\n\n
+
+MPI Build Options:\n
+\t--no-mpi\t do not use MPI\n
+\t--mpi-root\t path to MPI installation\n\n
+
+"
+
+# Parse command line options
+for i in "$@"
+do
+case $i in
+    -h|--help)
+    start
+    echo -ne $help
+    shift
+    exit
+    ;;
+    -v|--version)
+    echo $(basename "$0") $version
+    shift
+    exit
+    ;;
+    -p|--packages)
+    package_report
+    shift
+    exit
+    ;;
+    --uninstall)
+    uninstall
+    shift
+    exit
+    ;;
+    --pipe-dir=*)
+    PIPE_DIR="${i#*=}"
+    shift
+    ;;
+    --build-dir=*)
+    BUILD_DIR="${i#*=}"
+    shift
+    ;;
+    --env-name=*)
+    ENV_NAME="${i#*=}"
+    shift
+    ;;
+    --env-dev)
+    ENV_DEV=TRUE
+    shift
+    ;;
+    --use-conda-source)
+    USE_CONDA_SOURCE=TRUE
+    shift
+    ;;
+    --develop)
+    DEVELOP=TRUE
+    shift
+    ;;
+    --vos)
+    VOS=TRUE
+    shift
+    ;;
+    --no-env)
+    BUILD_ENV=FALSE
+    shift
+    ;;
+    --no-ww)
+    INSTALL_WW=FALSE
+    shift
+    ;;
+    --no-cds)
+    INSTALL_CDSCLIENT=FALSE
+    shift
+    ;;
+    --fftw-lib=*)
+    FFTW_LIB="${i#*=}"
+    shift
+    ;;
+    --fftw-inc=*)
+    FFTW_INC="${i#*=}"
+    shift
+    ;;
+    --blas-lib=*)
+    BLAS_LIB="${i#*=}"
+    shift
+    ;;
+    --blas-inc=*)
+    BLAS_INC="${i#*=}"
+    shift
+    ;;
+    --atlas-lib=*)
+    ATLAS_LIB="${i#*=}"
+    shift
+    ;;
+    --atlas-inc=*)
+    ATLAS_INC="${i#*=}"
+    shift
+    ;;
+    --atlas-search)
+    ATLAS_SEARCH=TRUE
+    shift
+    ;;
+    --no-mpi)
+    USE_MPI=FALSE
+    shift
+    ;;
+    --mpi-root=*)
+    MPI_ROOT="${i#*=}"
+    shift
+    ;;
+    --no-exe)
+    INSTALL_WW=FALSE
+    INSTALL_CDSCLIENT=FALSE
+    shift
+    ;;
+    *)
+    echo "Invalid option, see help!"
+    exit 1
+    ;;
+esac
+done
+
+##############################################################################
+# CREATE CONDA ENVIRONMENT
+##############################################################################
+
+# Start script
+start
+
+# Check if conda is installed
+check_conda
+
+# Check the Conda environment name
+check_env_name
+
+# Find the operating system
+case "$OSTYPE" in
+  darwin*)
+  SYSOS="macOS"
+  ;;
+  linux*)
+  SYSOS="LINUX"
+  ;;
+  *)
+  echo "unknown: $OSTYPE"
+  exit 1
+  ;;
+esac
+
+if [ ! -z "$ATLAS_LIB" ] || [ "$ATLAS_SEARCH" == TRUE ]
+then
+  use_atlas=TRUE
+fi
+
+# Create build directory if it does not already exist
+if [ ! -d "$BUILD_DIR" ]
+then
+  mkdir $BUILD_DIR
+fi
+
+# Print script set-up
+setup
+
+# Build conda environment
+if [ "$BUILD_ENV" == TRUE ]
+then
+  report_progress 'ShapePipe Environment'
+  if [ "$ENV_DEV" == TRUE ]
+  then
+    conda env create -f environment-dev.yml -n $ENV_NAME
+  else
+    conda env create -f environment.yml -n $ENV_NAME
+  fi
+fi
+
+# Activate conda environment
+activate_shapepipe
+
+if [ -z "$FFTW_LIB" ]; then FFTW_LIB=$CONDA_PREFIX/lib; fi
+if [ -z "$FFTW_INC" ]; then FFTW_INC=$CONDA_PREFIX/include; fi
+if [ -z "$BLAS_LIB" ]; then BLAS_LIB=$CONDA_PREFIX/lib; fi
+if [ -z "$BLAS_INC" ]; then BLAS_INC=$CONDA_PREFIX/include; fi
+
+##############################################################################
+# INSTALL THE VOS LIBRARY
+##############################################################################
+
+# Install the vos library
+if [ "$VOS" == TRUE ]
+then
+  report_progress 'vos'
+  python -m pip install vos
+fi
+
+##############################################################################
+# INSTALL MACOS REQUIRMENTS
+##############################################################################
+
+# Set up macOS environment
+if [ "$SYSOS" == "macOS" ]
+then
+  report_progress 'macOS Requirements'
+  echo "Setting C compiler flags"
+  export C_INCLUDE_PATH=$CONDA_PREFIX/include
+  export CFLAGS="-Wl,-rpath,$CONDA_PREFIX/lib"
+  export CPPFLAGS="-Wno-everything"
+else
+  export CPPFLAGS="-w"
+fi
+
+##############################################################################
+# INSTALL MPI REQUIRMENTS
+##############################################################################
+
+# Install mpi4py
+if [ "$USE_MPI" == TRUE ]
+then
+  report_progress 'MPI'
+  if [ -z "$MPI_ROOT" ]
+  then
+    if [ "$ENV_DEV" == TRUE ]
+    then
+      conda install -n $ENV_NAME -c conda-forge "mpi4py" -y
+    else
+      conda install -n $ENV_NAME -c conda-forge "mpi4py==${mpi4py_ver}" -y
+    fi
+  else
+    download_package $mpi_url $mpi_tar
+    build_mpi $mpi_dir
+  fi
+fi
+
+##############################################################################
+# BUILD EXTERNAL EXECUTABLES
+##############################################################################
+
+# Build WeightWatcher
+if [ "$INSTALL_WW" == TRUE ] && check_binary $ww_bin
+then
+  report_progress 'WeightWatcher'
+  download_package $ww_url $ww_tar
+  export CXXFLAGS="-fcommon"
+  build_standard $ww_dir
+fi
+
+# Build CDSclient
+if [ "$INSTALL_CDSCLIENT" == TRUE ] && check_binary $cdsclient_bin
+then
+  report_progress 'CDSclient'
+  download_package $cdsclient_url $cdscleint_tar
+  build_standard $cdsclient_dir
+fi
+
+##############################################################################
+# INSTALL SHAPEPIPE LIBRARY
+##############################################################################
+
+# Install ShapePipe package
+report_progress 'ShapePipe'
+cd $PIPE_DIR
+
+if [ "$DEVELOP" == "TRUE" ] || [ "$ENV_DEV" == "TRUE" ] 
+then
+  echo "Installing ShapePipe package with developer tools"
+  python -m pip install ".[dev]"
+else 
+  echo "Installing ShapePipe package"
+  python -m pip install .
+fi
+
+# Create symbolic links to scripts
+build_symlinks
+
+finish
+
+##############################################################################

From 38cb7ed5c39d2b9a7e464f1f8e8b596d113f888c Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 13:46:42 +0100
Subject: [PATCH 10/25] fix undefined __installs__ variable in run.py

---
 src/shapepipe/run.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py
index 19880748..82e773b1 100644
--- a/src/shapepipe/run.py
+++ b/src/shapepipe/run.py
@@ -14,7 +14,7 @@
 from modopt.interface.errors import catch_error
 from modopt.interface.log import close_log, set_up_log
 
-from shapepipe.info import line, shapepipe_logo
+from shapepipe.info import line, shapepipe_logo, __installs__
 from shapepipe.pipeline.args import create_arg_parser
 from shapepipe.pipeline.config import create_config_parser
 from shapepipe.pipeline.dependency_handler import DependencyHandler

From 88553924dee2de701a7f5b72a47a8e307ed6405a Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 14:29:07 +0100
Subject: [PATCH 11/25] add container install instructions

---
 docs/source/installation.md | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/docs/source/installation.md b/docs/source/installation.md
index 63f030c1..11638810 100644
--- a/docs/source/installation.md
+++ b/docs/source/installation.md
@@ -6,7 +6,32 @@ are expected to install the full ShapePipe environment on the system(s) where
 data should be processed.
 ```
 
-## Standard Installation
+## Container Installation (Recommended)
+
+The easiest way to install ShapePipe is via a container. Docker images are automatically built and pushed to the [Github Container Registry (GHCR)](ghcr.io/cosmostat/shapepipe) for each release. This images can be installed and run on most systems (including clusters) with just a few lines of code.
+
+We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command:
+```bash
+# build writeable "sandbox" container in the current directory
+# ./shapepipe will be a directory that functions like a vm
+apptainer build --sandbox shapepipe docker://ghcr.io/cosmostat/shapepipe:develop
+
+# open a shell in the container and run the example
+apptainer shell --writable shapepipe
+cd /app && shapepipe_run -c /app/example/config.ini
+```
+
+You can also run the image with **Docker**:
+
+```bash
+docker run --rm -it ghcr.io/cosmostat/shapepipe:develop shapepipe_run -c /app/example/config.ini  
+```
+
+```{attention}
+We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance.
+```
+
+## Conda Installation (Deprecated)
 
 ```{tip}
 :class: margin

From bfbcf24c3965298edf7f6b9cda32fa461bc17851 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 14:29:35 +0100
Subject: [PATCH 12/25] newline in docs

---
 docs/source/installation.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/source/installation.md b/docs/source/installation.md
index 11638810..831f12cc 100644
--- a/docs/source/installation.md
+++ b/docs/source/installation.md
@@ -11,6 +11,7 @@ data should be processed.
 The easiest way to install ShapePipe is via a container. Docker images are automatically built and pushed to the [Github Container Registry (GHCR)](ghcr.io/cosmostat/shapepipe) for each release. This images can be installed and run on most systems (including clusters) with just a few lines of code.
 
 We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command:
+
 ```bash
 # build writeable "sandbox" container in the current directory
 # ./shapepipe will be a directory that functions like a vm

From 99255c796a1de24b6d7fc5a1e57c7b5a7196d4cf Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 14:56:58 +0100
Subject: [PATCH 13/25] attempt to fix conda action

---
 .github/workflows/ci-dev.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index 9e666f77..b7fdbef7 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -32,11 +32,13 @@ jobs:
           brew install bigmac libomp
 
       - name: Set up conda
-        uses: conda-incubator/setup-miniconda@v2
+        uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           auto-activate-base: true
+          activate-environment: ""
+          miniconda-version: "latest"
 
       - name: Install package
         shell: bash -l {0}

From 0c79be9aacdc4f4f4c210d89a5798bf6ddd0997c Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 14:58:12 +0100
Subject: [PATCH 14/25] update conda python version

---
 .github/workflows/ci-dev.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index b7fdbef7..b691d0ad 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -15,7 +15,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.8]
+        python-version: [3.11]
 
     steps:
       - uses: actions/checkout@v2

From d728a2942950fe247be7a21d0d5f41bfda347a64 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 14:59:12 +0100
Subject: [PATCH 15/25] remove active-environment setting

---
 .github/workflows/ci-dev.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index b691d0ad..adc26ed5 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -37,7 +37,6 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
           auto-activate-base: true
-          activate-environment: ""
           miniconda-version: "latest"
 
       - name: Install package

From 7584999c7bc3062a5146d0755c48968b47aa65b3 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 15:06:00 +0100
Subject: [PATCH 16/25] try one more conda fix

---
 .github/workflows/ci-dev.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index adc26ed5..de2a5ba6 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -42,6 +42,7 @@ jobs:
       - name: Install package
         shell: bash -l {0}
         run: |
+          conda activate test
           ./install_shapepipe --env-dev --develop
 
       - name: Run tests

From 592adf501e1f025cd7f59141c5db0c22bf9bc0d3 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 15:08:01 +0100
Subject: [PATCH 17/25] one more attempt..

---
 .github/workflows/ci-dev.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index de2a5ba6..457e254e 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -38,11 +38,12 @@ jobs:
           python-version: ${{ matrix.python-version }}
           auto-activate-base: true
           miniconda-version: "latest"
+          activate-environment: shapepipe-dev
 
       - name: Install package
         shell: bash -l {0}
         run: |
-          conda activate test
+          conda activate shapepipe-dev
           ./install_shapepipe --env-dev --develop
 
       - name: Run tests

From d6c1e5f2d201eddd5c48f5c856cd05a8a68ccbc8 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 15:36:28 +0100
Subject: [PATCH 18/25] try testing docker container; deactivate conda CI

---
 .github/workflows/ci-dev.yml | 108 +++++++++++++++++------------------
 1 file changed, 54 insertions(+), 54 deletions(-)

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
index 457e254e..fd57c979 100644
--- a/.github/workflows/ci-dev.yml
+++ b/.github/workflows/ci-dev.yml
@@ -1,54 +1,54 @@
-name: CI
-
-on:
-  pull_request:
-    branches:
-     - develop
-
-jobs:
-
-  test-full:
-    name: Full Test Suite
-    runs-on: ${{ matrix.os }}
-
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: [3.11]
-
-    steps:
-      - uses: actions/checkout@v2
-
-      - name: Install Linux dependencies
-        if: runner.os == 'Linux'
-        run: sudo apt-get install libopenblas-dev
-
-      - name: Install macOS Dependencies
-        shell: bash -l {0}
-        if: runner.os == 'macOS'
-        run: |
-          brew tap sfarrens/sf
-          brew install bigmac libomp
-
-      - name: Set up conda
-        uses: conda-incubator/setup-miniconda@v3
-        with:
-          auto-update-conda: true
-          python-version: ${{ matrix.python-version }}
-          auto-activate-base: true
-          miniconda-version: "latest"
-          activate-environment: shapepipe-dev
-
-      - name: Install package
-        shell: bash -l {0}
-        run: |
-          conda activate shapepipe-dev
-          ./install_shapepipe --env-dev --develop
-
-      - name: Run tests
-        shell: bash -l {0}
-        run: |
-          conda activate shapepipe-dev
-          python setup.py test
-          shapepipe_run -c example/config.ini
+# name: CI
+
+# on:
+#   pull_request:
+#     branches:
+#      - develop
+
+# jobs:
+
+#   test-full:
+#     name: Full Test Suite
+#     runs-on: ${{ matrix.os }}
+
+#     strategy:
+#       fail-fast: false
+#       matrix:
+#         os: [ubuntu-latest, macos-latest]
+#         python-version: [3.11]
+
+#     steps:
+#       - uses: actions/checkout@v2
+
+#       - name: Install Linux dependencies
+#         if: runner.os == 'Linux'
+#         run: sudo apt-get install libopenblas-dev
+
+#       - name: Install macOS Dependencies
+#         shell: bash -l {0}
+#         if: runner.os == 'macOS'
+#         run: |
+#           brew tap sfarrens/sf
+#           brew install bigmac libomp
+
+#       - name: Set up conda
+#         uses: conda-incubator/setup-miniconda@v3
+#         with:
+#           auto-update-conda: true
+#           python-version: ${{ matrix.python-version }}
+#           auto-activate-base: true
+#           miniconda-version: "latest"
+#           activate-environment: shapepipe-dev
+
+#       - name: Install package
+#         shell: bash -l {0}
+#         run: |
+#           conda activate shapepipe-dev
+#           ./install_shapepipe --env-dev --develop
+
+#       - name: Run tests
+#         shell: bash -l {0}
+#         run: |
+#           conda activate shapepipe-dev
+#           python setup.py test
+#           shapepipe_run -c example/config.ini

From c0d005cfd75bca41162a4e8ec8be0bb276645905 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 15:36:50 +0100
Subject: [PATCH 19/25] actually deactivate conda CI

---
 .github/workflows/deploy-image.yml | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index b4bd7f0e..e3786c51 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -36,3 +36,20 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+      
+  test-shapepipe-image:
+    runs-on: ubuntu-latest
+    needs: build-and-push-image
+    steps:
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Pull Docker Image
+        run: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+
+      - name: Test Shapepipe
+        run: docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest shapepipe_run -c /app/example/config.ini

From a6e906f25a8e32dbf7cb5ac90def7cab387b91f0 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 16:03:54 +0100
Subject: [PATCH 20/25] lowercase image name

---
 .github/workflows/deploy-image.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index e3786c51..09701962 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -5,7 +5,7 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ${{ github.repository | toLower }}
 
 jobs:
   build-and-push-image:

From 9e8e4108de2f9d57fed4b809ff73b32b90faf2d3 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 16:12:05 +0100
Subject: [PATCH 21/25] hard-code lowercase image name (pull fails if there are
 capitals, and there doesn't seem to be a failsafe way to lowercase a string
 in github actions)

---
 .github/workflows/ci-dev.yml | 54 ------------------------------------
 1 file changed, 54 deletions(-)
 delete mode 100644 .github/workflows/ci-dev.yml

diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml
deleted file mode 100644
index fd57c979..00000000
--- a/.github/workflows/ci-dev.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-# name: CI
-
-# on:
-#   pull_request:
-#     branches:
-#      - develop
-
-# jobs:
-
-#   test-full:
-#     name: Full Test Suite
-#     runs-on: ${{ matrix.os }}
-
-#     strategy:
-#       fail-fast: false
-#       matrix:
-#         os: [ubuntu-latest, macos-latest]
-#         python-version: [3.11]
-
-#     steps:
-#       - uses: actions/checkout@v2
-
-#       - name: Install Linux dependencies
-#         if: runner.os == 'Linux'
-#         run: sudo apt-get install libopenblas-dev
-
-#       - name: Install macOS Dependencies
-#         shell: bash -l {0}
-#         if: runner.os == 'macOS'
-#         run: |
-#           brew tap sfarrens/sf
-#           brew install bigmac libomp
-
-#       - name: Set up conda
-#         uses: conda-incubator/setup-miniconda@v3
-#         with:
-#           auto-update-conda: true
-#           python-version: ${{ matrix.python-version }}
-#           auto-activate-base: true
-#           miniconda-version: "latest"
-#           activate-environment: shapepipe-dev
-
-#       - name: Install package
-#         shell: bash -l {0}
-#         run: |
-#           conda activate shapepipe-dev
-#           ./install_shapepipe --env-dev --develop
-
-#       - name: Run tests
-#         shell: bash -l {0}
-#         run: |
-#           conda activate shapepipe-dev
-#           python setup.py test
-#           shapepipe_run -c example/config.ini

From 97e0be6060bfabcea1de9785320ee8c370970a5a Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 16:12:49 +0100
Subject: [PATCH 22/25] actually do what i said the last commit did

---
 .github/workflows/deploy-image.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index 09701962..51a559a9 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -5,7 +5,7 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository | toLower }}
+  IMAGE_NAME: cosmostat/shapepipe
 
 jobs:
   build-and-push-image:

From 7c0c60d47e17fe96df401d03402616917cf0f47e Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 16:38:42 +0100
Subject: [PATCH 23/25] try bumping pandas version to use wheel instead of
 build from source (which currently takes 50% of the build time)

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 0e7687ad..7281eccf 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -69,7 +69,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
     numba==0.58.1 \
     numpy==1.26.4 \
     numpydoc==1.2  \
-    pandas==1.4.1  \
+    pandas==2.2  \
     PyQt5==5.15.6 \
     pyqtgraph==0.12.4 \
     pytest==8.3.3 \

From 39b6897cd08802016dcb209d594428eafdbc900d Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 16:53:59 +0100
Subject: [PATCH 24/25] try building and testing in the same job

---
 .github/workflows/deploy-image.yml | 33 ++++++++++++++----------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml
index 51a559a9..2478c24d 100644
--- a/.github/workflows/deploy-image.yml
+++ b/.github/workflows/deploy-image.yml
@@ -5,7 +5,8 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: cosmostat/shapepipe
+  IMAGE_NAME: ${{ github.repository }}
+  BRANCH: ${{ github.ref }}
 
 jobs:
   build-and-push-image:
@@ -24,32 +25,28 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Build and push Docker image
+      - name: Build and export to Docker
         uses: docker/build-push-action@v6
         with:
-          push: true
+          load: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Test
+        run: docker run --rm ${{ steps.meta.outputs.tags }} shapepipe_run -c /app/example/config.ini
       
-  test-shapepipe-image:
-    runs-on: ubuntu-latest
-    needs: build-and-push-image
-    steps:
-      - name: Log in to the Container registry
-        uses: docker/login-action@v3
+      - name: Push
+        uses: docker/build-push-action@v6
         with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Pull Docker Image
-        run: docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
-
-      - name: Test Shapepipe
-        run: docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest shapepipe_run -c /app/example/config.ini
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
\ No newline at end of file

From f20d1157e3006a0de899b0d24bd7f3c8824f8905 Mon Sep 17 00:00:00 2001
From: Cail Daley <cailmdaley@gmail.com>
Date: Fri, 28 Feb 2025 17:19:00 +0100
Subject: [PATCH 25/25] bump treecorr version

---
 Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Dockerfile b/Dockerfile
index 7281eccf..5d8dce70 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -83,7 +83,7 @@ RUN pip install --no-cache-dir --upgrade pip && \
     sqlitedict==2.0.0 \
     termcolor==1.1.0 \
     tqdm==4.63.0 \
-    treecorr==4.2.6 \
+    treecorr==5.1.1 \
     vos==3.6.1.1 \
     git+https://github.com/aguinot/ngmix@stable_version \
     git+https://github.com/tobias-liaudat/Stile@v0.1 \