Automatic Docker Builds

.github/workflows/deploy-image.yml

@@ -2,40 +2,51 @@ name: Create and publish a Docker image
 
 on:
   push:
-    branches: ['master', 'develop']
 
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  BRANCH: ${{ github.ref }}
 
 jobs:
   build-and-push-image:
-    runs-on: ubuntu-latest
+    runs-on: 
+      - ubuntu-latest
+      # - macos-latest
     permissions:
       contents: read
       packages: write
 
     steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-
       - name: Log in to the Container registry
-        uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9
+        uses: docker/login-action@v3
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
 
-      - name: Build and push Docker image
-        uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc
+      - name: Build and export to Docker
+        uses: docker/build-push-action@v6
         with:
-          context: .
-          push: true
+          load: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+      - name: Test
+        run: docker run --rm ${{ steps.meta.outputs.tags }} shapepipe_run -c /app/example/config.ini
+
+      - name: Push
+        uses: docker/build-push-action@v6
+        with:
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

Dockerfile

@@ -1,57 +1,103 @@
-FROM continuumio/miniconda3
+FROM python:3.11-bookworm
 
-LABEL Description="ShapePipe Docker Image"
-ENV SHELL /bin/bash
+LABEL Description="Conda-Free ShapePipe Docker Image"
+ENV SHELL=/bin/bash
 
-ARG CC=gcc-9
-ARG CXX=g++-9
+# Install system dependencies
+RUN apt-get update -y --quiet --fix-missing && \
+    apt-get dist-upgrade -y --quiet --fix-missing && \
+    apt-cache policy autconf && \ 
+    apt-get install -y --quiet \
+    apt-utils \
+    autoconf \
+    automake \
+    build-essential \
+    cmake \
+    curl \
+    ffmpeg \
+    g++ \
+    gcc  \
+    gfortran \
+    git-lfs \
+    libatlas-base-dev \
+    libblas-dev \
+    liblapack-dev \
+    libcfitsio-dev \
+    libfftw3-bin \
+    libfftw3-dev \
+    libgl1-mesa-glx \
+    libtool \
+    libtool-bin \
+    libtool-doc \
+    locales \
+    locate \
+    make \
+    openmpi-bin \
+    libopenmpi-dev \
+    pkg-config \
+    protobuf-compiler \
+    psfex=3.21.1-1 \
+    source-extractor=2.25.0+ds-3 \
+    weightwatcher=1.12+dfsg-3 \
+    vim \
+    xterm && \
+    apt-get clean -y && \
+    apt-get autoremove --purge --quiet -y && \
+    rm -rf /var/lib/apt/lists/* /var/tmp/*
 
-# gcc < 10 is required to compile ww
-ENV CC=gcc-9
-ENV CXX=g++-9
+# Install CDS client by hand
+RUN cd /tmp && \
+    curl -O http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz && \
+    tar xvfz cdsclient.tar.gz \ 
+    && cd cdsclient-* \
+    && ./configure && make && make install \
+    && rm -rf /tmp/*
 
-RUN apt-get update --allow-releaseinfo-change && \
-    apt-get update && \
-    apt-get upgrade -y && \
-    apt-get install apt-utils -y && \
-    apt-get install make -y && \
-    apt-get install automake -y && \
-    apt-get install autoconf -y && \
-    apt-get install gcc-9 g++-9 -y && \
-    apt-get install gfortran -y && \
-    apt-get install locales -y && \
-    apt-get install libgl1-mesa-glx -y && \
-    apt-get install xterm -y && \
-    apt-get install cmake protobuf-compiler -y && \
-    apt-get install libtool libtool-bin libtool-doc -y && \
-    apt-get install libfftw3-bin libfftw3-dev -y && \
-    apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \
-    apt-get install vim -y && \
-    apt-get install locate -y && \
-    apt-get install curl -y && \
-    apt-get install acl -y && \
-    apt-get install sssd -y && \
-    apt-get clean
+# Install python dependencies
+RUN pip install --no-cache-dir --upgrade pip && \
+    pip install --no-cache-dir \
+    astropy==5.2 \
+    cs_util==0.1.0 \
+    galsim==2.2.6 \
+    ipython==8.18.1 \
+    joblib==1.1.0 \
+    jupyterlab==4.3.1 \
+    matplotlib==3.8.4 \
+    mccd==1.2.4 \
+    modopt==1.6.1 \
+    mpi4py==4.0.3 \
+    numba==0.58.1 \
+    numpy==1.26.4 \
+    numpydoc==1.2  \
+    pandas==2.2  \
+    PyQt5==5.15.6 \
+    pyqtgraph==0.12.4 \
+    pytest==8.3.3 \
+    pytest-cov==5.0.0 \
+    pytest-pycodestyle==2.4.1 \
+    pytest-pydocstyle==2.4.0 \
+    reproject==0.14.1 \
+    sf_tools==2.0.4 \
+    sip_tpv==1.1 \
+    skaha==1.4.3 \
+    sqlitedict==2.0.0 \
+    termcolor==1.1.0 \
+    tqdm==4.63.0 \
+    treecorr==5.1.1 \
+    vos==3.6.1.1 \
+    git+https://github.com/aguinot/ngmix@stable_version \
+    git+https://github.com/tobias-liaudat/Stile@v0.1 \ 
+    git+https://github.com/CEA-COSMIC/pysap@develop
 
-ADD nsswitch.conf /etc/
 
-RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \
-    locale-gen
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+WORKDIR /app
+COPY . /app/.
 
-SHELL ["/bin/bash", "--login", "-c"]
-
-COPY ./environment.yml ./
-COPY install_shapepipe README.rst setup.py setup.cfg ./
-RUN touch ./README.md
-
-RUN conda update -n base -c defaults conda -c defaults
-RUN conda env create --file environment.yml
-
-COPY shapepipe ./shapepipe
-COPY scripts ./scripts
-
-RUN source activate shapepipe
-#RUN pip install jupyter
+# Install shapepipe and symlink scripts
+RUN pip install --no-cache-dir -e . && \ 
+    for ext in .py .sh .bash; do \
+    for script in /app/scripts/*/*$ext; do \
+    link_name=`basename $script $ext`; \
+    ln -s $script /usr/local/bin/$link_name; \
+    done; \
+    done

docs/source/installation.md

@@ -6,7 +6,33 @@ are expected to install the full ShapePipe environment on the system(s) where
 data should be processed.
 ```
 
-## Standard Installation
+## Container Installation (Recommended)
+
+The easiest way to install ShapePipe is via a container. Docker images are automatically built and pushed to the [Github Container Registry (GHCR)](ghcr.io/cosmostat/shapepipe) for each release. This images can be installed and run on most systems (including clusters) with just a few lines of code.
+
+We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command:
+
+```bash
+# build writeable "sandbox" container in the current directory
+# ./shapepipe will be a directory that functions like a vm
+apptainer build --sandbox shapepipe docker://ghcr.io/cosmostat/shapepipe:develop
+
+# open a shell in the container and run the example
+apptainer shell --writable shapepipe
+cd /app && shapepipe_run -c /app/example/config.ini
+```
+
+You can also run the image with **Docker**:
+
+```bash
+docker run --rm -it ghcr.io/cosmostat/shapepipe:develop shapepipe_run -c /app/example/config.ini  
+```
+
+```{attention}
+We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance.
+```
+
+## Conda Installation (Deprecated)
 
 ```{tip}
 :class: margin

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "shapepipe"
-version = "1.0.2"
+version = "1.1.0"
 description = "Galaxy shape measurement pipeline."
 readme = "README.rst"
 authors = [

src/shapepipe/run.py

@@ -14,7 +14,7 @@
 from modopt.interface.errors import catch_error
 from modopt.interface.log import close_log, set_up_log
 
-from shapepipe.info import line, shapepipe_logo
+from shapepipe.info import line, shapepipe_logo, __installs__
 from shapepipe.pipeline.args import create_arg_parser
 from shapepipe.pipeline.config import create_config_parser
 from shapepipe.pipeline.dependency_handler import DependencyHandler