diff --git a/.github/workflows/ci-dev.yml b/.github/workflows/ci-dev.yml deleted file mode 100644 index 9e666f77..00000000 --- a/.github/workflows/ci-dev.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: CI - -on: - pull_request: - branches: - - develop - -jobs: - - test-full: - name: Full Test Suite - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - python-version: [3.8] - - steps: - - uses: actions/checkout@v2 - - - name: Install Linux dependencies - if: runner.os == 'Linux' - run: sudo apt-get install libopenblas-dev - - - name: Install macOS Dependencies - shell: bash -l {0} - if: runner.os == 'macOS' - run: | - brew tap sfarrens/sf - brew install bigmac libomp - - - name: Set up conda - uses: conda-incubator/setup-miniconda@v2 - with: - auto-update-conda: true - python-version: ${{ matrix.python-version }} - auto-activate-base: true - - - name: Install package - shell: bash -l {0} - run: | - ./install_shapepipe --env-dev --develop - - - name: Run tests - shell: bash -l {0} - run: | - conda activate shapepipe-dev - python setup.py test - shapepipe_run -c example/config.ini diff --git a/.github/workflows/deploy-image.yml b/.github/workflows/deploy-image.yml index 09b7a790..2478c24d 100644 --- a/.github/workflows/deploy-image.yml +++ b/.github/workflows/deploy-image.yml @@ -2,40 +2,51 @@ name: Create and publish a Docker image on: push: - branches: ['master', 'develop'] env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} + BRANCH: ${{ github.ref }} jobs: build-and-push-image: - runs-on: ubuntu-latest + runs-on: + - ubuntu-latest + # - macos-latest permissions: contents: read packages: write steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Log in to the Container registry - uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + uses: docker/login-action@v3 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Extract metadata (tags, labels) for Docker id: meta - uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Build and push Docker image - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + - name: Build and export to Docker + uses: docker/build-push-action@v6 with: - context: . - push: true + load: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + + - name: Test + run: docker run --rm ${{ steps.meta.outputs.tags }} shapepipe_run -c /app/example/config.ini + + - name: Push + uses: docker/build-push-action@v6 + with: + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 7d2785ab..5d8dce70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,57 +1,103 @@ -FROM continuumio/miniconda3 +FROM python:3.11-bookworm -LABEL Description="ShapePipe Docker Image" -ENV SHELL /bin/bash +LABEL Description="Conda-Free ShapePipe Docker Image" +ENV SHELL=/bin/bash -ARG CC=gcc-9 -ARG CXX=g++-9 +# Install system dependencies +RUN apt-get update -y --quiet --fix-missing && \ + apt-get dist-upgrade -y --quiet --fix-missing && \ + apt-cache policy autconf && \ + apt-get install -y --quiet \ + apt-utils \ + autoconf \ + automake \ + build-essential \ + cmake \ + curl \ + ffmpeg \ + g++ \ + gcc \ + gfortran \ + git-lfs \ + libatlas-base-dev \ + libblas-dev \ + liblapack-dev \ + libcfitsio-dev \ + libfftw3-bin \ + libfftw3-dev \ + libgl1-mesa-glx \ + libtool \ + libtool-bin \ + libtool-doc \ + locales \ + locate \ + make \ + openmpi-bin \ + libopenmpi-dev \ + pkg-config \ + protobuf-compiler \ + psfex=3.21.1-1 \ + source-extractor=2.25.0+ds-3 \ + weightwatcher=1.12+dfsg-3 \ + vim \ + xterm && \ + apt-get clean -y && \ + apt-get autoremove --purge --quiet -y && \ + rm -rf /var/lib/apt/lists/* /var/tmp/* -# gcc < 10 is required to compile ww -ENV CC=gcc-9 -ENV CXX=g++-9 +# Install CDS client by hand +RUN cd /tmp && \ + curl -O http://cdsarc.u-strasbg.fr/ftp/pub/sw/cdsclient.tar.gz && \ + tar xvfz cdsclient.tar.gz \ + && cd cdsclient-* \ + && ./configure && make && make install \ + && rm -rf /tmp/* -RUN apt-get update --allow-releaseinfo-change && \ - apt-get update && \ - apt-get upgrade -y && \ - apt-get install apt-utils -y && \ - apt-get install make -y && \ - apt-get install automake -y && \ - apt-get install autoconf -y && \ - apt-get install gcc-9 g++-9 -y && \ - apt-get install gfortran -y && \ - apt-get install locales -y && \ - apt-get install libgl1-mesa-glx -y && \ - apt-get install xterm -y && \ - apt-get install cmake protobuf-compiler -y && \ - apt-get install libtool libtool-bin libtool-doc -y && \ - apt-get install libfftw3-bin libfftw3-dev -y && \ - apt-get install libatlas-base-dev liblapack-dev libblas-dev -y && \ - apt-get install vim -y && \ - apt-get install locate -y && \ - apt-get install curl -y && \ - apt-get install acl -y && \ - apt-get install sssd -y && \ - apt-get clean +# Install python dependencies +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir \ + astropy==5.2 \ + cs_util==0.1.0 \ + galsim==2.2.6 \ + ipython==8.18.1 \ + joblib==1.1.0 \ + jupyterlab==4.3.1 \ + matplotlib==3.8.4 \ + mccd==1.2.4 \ + modopt==1.6.1 \ + mpi4py==4.0.3 \ + numba==0.58.1 \ + numpy==1.26.4 \ + numpydoc==1.2 \ + pandas==2.2 \ + PyQt5==5.15.6 \ + pyqtgraph==0.12.4 \ + pytest==8.3.3 \ + pytest-cov==5.0.0 \ + pytest-pycodestyle==2.4.1 \ + pytest-pydocstyle==2.4.0 \ + reproject==0.14.1 \ + sf_tools==2.0.4 \ + sip_tpv==1.1 \ + skaha==1.4.3 \ + sqlitedict==2.0.0 \ + termcolor==1.1.0 \ + tqdm==4.63.0 \ + treecorr==5.1.1 \ + vos==3.6.1.1 \ + git+https://github.com/aguinot/ngmix@stable_version \ + git+https://github.com/tobias-liaudat/Stile@v0.1 \ + git+https://github.com/CEA-COSMIC/pysap@develop -ADD nsswitch.conf /etc/ -RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && \ - locale-gen -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 +WORKDIR /app +COPY . /app/. -SHELL ["/bin/bash", "--login", "-c"] - -COPY ./environment.yml ./ -COPY install_shapepipe README.rst setup.py setup.cfg ./ -RUN touch ./README.md - -RUN conda update -n base -c defaults conda -c defaults -RUN conda env create --file environment.yml - -COPY shapepipe ./shapepipe -COPY scripts ./scripts - -RUN source activate shapepipe -#RUN pip install jupyter +# Install shapepipe and symlink scripts +RUN pip install --no-cache-dir -e . && \ + for ext in .py .sh .bash; do \ + for script in /app/scripts/*/*$ext; do \ + link_name=`basename $script $ext`; \ + ln -s $script /usr/local/bin/$link_name; \ + done; \ + done diff --git a/docs/source/installation.md b/docs/source/installation.md index 63f030c1..831f12cc 100644 --- a/docs/source/installation.md +++ b/docs/source/installation.md @@ -6,7 +6,33 @@ are expected to install the full ShapePipe environment on the system(s) where data should be processed. ``` -## Standard Installation +## Container Installation (Recommended) + +The easiest way to install ShapePipe is via a container. Docker images are automatically built and pushed to the [Github Container Registry (GHCR)](ghcr.io/cosmostat/shapepipe) for each release. This images can be installed and run on most systems (including clusters) with just a few lines of code. + +We recommend running the image with **Apptainer** (formerly Singularity) which is installed on most HPC clusters. To simply run the image, use the following command: + +```bash +# build writeable "sandbox" container in the current directory +# ./shapepipe will be a directory that functions like a vm +apptainer build --sandbox shapepipe docker://ghcr.io/cosmostat/shapepipe:develop + +# open a shell in the container and run the example +apptainer shell --writable shapepipe +cd /app && shapepipe_run -c /app/example/config.ini +``` + +You can also run the image with **Docker**: + +```bash +docker run --rm -it ghcr.io/cosmostat/shapepipe:develop shapepipe_run -c /app/example/config.ini +``` + +```{attention} +We do not currently build images for Apple Silicon/amr64; however the amd64 images should work on these systems, albeit with reduced performance. +``` + +## Conda Installation (Deprecated) ```{tip} :class: margin diff --git a/pyproject.toml b/pyproject.toml index ab87fac6..bd83613c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "shapepipe" -version = "1.0.2" +version = "1.1.0" description = "Galaxy shape measurement pipeline." readme = "README.rst" authors = [ diff --git a/src/shapepipe/run.py b/src/shapepipe/run.py index 19880748..82e773b1 100644 --- a/src/shapepipe/run.py +++ b/src/shapepipe/run.py @@ -14,7 +14,7 @@ from modopt.interface.errors import catch_error from modopt.interface.log import close_log, set_up_log -from shapepipe.info import line, shapepipe_logo +from shapepipe.info import line, shapepipe_logo, __installs__ from shapepipe.pipeline.args import create_arg_parser from shapepipe.pipeline.config import create_config_parser from shapepipe.pipeline.dependency_handler import DependencyHandler