From 602757e6b441f4e6a7db57b7ccd78595f70da2df Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Mon, 14 Feb 2022 11:56:27 -0500 Subject: [PATCH 1/3] upstream testing --- .github/workflows/upstream-dev-ci.yaml | 174 +++++++++++++++++++++++++ ci/install-upstream-wheels.sh | 30 +++++ 2 files changed, 204 insertions(+) create mode 100644 .github/workflows/upstream-dev-ci.yaml create mode 100644 ci/install-upstream-wheels.sh diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml new file mode 100644 index 0000000..8824ae0 --- /dev/null +++ b/.github/workflows/upstream-dev-ci.yaml @@ -0,0 +1,174 @@ +name: CI Upstream +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: "0 0 * * *" # Daily “At 00:00” UTC + workflow_dispatch: # allows you to trigger the workflow run manually + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + detect-ci-trigger: + name: detect upstream-dev ci trigger + runs-on: ubuntu-latest + if: | + github.repository == 'intake/intake-xarray' + && (github.event_name == 'push' || github.event_name == 'pull_request') + outputs: + triggered: ${{ steps.detect-trigger.outputs.trigger-found }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 2 + - uses: xarray-contrib/ci-trigger@v1.1 + id: detect-trigger + with: + keyword: "[test-upstream]" + + upstream-dev: + name: upstream-dev + runs-on: ubuntu-latest + needs: detect-ci-trigger + if: | + always() + && ( + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') + || needs.detect-ci-trigger.outputs.triggered == 'true' + ) + defaults: + run: + shell: bash -l {0} + strategy: + fail-fast: false + matrix: + python-version: ["3.10"] + outputs: + artifacts_availability: ${{ steps.status.outputs.ARTIFACTS_AVAILABLE }} + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 # Fetch all history for all branches and tags. + - uses: conda-incubator/setup-miniconda@v2 + with: + channels: conda-forge + channel-priority: strict + mamba-version: "*" + activate-environment: xarray-tests + auto-update-conda: false + python-version: ${{ matrix.python-version }} + - name: Set up conda environment + run: | + mamba env update -f ci/environment-upstream.yml + bash ci/install-upstream-wheels.sh + - name: Install intake-xarray + run: | + python -m pip install --no-deps -e . + - name: import intake-xarray + run: | + python -c 'import intake_xarray' + - name: Run Tests + if: success() + id: status + run: | + set -euo pipefail + python -m pytest --timeout=60 -rf | tee output-${{ matrix.python-version }}-log || ( + echo '::set-output name=ARTIFACTS_AVAILABLE::true' && false + ) + - name: Upload artifacts + if: | + failure() + && steps.status.outcome == 'failure' + && github.event_name == 'schedule' + && github.repository == 'intake/intake-xarray' + uses: actions/upload-artifact@v2 + with: + name: output-${{ matrix.python-version }}-log + path: output-${{ matrix.python-version }}-log + retention-days: 5 + + report: + name: report + needs: upstream-dev + if: | + failure() + && github.event_name == 'schedule' + && needs.upstream-dev.outputs.artifacts_availability == 'true' + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + with: + python-version: "3.x" + - uses: actions/download-artifact@v2 + with: + path: /tmp/workspace/logs + - name: Move all log files into a single directory + run: | + rsync -a /tmp/workspace/logs/output-*/ ./logs + ls -R ./logs + - name: Parse logs + run: | + shopt -s globstar + python .github/workflows/parse_logs.py logs/**/*-log + - name: Report failures + uses: actions/github-script@v6 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const pytest_logs = fs.readFileSync('pytest-logs.txt', 'utf8'); + const title = "⚠️ Nightly upstream-dev CI failed ⚠️" + const workflow_url = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}` + const issue_body = `[Workflow Run URL](${workflow_url})\n${pytest_logs}` + + // Run GraphQL query against GitHub API to find the most recent open issue used for reporting failures + const query = `query($owner:String!, $name:String!, $creator:String!, $label:String!){ + repository(owner: $owner, name: $name) { + issues(first: 1, states: OPEN, filterBy: {createdBy: $creator, labels: [$label]}, orderBy: {field: CREATED_AT, direction: DESC}) { + edges { + node { + body + id + number + } + } + } + } + }`; + + const variables = { + owner: context.repo.owner, + name: context.repo.repo, + label: 'CI', + creator: "github-actions[bot]" + } + const result = await github.graphql(query, variables) + + // If no issue is open, create a new issue, + // else update the body of the existing issue. + if (result.repository.issues.edges.length === 0) { + github.rest.issues.create({ + owner: variables.owner, + repo: variables.name, + body: issue_body, + title: title, + labels: [variables.label] + }) + } else { + github.rest.issues.update({ + owner: variables.owner, + repo: variables.name, + issue_number: result.repository.issues.edges[0].node.number, + body: issue_body + }) + } \ No newline at end of file diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh new file mode 100644 index 0000000..a5828be --- /dev/null +++ b/ci/install-upstream-wheels.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +conda uninstall -y --force \ + aiohttp \ + flask \ + h5netcdf \ + netcdf4 \ + pydap \ + rasterio \ + tornado \ + scikit-image \ + zarr \ + s3fs \ + moto +# to limit the runtime of Upstream CI +python -m pip install pytest-timeout +python -m pip install \ + --no-deps \ + --upgrade \ + git+https://github.com/aio-libs/aiohttp \ + git+https://github.com/boto/boto3 \ + git+https://github.com/pallets/flask \ + git+https://github.com/h5netcdf/h5netcdf \ + git+https://github.com/Unidata/netcdf4-python \ + git+https://github.com/pydap/pydap \ + git+https://github.com/rasterio/rasterio \ + git+https://github.com/scikit-image/scikit-image \ + git+https://github.com/zarr-developers/zarr-python \ + git+https://github.com/fsspec/s3fs \ + git+https://github.com/spulec/moto From c1c3668cf8d56712fa72d214ed3a01dbc62cf883 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Mon, 14 Feb 2022 13:38:00 -0500 Subject: [PATCH 2/3] other packages --- ci/install-upstream-wheels.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ci/install-upstream-wheels.sh b/ci/install-upstream-wheels.sh index a5828be..a427fc6 100644 --- a/ci/install-upstream-wheels.sh +++ b/ci/install-upstream-wheels.sh @@ -27,4 +27,7 @@ python -m pip install \ git+https://github.com/scikit-image/scikit-image \ git+https://github.com/zarr-developers/zarr-python \ git+https://github.com/fsspec/s3fs \ - git+https://github.com/spulec/moto + git+https://github.com/spulec/moto \ + git+https://github.com/fsspec/filesystem_spec.git \ + git+https://github.com/pydata/xarray.git \ + git+https://github.com/danvk/RangeHTTPServer.git From 369723271ee3c99682af659034e2b8f0a73db1c9 Mon Sep 17 00:00:00 2001 From: Ray Bell Date: Mon, 14 Feb 2022 13:40:05 -0500 Subject: [PATCH 3/3] master --- .github/workflows/upstream-dev-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upstream-dev-ci.yaml b/.github/workflows/upstream-dev-ci.yaml index 8824ae0..e623697 100644 --- a/.github/workflows/upstream-dev-ci.yaml +++ b/.github/workflows/upstream-dev-ci.yaml @@ -2,10 +2,10 @@ name: CI Upstream on: push: branches: - - main + - master pull_request: branches: - - main + - master schedule: - cron: "0 0 * * *" # Daily “At 00:00” UTC workflow_dispatch: # allows you to trigger the workflow run manually