diff --git a/.github/workflows/_build.yaml b/.github/workflows/_build.yaml index fe52fc30..7fb7518a 100644 --- a/.github/workflows/_build.yaml +++ b/.github/workflows/_build.yaml @@ -55,18 +55,18 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python }} @@ -134,7 +134,7 @@ jobs: # Currently reusable workflows do not support setting strategy property from the caller workflow. - name: Upload the package artifact for debugging and release if: matrix.os == env.ARTIFACT_OS && matrix.python == env.ARTIFACT_PYTHON - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: artifact-${{ matrix.os }}-python-${{ matrix.python }} path: dist diff --git a/.github/workflows/_generate-rebase.yaml b/.github/workflows/_generate-rebase.yaml index b35323d3..fedbe4cf 100644 --- a/.github/workflows/_generate-rebase.yaml +++ b/.github/workflows/_generate-rebase.yaml @@ -34,12 +34,12 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 token: ${{ secrets.REPO_ACCESS_TOKEN }} diff --git a/.github/workflows/_wiki-documentation.yaml b/.github/workflows/_wiki-documentation.yaml index 87e399dd..d8332739 100644 --- a/.github/workflows/_wiki-documentation.yaml +++ b/.github/workflows/_wiki-documentation.yaml @@ -41,7 +41,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true @@ -49,7 +49,7 @@ jobs: # Check out the repository's Wiki repo into the wiki/ folder. The token is required # only for private repositories. - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: token: ${{ secrets.REPO_ACCESS_TOKEN }} repository: ${{ format('{0}.wiki', github.repository) }} @@ -57,7 +57,7 @@ jobs: # Download the build artifacts attached to this workflow run. - name: Download artifact - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: ${{ inputs.artifact-name }} path: dist diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index 826ef9e1..e6a05219 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -33,16 +33,16 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true - name: Checkout repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: ${{ matrix.python }} @@ -54,7 +54,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@192325c86100d080feab897ff886c34abd4c83a3 # v3.30.3 + uses: github/codeql-action/init@fdbfb4d2750291e159f0156def62b853c2798ca2 # v4.31.5 with: languages: ${{ matrix.language }} config-file: .github/codeql/codeql-config.yaml @@ -67,4 +67,4 @@ jobs: # queries: ./path/to/local/query, your-org/your-repo/queries@main - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@192325c86100d080feab897ff886c34abd4c83a3 # v3.30.3 + uses: github/codeql-action/analyze@fdbfb4d2750291e159f0156def62b853c2798ca2 # v4.31.5 diff --git a/.github/workflows/pr-conventional-commits.yaml b/.github/workflows/pr-conventional-commits.yaml index f43cfe0a..482607df 100644 --- a/.github/workflows/pr-conventional-commits.yaml +++ b/.github/workflows/pr-conventional-commits.yaml @@ -22,12 +22,12 @@ jobs: steps: - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.13' @@ -36,7 +36,7 @@ jobs: - name: Set up Commitizen run: | pip install --upgrade pip wheel - pip install 'commitizen ==4.9.1' + pip install 'commitizen ==4.10.0' # Run Commitizen to check the title of the PR which triggered this workflow, and check # all commit messages of the PR's branch. If any of the checks fails then this job fails. diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d2e8295d..fbe15da0 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -30,26 +30,26 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 token: ${{ secrets.REPO_ACCESS_TOKEN }} - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.13' - name: Set up Commitizen run: | pip install --upgrade pip wheel - pip install 'commitizen ==4.9.1' + pip install 'commitizen ==4.10.0' - name: Set up user run: | @@ -98,18 +98,18 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 - name: Download artifact - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: artifact-ubuntu-latest-python-3.13 path: dist @@ -126,14 +126,14 @@ jobs: # Create the Release Notes using commitizen. - name: Set up Python - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 + uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 with: python-version: '3.13' - name: Set up Commitizen run: | pip install --upgrade pip wheel - pip install 'commitizen ==4.9.1' + pip install 'commitizen ==4.10.0' - name: Create Release Notes run: cz changelog --dry-run "$(cz version --project)" > RELEASE_NOTES.md @@ -199,18 +199,18 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: fetch-depth: 0 - name: Download provenance - uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0 + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 with: name: ${{ needs.provenance.outputs.provenance-name }} diff --git a/.github/workflows/scorecards-analysis.yaml b/.github/workflows/scorecards-analysis.yaml index 13f9e50a..4f53c322 100644 --- a/.github/workflows/scorecards-analysis.yaml +++ b/.github/workflows/scorecards-analysis.yaml @@ -26,18 +26,18 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@f4a75cfd619ee5ce8d5b864b0d183aff3c69b55a # v2.13.1 + uses: step-security/harden-runner@95d9a5deda9de15063e7595e9719c11c38c90ae2 # v2.13.2 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs disable-sudo: true - name: Check out repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: persist-credentials: false - name: Run analysis - uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2 + uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3 with: results_file: results.sarif results_format: sarif @@ -52,13 +52,13 @@ jobs: # Upload the results as artifacts (optional). - name: Upload artifact - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # v5.0.0 with: name: SARIF file path: results.sarif # Upload the results to GitHub's code scanning dashboard. - name: Upload to code-scanning - uses: github/codeql-action/upload-sarif@192325c86100d080feab897ff886c34abd4c83a3 # v3.30.3 + uses: github/codeql-action/upload-sarif@fdbfb4d2750291e159f0156def62b853c2798ca2 # v4.31.5 with: sarif_file: results.sarif diff --git a/.github/workflows/sync-with-upstream.yaml b/.github/workflows/sync-with-upstream.yaml index cfb5f152..0f0599f3 100644 --- a/.github/workflows/sync-with-upstream.yaml +++ b/.github/workflows/sync-with-upstream.yaml @@ -21,7 +21,7 @@ jobs: steps: - name: Check out template repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: # If you decide to change the upstream template repository to a private one, uncomment # the following argument to pass the required token to be able to check it out. @@ -31,7 +31,7 @@ jobs: path: template - name: Check out current repository - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: token: ${{ secrets.REPO_ACCESS_TOKEN }} fetch-depth: 0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8bb0182d..04527550 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,7 @@ repos: # Commitizen enforces semantic and conventional commit messages. - repo: https://github.com/commitizen-tools/commitizen - rev: v4.9.1 + rev: v4.10.0 hooks: - id: commitizen name: Check conventional commit message @@ -23,7 +23,7 @@ repos: # Sort imports. - repo: https://github.com/pycqa/isort - rev: 6.0.1 + rev: 7.0.0 hooks: - id: isort name: Sort import statements @@ -32,7 +32,7 @@ repos: # Add Black code formatters. - repo: https://github.com/ambv/black - rev: 25.9.0 + rev: 25.11.0 hooks: - id: black name: Format code @@ -43,11 +43,11 @@ repos: - id: blacken-docs name: Format code in docstrings args: [--line-length, '120'] - additional_dependencies: [black==25.9.0] + additional_dependencies: [black==25.11.0] # Upgrade and rewrite Python idioms. - repo: https://github.com/asottile/pyupgrade - rev: v3.20.0 + rev: v3.21.2 hooks: - id: pyupgrade name: Upgrade code idioms @@ -63,7 +63,7 @@ repos: name: Check flake8 issues files: ^src/package/|^tests/ types: [text, python] - additional_dependencies: [flake8-bugbear==24.12.12, flake8-builtins==3.0.0, flake8-comprehensions==3.17.0, flake8-docstrings==1.7.0, flake8-logging==1.8.0, flake8-mutable==1.2.0, flake8-noqa==1.4.0, flake8-print==5.0.0, flake8-pyi==25.5.0, flake8-pytest-style==2.1.0, flake8-rst-docstrings==0.3.1, pep8-naming==0.15.1] + additional_dependencies: [flake8-bugbear==25.11.29, flake8-builtins==3.1.0, flake8-comprehensions==3.17.0, flake8-docstrings==1.7.0, flake8-logging==1.8.0, flake8-mutable==1.2.0, flake8-noqa==1.4.0, flake8-print==5.0.0, flake8-pyi==25.5.0, flake8-pytest-style==2.2.0, flake8-rst-docstrings==0.4.0, pep8-naming==0.15.1] args: [--config, .flake8] # Run Pylint from the local repo to make sure venv packages @@ -87,11 +87,11 @@ repos: language: python files: ^src/package/|^tests/ types: [text, python] - args: [--config-file, pyproject.toml] + args: [--explicit-package-bases, --config-file, pyproject.toml] # Check for potential security issues. - repo: https://github.com/PyCQA/bandit - rev: 1.8.6 + rev: 1.9.2 hooks: - id: bandit name: Check for security issues @@ -156,18 +156,18 @@ repos: # Check GitHub Actions workflow files. - repo: https://github.com/Mateusz-Grzelinski/actionlint-py - rev: v1.7.7.24 + rev: v1.7.9.24 hooks: - id: actionlint -# On push to the remote, run the unit tests. Note that the `COVERAGE_CORE` variable is -# required for Python 3.12+ to make sure Coverage uses the new Python monitoring module. +# On push to the remote, run all tests. Note that the `COVERAGE_CORE` variable is required +# for Python 3.12+ to make sure Coverage uses the new Python monitoring module. # See also: https://blog.trailofbits.com/2025/05/01/making-pypis-test-suite-81-faster/#optimizing-coverage-with-python-312s-sysmonitoring - repo: local hooks: - id: pytest name: Run unit tests - entry: env COVERAGE_CORE=sysmon pytest -c pyproject.toml --cov-config pyproject.toml src/package/ tests/ docs/ + entry: env COVERAGE_CORE=sysmon pytest --config-file pyproject.toml --cov-config pyproject.toml -m 'not integration and not performance' src/package/ tests/ docs/ language: python verbose: true always_run: true diff --git a/Makefile b/Makefile index b5a71800..8a59187c 100644 --- a/Makefile +++ b/Makefile @@ -165,21 +165,28 @@ check-actionlint: check: pre-commit run --all-files -# Run all unit tests. The --files option avoids stashing but passes files; however, -# the hook setup itself does not pass files to pytest (see .pre-commit-config.yaml). -.PHONY: test -test: - pre-commit run pytest --hook-stage push --files tests/ +# Run different kinds of tests: unit tests, integration tests, performance tests. +# Note that the default goal 'test' runs the unit tests only, mainly for convenience +# and compatibility with existing scripts. +.PHONY: test test-all test-unit test-integration test-performance +test: test-unit +test-unit: + COVERAGE_CORE=sysmon python -m pytest --config-file pyproject.toml --cov-config pyproject.toml -m 'not integration and not performance' src/package/ tests/ docs/ +test-integration: + python -m pytest --config-file pyproject.toml --no-cov -m integration tests/ +test-performance: + python -m pytest --config-file pyproject.toml --no-cov -m performance tests/ +test-all: test-unit test-integration test-performance # Build a source distribution package and a binary wheel distribution artifact. # When building these artifacts, we need the environment variable SOURCE_DATE_EPOCH # set to the build date/epoch. For more details, see: https://flit.pypa.io/en/latest/reproducible.html .PHONY: dist dist: dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: check test dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt - SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format wheel -dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: check test dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt - SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) flit build --setup-py --format sdist +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-py3-none-any.whl: check test-all dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt + SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) python -m flit build --setup-py --format wheel +dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION).tar.gz: check test-all dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-build-epoch.txt + SOURCE_DATE_EPOCH=$(SOURCE_DATE_EPOCH) python -m flit build --setup-py --format sdist dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip: docs-html python -m zipfile -c dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-html.zip docs/_build/html/ dist/$(PACKAGE_NAME)-$(PACKAGE_VERSION)-docs-md.zip: docs-md diff --git a/README.md b/README.md index 44a7445f..14595542 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ This repository is intended to be a base template, a cookiecutter for a new Pyth [Features](#features)  [Typing](#typing)  [Quality assurance](#quality-assurance) - [Unit testing](#unit-testing) + [Extensive testing](#extensive-testing)  [Documentation](#documentation)  [Versioning and publishing](#versioning-and-publishing)  [Dependency analysis](#dependency-analysis) @@ -38,9 +38,9 @@ The package requires a minimum of [Python 3.10](https://www.python.org/downloads A number of git hooks are invoked before and after a commit, and before push. These hooks are all managed by the [pre-commit](https://pre-commit.com/) tool and enforce a number of [software quality assurance](https://en.wikipedia.org/wiki/Software_quality_assurance) measures (see [below](#git-hooks)). -### Unit testing +### Extensive testing -Comprehensive unit testing is enabled using [pytest](https://pytest.org/) combined with [doctest](https://docs.python.org/3/library/doctest.html) and [Hypothesis](https://hypothesis.works/) (to support [property-based testing](https://en.wikipedia.org/wiki/Software_testing#Property_testing)), and both code and branch coverage are measured using [coverage](https://github.com/nedbat/coveragepy) (see [below](#testing)). +Comprehensive testing is enabled using [pytest](https://pytest.org/) as a test runner, combined with [doctest](https://docs.python.org/3/library/doctest.html), [Hypothesis](https://hypothesis.works/) (to support [property-based testing](https://en.wikipedia.org/wiki/Software_testing#Property_testing)), [Faker](https://github.com/joke2k/faker) and [Cases](https://github.com/smarie/python-pytest-cases) (to produce valid, localized test case data), as well as both code and branch coverage are measured using [coverage](https://github.com/nedbat/coveragepy) (see [below](#testing)). ### Documentation @@ -80,7 +80,7 @@ If you’d like to start your own Python project from scratch, you can either co - Change the `LICENSE.md` file and the license badge according to your needs, and adjust the `SECURITY.md` file to your needs (more details [here](https://docs.github.com/en/code-security/getting-started/adding-a-security-policy-to-your-repository)). Remove all content from the `CHANGELOG.md` file. -- Rename the `src/package/` folder to whatever your own package’s name will be, adjust the Github Actions in `.github/workflows/`, and review the `Makefile`, `pyproject.toml`, `.pre-commit-config.yaml` files as well as the unit tests accordingly. **Note**: by default all Actions run on three different host types (Linux, MacOS, and Windows) whose [rates vary widely](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#minute-multipliers), so make sure that you disable or budget accordingly if you’re in a private repository! +- Rename the `src/package/` folder to whatever your own package’s name will be, adjust the Github Actions in `.github/workflows/`, and review the `Makefile`, `pyproject.toml`, `.pre-commit-config.yaml` files as well as the tests accordingly. **Note**: by default all Actions run on three different host types (Linux, MacOS, and Windows) whose [rates vary widely](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#minute-multipliers), so make sure that you disable or budget accordingly if you’re in a private repository! - A new protected `release` branch, should be created if it doesn't already exist. This branch should be configured with appropriate security policies and essential checks to ensure the integrity and stability of the release process. @@ -144,7 +144,7 @@ Using the pre-commit tool and its `.pre-commit-config.yaml` configuration, the f - When committing code, a number of [pre-commit hooks](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_committing_workflow_hooks) ensure that your code is formatted according to [PEP 8](https://www.python.org/dev/peps/pep-0008/) using the [`black`](https://github.com/psf/black) tool, and they’ll invoke [`flake8`](https://github.com/PyCQA/flake8) (and various plugins), [`pylint`](https://github.com/PyCQA/pylint) and [`mypy`](https://github.com/python/mypy) to check for lint and correct types. There are more checks, but those two are the important ones. You can adjust the settings for these tools in the `pyproject.toml` or `.flake8` configuration files. - The [commit message hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_committing_workflow_hooks) enforces [conventional commit messages](https://www.conventionalcommits.org/) and that, in turn, enables a _semantic release_ of this package on the Github side: upon merging changes into the `release` branch, the [release action](https://github.com/jenstroeger/python-package-template/blob/main/.github/workflows/release.yaml) uses the [Commitizen tool](https://commitizen-tools.github.io/commitizen/) to produce a [changelog](https://en.wikipedia.org/wiki/Changelog) and it computes the next version of this package and publishes a release — all based on the commit messages of a release. -- Using a [pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_other_client_hooks) this package is also set up to run [`pytest`](https://github.com/pytest-dev/pytest); in addition, the [`coverage`](https://github.com/nedbat/coveragepy) plugin makes sure that _all_ of your package’s code is covered by tests and [Hypothesis](https://hypothesis.works/) is already installed to help with generating test payloads. +- Using a [pre-push hook](https://git-scm.com/book/en/v2/Customizing-Git-Git-Hooks#_other_client_hooks) this package is also set up to run [`pytest`](https://github.com/pytest-dev/pytest); in addition, the [`coverage`](https://github.com/nedbat/coveragepy) plugin makes sure that _all_ of your package’s code is covered by unit tests and [Hypothesis](https://hypothesis.works/) and [Faker](https://github.com/joke2k/faker) are already installed to help with generating test case payloads. - The [`actionlint`](https://github.com/Mateusz-Grzelinski/actionlint-py) hook is set up to lint GitHub Actions workflows. If [`shellcheck`](https://github.com/koalaman/shellcheck) is installed on the system, `actionlint` runs `shellcheck` to lint the `run` steps in GitHub Actions. Note that `shellcheck` is available on [Ubuntu GitHub Actions runners](https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md) by default. You can also run these hooks manually, which comes in very handy during daily development tasks. For example @@ -163,43 +163,49 @@ runs _all_ installed git hooks over your code. For more control over the code ch ## Testing -As mentioned above, this repository is set up to use [pytest](https://pytest.org/) either standalone or as a pre-push git hook. Tests are stored in the `tests/` folder, and you can run them manually like so: +As mentioned above, this repository is set up to use [pytest](https://pytest.org/) as a test runner, either standalone or as a pre-push git hook. Tests are stored in the `tests/` folder, they’re organized into unit tests, integration tests, and performance tests. You can run the tests manually like so: ```bash -make test +make test-all # Run all tests: unit, integration, performance. ``` -which runs all tests in both your local Python virtual environment. For more options, see the [pytest command-line flags](https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags). Also note that pytest includes [doctest](https://docs.python.org/3/library/doctest.html), which means that module and function [docstrings](https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring), as well as the documentation, may contain test code that executes as part of the unit tests. +which runs all tests in your local Python virtual environment. For more options, see the [pytest command-line flags](https://docs.pytest.org/en/7.4.x/reference/reference.html#command-line-flags). -Both statement and branch coverage are being tracked using [coverage](https://github.com/nedbat/coveragepy) and the [pytest-cov](https://github.com/pytest-dev/pytest-cov) plugin for pytest, and it measures how much code in the `src/package/` folder is covered by tests: -``` -Run unit tests...........................................................Passed -- hook id: pytest -- duration: 0.76s +There are three kinds of tests: + +1. [Unit tests](https://en.wikipedia.org/wiki/Unit_testing) are invoked with `make test` or `make test-unit`; +2. [Integration tests](https://en.wikipedia.org/wiki/Integration_testing) are invoked with `make test-integration`; and +3. [Performance tests](https://en.wikipedia.org/wiki/Software_performance_testing) (using the [pytest-benchmark](https://github.com/ionelmc/pytest-benchmark) plugin) are invoked with `make test-performance`. + +Note that the unit tests include [doctest](https://docs.python.org/3/library/doctest.html), which means that module and function [docstrings](https://www.python.org/dev/peps/pep-0257/#what-is-a-docstring), as well as the documentation, may contain test code that executes as part of the unit tests. When pushing changes to the remote, the pre-push hook runs *only* unit tests whereas *all* three kinds of tests run as part of Github’s CI. -============================= test session starts ============================== -platform darwin -- Python 3.13.1, pytest-8.3.4, pluggy-1.5.0 -- /path/to/python-package-template/.venv/bin/python +Both statement and branch coverage are being tracked using [coverage](https://github.com/nedbat/coveragepy) and the [pytest-cov](https://github.com/pytest-dev/pytest-cov) plugin for pytest when running unit tests, and it measures how much code in the `src/package/` folder is covered by tests: +``` +=========================================== test session starts =========================================== +platform darwin -- Python 3.13.9, pytest-8.4.2, pluggy-1.6.0 -- /path/to/python-package-template/.venv/bin/python cachedir: .pytest_cache -hypothesis profile 'default-with-verbose-verbosity' -> max_examples=500, verbosity=Verbosity.verbose, database=DirectoryBasedExampleDatabase(PosixPath('/path/to/python-package-template/.hypothesis/examples')) +hypothesis profile 'default-with-verbose-verbosity' -> verbosity=Verbosity.verbose +benchmark: 5.2.0 (defaults: timer=time.perf_counter disable_gc=False min_rounds=5 min_time=0.000005 max_time=1.0 calibration_precision=10 warmup=False warmup_iterations=100000) rootdir: /path/to/python-package-template configfile: pyproject.toml -plugins: cov-6.0.0, hypothesis-6.122.7, env-1.1.5, custom-exit-code-0.3.0, doctestplus-1.3.0 -collected 3 items +plugins: cases-3.9.1, hypothesis-6.138.16, env-1.1.5, cov-6.3.0, custom-exit-code-0.3.0, doctestplus-1.4.0, Faker-37.6.0, benchmark-5.2.0 +collected 6 items / 2 deselected / 4 selected + +src/package/something.py::package.something.Something.do_something PASSED [ 25%] +tests/unit/test_something.py::test_something_hypothesis PASSED [ 50%] +tests/unit/test_something.py::test_something_cases[_case_boolean] PASSED [ 75%] +docs/source/index.rst::index.rst PASSED [100%] -src/package/something.py::package.something.Something.do_something PASSED [ 33%] -tests/test_something.py::test_something PASSED [ 66%] -docs/source/index.rst::index.rst PASSED [100%] +============================================= tests coverage ============================================== +____________________________ coverage: platform darwin, python 3.13.9-final-0 _____________________________ ----------- coverage: platform darwin, python 3.13.1-final-0 ---------- Name Stmts Miss Branch BrPart Cover Missing ---------------------------------------------------------------------- src/package/__init__.py 1 0 0 0 100% src/package/something.py 4 0 0 0 100% ---------------------------------------------------------------------- TOTAL 5 0 0 0 100% - Required test coverage of 100.0% reached. Total coverage: 100.00% -============================ Hypothesis Statistics ============================= - -tests/test_something.py::test_something: +========================================== Hypothesis Statistics ========================================== +tests/unit/test_something.py::test_something_hypothesis: - during generate phase (0.00 seconds): - Typical runtimes: ~ 0-1 ms, of which < 1ms in data generation @@ -207,22 +213,24 @@ tests/test_something.py::test_something: - Stopped because nothing left to do - -============================== slowest durations =============================== -0.01s call tests/test_something.py::test_something +============================================ slowest durations ============================================ +0.10s setup src/package/something.py::package.something.Something.do_something +0.01s call tests/unit/test_something.py::test_something_hypothesis 0.00s call src/package/something.py::package.something.Something.do_something -0.00s setup src/package/something.py::package.something.Something.do_something +0.00s setup tests/unit/test_something.py::test_something_cases[_case_boolean] 0.00s call docs/source/index.rst::index.rst -0.00s teardown src/package/something.py::package.something.Something.do_something 0.00s setup docs/source/index.rst::index.rst +0.00s teardown src/package/something.py::package.something.Something.do_something +0.00s teardown tests/unit/test_something.py::test_something_cases[_case_boolean] +0.00s call tests/unit/test_something.py::test_something_cases[_case_boolean] +0.00s setup tests/unit/test_something.py::test_something_hypothesis 0.00s teardown docs/source/index.rst::index.rst -0.00s setup tests/test_something.py::test_something -0.00s teardown tests/test_something.py::test_something -============================== 3 passed in 0.09s =============================== +0.00s teardown tests/unit/test_something.py::test_something_hypothesis +===================================== 4 passed, 2 deselected in 0.23s ===================================== ``` Note that code that’s not covered by tests is listed under the `Missing` column, and branches not taken too. The net effect of enforcing 100% code and branch coverage is that every new major and minor feature, every code change, and every fix are being tested (keeping in mind that high _coverage_ does not imply comprehensive, meaningful _test data_). -Hypothesis is a package that implements [property based testing](https://en.wikipedia.org/wiki/Software_testing#Property_testing) and that provides payload generation for your tests based on strategy descriptions ([more](https://hypothesis.works/#what-is-hypothesis)). Using its [pytest plugin](https://hypothesis.readthedocs.io/en/latest/details.html#the-hypothesis-pytest-plugin) Hypothesis is ready to be used for this package. +Hypothesis is a package that implements [property based testing](https://en.wikipedia.org/wiki/Software_testing#Property_testing) and that provides payload generation for your tests based on strategy descriptions ([more](https://hypothesis.works/#what-is-hypothesis)). Using its [pytest plugin](https://hypothesis.readthedocs.io/en/latest/details.html#the-hypothesis-pytest-plugin) Hypothesis is ready to be used for this package. Likewise, the [Faker](https://github.com/joke2k/faker) package and its [pytest plugin](https://faker.readthedocs.io/en/master/#pytest-fixtures) are installed to provide valid, localized test case data (see also the [pytest-cases](https://github.com/smarie/python-pytest-cases) plugin for more details on how to separate tests and test cases). ## Generating documentation diff --git a/pyproject.toml b/pyproject.toml index 76bd76e1..8942685c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,14 +41,14 @@ something = "package.__main__:main" # We keep them listed here to ensure the infrastructure BOM is consistent with what's # installed. Make sure to keep the requirements in sync with the workflows! actions = [ - "commitizen ==4.9.1", + "commitizen ==4.10.0", "twine ==6.2.0", ] dev = [ "flit >=3.2.0,<4.0.0", - "mypy >=1.0.0,<1.19", + "mypy >=1.0.0,<1.20", "pip-audit >=2.4.4,<3.0.0", - "pylint >=3.0.0,<3.4.0", + "pylint >=3.0.0,<4.1.0", "perflint >=0.8.0,<1.0.0", "cyclonedx-bom >=4.0.0,<5.0.0", ] @@ -57,19 +57,20 @@ docs = [ "sphinx-markdown-builder >=0.6.4,<1.0.0", ] hooks = [ - "pre-commit >=3.0.0,<4.4.0", + "pre-commit >=3.0.0,<4.6.0", ] # Note that the `custom_exit_code` and `env` plugins may currently be unmaintained. test = [ "coverage ==7.6.12; python_version<'3.14'", # https://github.com/pypi/warehouse/pull/17872#issuecomment-2845932281 - "faker ==37.6.0", - "hypothesis >=6.21.0,<6.138.17", + "faker ==38.2.0", + "hypothesis >=6.21.0,<6.148.6", "pytest >=7.2.0,<9.0.0", + "pytest-benchmark ==5.2.0", "pytest-cases ==3.9.1", "pytest-custom_exit_code ==0.3.0", "pytest-cov ==6.3.0", # Uses: coverage[toml] >=7.5 - "pytest-doctestplus ==1.4.0", - "pytest-env ==1.1.5", + "pytest-doctestplus ==1.6.0", + "pytest-env ==1.2.0", ] [project.urls] @@ -258,7 +259,7 @@ max-line-length = 120 # https://github.com/yashtodi94/pytest-custom_exit_code [tool.pytest.ini_options] minversion = "7.0" -addopts = """-vv -ra --tb native --durations 0 \ +addopts = """-vv -ra --tb native --durations 0 --strict-markers --import-mode importlib \ --hypothesis-show-statistics --hypothesis-explain --hypothesis-verbosity verbose \ --doctest-modules --doctest-continue-on-failure --doctest-glob '*.rst' --doctest-plus \ --suppress-no-test-exit-code \ @@ -281,3 +282,7 @@ filterwarnings = [ "error::pytest.PytestUnraisableExceptionWarning", "error::pytest.PytestUnhandledThreadExceptionWarning", ] +markers = [ + "integration: more complex application-level integration tests.", + "performance: performance tests.", +] diff --git a/tests/integration/test_something.py b/tests/integration/test_something.py new file mode 100644 index 00000000..14bb9f3c --- /dev/null +++ b/tests/integration/test_something.py @@ -0,0 +1,16 @@ +"""Test the Package itself using its external interface as in integration into a larger run context.""" + +# https://bandit.readthedocs.io/en/latest/blacklists/blacklist_imports.html#b404-import-subprocess +import subprocess # nosec B404 + +import pytest + + +@pytest.mark.integration +def test_package() -> None: + """Test the Something command.""" + # For testing we disable this warning here: + # https://bandit.readthedocs.io/en/latest/plugins/b603_subprocess_without_shell_equals_true.html + # https://bandit.readthedocs.io/en/latest/plugins/b607_start_process_with_partial_path.html + completed = subprocess.run(["something"], check=True, shell=False) # nosec B603, B607 + assert completed.returncode == 0 diff --git a/tests/performance/test_something.py b/tests/performance/test_something.py new file mode 100644 index 00000000..c972db22 --- /dev/null +++ b/tests/performance/test_something.py @@ -0,0 +1,12 @@ +"""Test the performance of various package parts, or the package as a whole.""" + +import pytest +from pytest_benchmark.fixture import BenchmarkFixture + +from package.something import Something + + +@pytest.mark.performance +def test_something(benchmark: BenchmarkFixture) -> None: + """Test performance of the function.""" + benchmark.pedantic(Something.do_something, iterations=10, rounds=100) # type: ignore[no-untyped-call] diff --git a/tests/conftest.py b/tests/unit/conftest.py similarity index 100% rename from tests/conftest.py rename to tests/unit/conftest.py diff --git a/tests/test_something.py b/tests/unit/test_something.py similarity index 88% rename from tests/test_something.py rename to tests/unit/test_something.py index 7d59eb9e..eb0d07b8 100644 --- a/tests/test_something.py +++ b/tests/unit/test_something.py @@ -1,4 +1,4 @@ -"""Test the Something module. Add more tests here, as needed.""" +"""Test the Something module as a unit test. Add more tests here, as needed.""" import faker from hypothesis import given, strategies