diff --git a/.gitignore b/.gitignore index 67a5299c..86b63144 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,7 @@ dist # Terraform .terraform *.tfstate.backup + +# Coverage results +*.coverage +test/coverage/results/ diff --git a/Makefile b/Makefile index 4dbce3c2..9060dae8 100644 --- a/Makefile +++ b/Makefile @@ -5,9 +5,13 @@ include vars.mk .PHONY: check check: environment pipenv run yapf -rd . + rm -rf test/coverage/results + mkdir test/coverage/results $(MAKE) -C cli check $(MAKE) -C services/controller check BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) python3 test/run.py + RUN_WITH_COVERAGE=yes BUILD_TIMESTAMP=$(BUILD_TIMESTAMP) python3 test/run.py + pipenv run test/coverage/coverage_report.sh terraform fmt -check .PHONY: format diff --git a/Pipfile b/Pipfile index 835704d2..1df6f7d0 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ verify_ssl = true name = "pypi" [packages] +coverage = "*" [dev-packages] yapf = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 6c07fb4a..6d373f89 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3ec5d8bbb099edcd508a073566bb71a44558e8bdc9c67f3755c94e9b60c927bd" + "sha256": "091a3f897accfb06d283b2158fb03f053fa62cbe5c3adf3702a024b25db1b5e5" }, "pipfile-spec": 6, "requires": { @@ -15,7 +15,45 @@ } ] }, - "default": {}, + "default": { + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + } + }, "develop": { "yapf": { "hashes": [ diff --git a/cli/Dockerfile b/cli/Dockerfile index 563e7a90..1567ebf4 100644 --- a/cli/Dockerfile +++ b/cli/Dockerfile @@ -14,7 +14,7 @@ RUN python setup.py bdist_wheel FROM python:3.6 COPY --from=builder /cli/dist/plz_cli-0.1.0-py3-none-any.whl /tmp/ -RUN pip install /tmp/plz_cli-0.1.0-py3-none-any.whl +RUN pip install --find-links /tmp/ plz_cli[test] ENV PYTHONUNBUFFERED 1 ENTRYPOINT ["plz"] diff --git a/cli/Makefile b/cli/Makefile index f871bf51..85bb0d6a 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -5,4 +5,4 @@ check: test lint .PHONY: test test: environment - pipenv run nosetests + COVERAGE_FILE=../test/coverage/results/cli_nose_tests.coverage pipenv run coverage run --source=. -m nose diff --git a/cli/Pipfile b/cli/Pipfile index ca1d6931..322612ad 100644 --- a/cli/Pipfile +++ b/cli/Pipfile @@ -8,9 +8,10 @@ python_version = "3.6" [packages] "e1839a8" = {path = ".", editable = true} -yapf = "*" [dev-packages] flake8 = "*" flask = "*" nose = "*" +coverage = "*" +yapf = "*" diff --git a/cli/Pipfile.lock b/cli/Pipfile.lock index 42a975c6..e5dab151 100644 --- a/cli/Pipfile.lock +++ b/cli/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "51806429b4eeb894e27ed8de24d388614b5075b031e3318c5e9b03d43f534908" + "sha256": "0507776da6744c0b36c405cde301a0b58a0cc1bc24ac5d0863e862480308bf73" }, "pipfile-spec": 6, "requires": { @@ -235,14 +235,6 @@ "sha256:1fd5520878b68b84b5748bb30e592b10d0a91529d5383f74f4964e72b297fd3a" ], "version": "==0.56.0" - }, - "yapf": { - "hashes": [ - "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", - "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" - ], - "index": "pypi", - "version": "==0.27.0" } }, "develop": { @@ -253,6 +245,43 @@ ], "version": "==7.0" }, + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -359,6 +388,14 @@ "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], "version": "==0.15.2" + }, + "yapf": { + "hashes": [ + "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", + "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" + ], + "index": "pypi", + "version": "==0.27.0" } } } diff --git a/cli/setup.py b/cli/setup.py index 705a25de..df29190d 100644 --- a/cli/setup.py +++ b/cli/setup.py @@ -53,9 +53,10 @@ ], extras_require={ 'test': [ - 'flake8==3.5.0', - 'nose==1.3.7', + 'flake8>=3.5.0', + 'nose>=1.3.7', 'flask >= 1.0.2', + 'coverage >= 4.5.3', ], }, entry_points={ diff --git a/services/controller/Dockerfile b/services/controller/Dockerfile index 8ae79be4..de524cf6 100644 --- a/services/controller/Dockerfile +++ b/services/controller/Dockerfile @@ -26,6 +26,6 @@ ARG BUILD_TIMESTAMP=0 RUN echo -n $BUILD_TIMESTAMP > ./src/plz/controller/BUILD_TIMESTAMP -HEALTHCHECK --interval=5s CMD curl -f http://localhost/ +HEALTHCHECK --interval=5s --start-period=60s CMD curl -f http://localhost/ ENTRYPOINT ["/tini", "--", "./run"] diff --git a/services/controller/Pipfile b/services/controller/Pipfile index 035fb678..2be703b0 100644 --- a/services/controller/Pipfile +++ b/services/controller/Pipfile @@ -12,7 +12,8 @@ pyhocon = "*" urllib3 = ">=1.24.2" redis = "*" requests = ">=2.20.0" -yapf = "*" [dev-packages] -"flake8" = "*" +coverage = "*" +flake8 = "*" +yapf = "*" diff --git a/services/controller/Pipfile.lock b/services/controller/Pipfile.lock index cd9bf2b9..7c92f424 100644 --- a/services/controller/Pipfile.lock +++ b/services/controller/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "12e843ac392d34367a32f8d6d38930c7b4e37080667c2e3ce325798d1ee2a4ff" + "sha256": "d1f18359aa6c33178f8274d8f083be57ff8d69e18caba1410993168ade0141da" }, "pipfile-spec": 6, "requires": {}, @@ -16,18 +16,18 @@ "default": { "boto3": { "hashes": [ - "sha256:2292bb7cb9de3e5a2bda9a78a4ac890961b80fc453add0ac87eca6f726b0cd6d", - "sha256:dc94237a8e9c104c86d77ec9b3ecebd864680b5a4fbd22b28722376111dbec47" + "sha256:679d8084ad40d18908a97c785d614fed554a424924d4ab30e464c16bfe95722b", + "sha256:d5ccb985caf4ea522f2fbfe4fbf270cd1e2c0c6d46ea7d13e9cda6bb6c36deb6" ], "index": "pypi", - "version": "==1.9.136" + "version": "==1.9.138" }, "botocore": { "hashes": [ - "sha256:b12e0efe214e35d0217e13e8e57677ae70b63c8bd01c960dcda92d05dd55ac6b", - "sha256:c24b9c32fdf2643a6cdeca55a43dc57e5b7b92a437d6e092807c9dc3f98c0c78" + "sha256:73bf439ba6d97606f8acbe4e037cc7a6e7a2b83f080b472c37c22d810c7dd8a8", + "sha256:ff4171f850cfb221b553f32948b93ea0a8d82e636fe121ff08945f4581c21ad9" ], - "version": "==1.12.136" + "version": "==1.12.138" }, "certifi": { "hashes": [ @@ -223,17 +223,46 @@ "sha256:7fad9770a8778f9576693f0cc29c7dcc36964df916b83734f4431c0e612a7fbc" ], "version": "==0.15.2" - }, - "yapf": { - "hashes": [ - "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", - "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" - ], - "index": "pypi", - "version": "==0.27.0" } }, "develop": { + "coverage": { + "hashes": [ + "sha256:3684fabf6b87a369017756b551cef29e505cb155ddb892a7a29277b978da88b9", + "sha256:39e088da9b284f1bd17c750ac672103779f7954ce6125fd4382134ac8d152d74", + "sha256:3c205bc11cc4fcc57b761c2da73b9b72a59f8d5ca89979afb0c1c6f9e53c7390", + "sha256:465ce53a8c0f3a7950dfb836438442f833cf6663d407f37d8c52fe7b6e56d7e8", + "sha256:48020e343fc40f72a442c8a1334284620f81295256a6b6ca6d8aa1350c763bbe", + "sha256:5296fc86ab612ec12394565c500b412a43b328b3907c0d14358950d06fd83baf", + "sha256:5f61bed2f7d9b6a9ab935150a6b23d7f84b8055524e7be7715b6513f3328138e", + "sha256:68a43a9f9f83693ce0414d17e019daee7ab3f7113a70c79a3dd4c2f704e4d741", + "sha256:6b8033d47fe22506856fe450470ccb1d8ba1ffb8463494a15cfc96392a288c09", + "sha256:7ad7536066b28863e5835e8cfeaa794b7fe352d99a8cded9f43d1161be8e9fbd", + "sha256:7bacb89ccf4bedb30b277e96e4cc68cd1369ca6841bde7b005191b54d3dd1034", + "sha256:839dc7c36501254e14331bcb98b27002aa415e4af7ea039d9009409b9d2d5420", + "sha256:8f9a95b66969cdea53ec992ecea5406c5bd99c9221f539bca1e8406b200ae98c", + "sha256:932c03d2d565f75961ba1d3cec41ddde00e162c5b46d03f7423edcb807734eab", + "sha256:988529edadc49039d205e0aa6ce049c5ccda4acb2d6c3c5c550c17e8c02c05ba", + "sha256:998d7e73548fe395eeb294495a04d38942edb66d1fa61eb70418871bc621227e", + "sha256:9de60893fb447d1e797f6bf08fdf0dbcda0c1e34c1b06c92bd3a363c0ea8c609", + "sha256:9e80d45d0c7fcee54e22771db7f1b0b126fb4a6c0a2e5afa72f66827207ff2f2", + "sha256:a545a3dfe5082dc8e8c3eb7f8a2cf4f2870902ff1860bd99b6198cfd1f9d1f49", + "sha256:a5d8f29e5ec661143621a8f4de51adfb300d7a476224156a39a392254f70687b", + "sha256:aca06bfba4759bbdb09bf52ebb15ae20268ee1f6747417837926fae990ebc41d", + "sha256:bb23b7a6fd666e551a3094ab896a57809e010059540ad20acbeec03a154224ce", + "sha256:bfd1d0ae7e292105f29d7deaa9d8f2916ed8553ab9d5f39ec65bcf5deadff3f9", + "sha256:c62ca0a38958f541a73cf86acdab020c2091631c137bd359c4f5bddde7b75fd4", + "sha256:c709d8bda72cf4cd348ccec2a4881f2c5848fd72903c185f363d361b2737f773", + "sha256:c968a6aa7e0b56ecbd28531ddf439c2ec103610d3e2bf3b75b813304f8cb7723", + "sha256:df785d8cb80539d0b55fd47183264b7002077859028dfe3070cf6359bf8b2d9c", + "sha256:f406628ca51e0ae90ae76ea8398677a921b36f0bd71aab2099dfed08abd0322f", + "sha256:f46087bbd95ebae244a0eda01a618aff11ec7a069b15a3ef8f6b520db523dcf1", + "sha256:f8019c5279eb32360ca03e9fac40a12667715546eed5c5eb59eb381f2f501260", + "sha256:fc5f4d209733750afd2714e9109816a29500718b32dd9a5db01c0cb3a019b96a" + ], + "index": "pypi", + "version": "==4.5.3" + }, "entrypoints": { "hashes": [ "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", @@ -269,6 +298,14 @@ "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], "version": "==2.1.1" + }, + "yapf": { + "hashes": [ + "sha256:34f6f80c446dcb2c44bd644c4037a2024b6645e293a4c9c4521983dd0bb247a1", + "sha256:613deba14233623ff3432d9d5032631b5f600be97b39f66932cbe67648bfa8ea" + ], + "index": "pypi", + "version": "==0.27.0" } } } diff --git a/services/controller/run b/services/controller/run index d59ed837..29a76950 100755 --- a/services/controller/run +++ b/services/controller/run @@ -20,12 +20,18 @@ if [[ "${CREATE_AWS_RESOURCES:-}" ]]; then python src/plz/controller/utils/create_aws_resources.py fi -exec gunicorn \ - --bind="0.0.0.0:${PORT}" \ - --workers=16 \ - --timeout=2000 \ - --pythonpath="${PYTHONPATH}" \ - --capture-output \ - --log-level="${LOG_LEVEL}" \ - plz.controller.main:app \ - -- $@ +if [[ "${RUN_WITH_COVERAGE:-}" ]]; then + pipenv install --dev --system --deploy + COVERAGE_FILE=controller.coverage exec coverage run \ + --source=. src/plz/controller/main.py +else + exec gunicorn \ + --bind="0.0.0.0:${PORT}" \ + --workers=16 \ + --timeout=2000 \ + --pythonpath="${PYTHONPATH}" \ + --capture-output \ + --log-level="${LOG_LEVEL}" \ + plz.controller.main:app \ + -- $@; +fi diff --git a/services/controller/src/plz/controller/main.py b/services/controller/src/plz/controller/main.py index c9679d33..6ee21e24 100644 --- a/services/controller/src/plz/controller/main.py +++ b/services/controller/src/plz/controller/main.py @@ -8,6 +8,7 @@ import requests from flask import Flask, Response, abort, jsonify, request, stream_with_context +import plz.controller from plz.controller import configuration from plz.controller.api.exceptions import AbortedExecutionException, \ InstanceNotRunningException, JSONResponseException, \ @@ -40,8 +41,7 @@ def _setup_logging(): print(f'Setting log level to: {log_level}', file=sys.stderr, flush=True) - controller_logger = logging.getLogger('.'.join( - __name__.split('.')[:-1])) + controller_logger = logging.getLogger(plz.controller.__name__) controller_logger.setLevel(log_level) diff --git a/test/coverage/coverage_report.sh b/test/coverage/coverage_report.sh new file mode 100755 index 00000000..12e1deb9 --- /dev/null +++ b/test/coverage/coverage_report.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +if command -v realpath > /dev/null; then + DIR_OF_THIS_SCRIPT="$(cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")" && pwd)" +else + DIR_OF_THIS_SCRIPT="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +fi + + +RESULTS_DIR="${DIR_OF_THIS_SCRIPT}/results" +RCFILE="${DIR_OF_THIS_SCRIPT}/coveragerc" + +# Use a new shell to cd to the plz root path. Otherwise the script would +# depend on pwd, as the paths in the rcfile are constant +( \ + cd "$DIR_OF_THIS_SCRIPT"/../.. && \ + COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage combine \ + --rcfile="${RCFILE}" "${RESULTS_DIR}"/*.coverage; + COVERAGE_FILE="${RESULTS_DIR}/total.coverage" coverage report \ + --rcfile="${RCFILE}" | sed "s:`pwd`::" +) + diff --git a/test/coverage/coveragerc b/test/coverage/coveragerc new file mode 100644 index 00000000..5b8b685e --- /dev/null +++ b/test/coverage/coveragerc @@ -0,0 +1,19 @@ +[paths] +# The containers that run the CLI install plz at the system level +cli_source = + cli/src/plz/cli + /usr/*/plz/cli +# The CLI uses the api submodule of the controller +controller_api_source = + services/controller/src/plz/controller + /usr/*/plz/controller +# The container for the controller has the code inside src/src +controller_container_source = + services/controller/src/ + /src/src + +[report] +# Do not count "pass" lines in abstract methods and properties +exclude_lines = + pragma: no cover + @abstract diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 0c31b13a..141e0bb8 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -16,6 +16,7 @@ services: context: ../services/controller container_name: $CONTROLLER_CONTAINER environment: + RUN_WITH_COVERAGE: ${RUN_WITH_COVERAGE:-} CONFIGURATION: | # copied from ../start/docker.conf, as we can't mount files inside CircleCI log_level = DEBUG diff --git a/test/run.py b/test/run.py index d13d81f9..e7831030 100644 --- a/test/run.py +++ b/test/run.py @@ -12,8 +12,8 @@ from test_utils import CLI_BUILDER_IMAGE, CLI_IMAGE, \ CONTROLLER_CONTAINER, CONTROLLER_HOSTNAME, CONTROLLER_IMAGE, \ CONTROLLER_PORT, CONTROLLER_TESTS_CONTAINER, CONTROLLER_TESTS_IMAGE, \ - DATA_DIRECTORY, PLZ_ROOT_DIRECTORY, PLZ_USER, TEST_DIRECTORY, \ - docker_compose, get_network + DATA_DIRECTORY, PLZ_ROOT_DIRECTORY, PLZ_USER, \ + TEST_DIRECTORY, docker_compose, get_network def start_controller(): @@ -171,6 +171,13 @@ def main(): options = parser.parse_args(sys.argv[1:]) + test_utils.print_info('=' * 70) + if test_utils.running_with_coverage(): + test_utils.print_info('Running with flask dev server, with coverage') + else: + test_utils.print_info('Running with gunicorn, without coverage') + test_utils.print_info('=' * 70) + if options.end_to_end_only and options.controller_tests_only: raise ValueError('Options end_to_end_only and controller_tests_only ' 'can\'t be specified simultaneously') diff --git a/test/run_end_to_end_test.py b/test/run_end_to_end_test.py index fc37b3a5..ee1fc38c 100644 --- a/test/run_end_to_end_test.py +++ b/test/run_end_to_end_test.py @@ -9,8 +9,8 @@ from tempfile import NamedTemporaryFile, TemporaryDirectory, TemporaryFile import test_utils -from test_utils import CLI_CONTAINER_PREFIX, DATA_DIRECTORY, PLZ_USER, \ - TEST_DIRECTORY, VOLUME_PREFIX, get_network +from test_utils import CLI_CONTAINER_PREFIX, COVERAGE_RESULTS_DIRECTORY, \ + DATA_DIRECTORY, PLZ_USER, TEST_DIRECTORY, VOLUME_PREFIX, get_network def run_end_to_end_test(network: str, plz_host: str, plz_port: int, @@ -26,6 +26,10 @@ def run_end_to_end_test(network: str, plz_host: str, plz_port: int, """ if not os.path.isdir(DATA_DIRECTORY): os.mkdir(DATA_DIRECTORY) + # Keep the coverage results separate because we are going to use them after + # the tests (the data directory gets wiped after all tests have run) + if not os.path.isdir(COVERAGE_RESULTS_DIRECTORY): + os.mkdir(COVERAGE_RESULTS_DIRECTORY) # Make sure the directory has a single slash at the end test_directory = os.path.join( @@ -59,6 +63,7 @@ def run_end_to_end_test(network: str, plz_host: str, plz_port: int, app_directory=test_directory, actual_logs_file=logs_file, output_directory_name=output_directory_name) + end = datetime.datetime.now() test_utils.print_info(f'Time taken: {end-start}') @@ -118,6 +123,15 @@ def run_end_to_end_test(network: str, plz_host: str, plz_port: int, return True +# Replace non-common characters by hyphens +def hiphenize(st: str): + return re.sub(r'[^0-9a-zA-Z_]', '-', st) + + +def coverage_file_name(test_name: str): + return hiphenize(test_name) + '.coverage' + + def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, app_directory: str, actual_logs_file: TemporaryFile, output_directory_name: str) -> int: @@ -125,10 +139,9 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, :return: exit code of the test """ output_directory_name = os.path.abspath(output_directory_name) - project_name = re.sub(r'[^0-9a-zA-Z_]', '-', test_name) + project_name = hiphenize(test_name) test_config_file = f'{app_directory}/test.config.json' - suffix = re.sub(r'[^0-9a-zA-Z_]', '-', - '-'.join(os.path.split(test_name)[-2:])) + suffix = hiphenize('-'.join(os.path.split(test_name)[-2:])) cli_container = f'{CLI_CONTAINER_PREFIX}_{suffix}' volume = f'{VOLUME_PREFIX}{suffix}' @@ -155,18 +168,44 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, 'docker:stable-git', 'git', 'init', '--quiet', '/data/app' ]) + if test_utils.running_with_coverage(): + docker_run_args = [ + f'--entrypoint', + 'coverage', + test_utils.CLI_IMAGE, + # This run is for `coverage` + 'run', + '-m', + '--source=plz', + 'plz.cli.main' + ] + else: + docker_run_args = [test_utils.CLI_IMAGE] + # Start the CLI process. test_utils.execute_command([ - 'docker', 'container', 'run', f'--name={cli_container}', f'--detach', - f'--network={network}', f'--env=PLZ_HOST={plz_host}', - f'--env=PLZ_PORT={plz_port}', f'--env=PLZ_USER={PLZ_USER}', + 'docker', + 'container', + 'run', + f'--name={cli_container}', + f'--detach', + f'--network={network}', + f'--env=PLZ_HOST={plz_host}', + f'--env=PLZ_PORT={plz_port}', + f'--env=PLZ_USER={PLZ_USER}', f'--env=PLZ_PROJECT={project_name}', f'--env=PLZ_INSTANCE_MARKET_TYPE=spot', f'--env=PLZ_MAX_BID_PRICE_IN_DOLLARS_PER_HOUR=0.5', f'--env=PLZ_INSTANCE_MAX_UPTIME_IN_MINUTES=0', - f'--env=PLZ_QUIET_BUILD=true', f'--workdir=/data/app', - f'--volume={volume}:/data', test_utils.CLI_IMAGE, 'run', - '--output=/data/output', *test_args + f'--env=PLZ_QUIET_BUILD=true', + f'--env=COVERAGE_FILE=/data/coverage', + f'--workdir=/data/app', + f'--volume={volume}:/data', + *docker_run_args, + # This run is for `plz` + 'run', + '--output=/data/output', + *test_args ]) # Capture the logs @@ -189,6 +228,13 @@ def run_cli(network: str, plz_host: str, plz_port: int, test_name: str, hide_output=True) exit_status = int(b''.join(stdout_holder)) + if test_utils.running_with_coverage(): + test_utils.execute_command([ + 'docker', 'cp', f'{cli_container}:/data/coverage', + os.path.join(COVERAGE_RESULTS_DIRECTORY, + coverage_file_name(test_name)) + ]) + test_utils.execute_command(['docker', 'container', 'rm', cli_container], hide_output=True) diff --git a/test/test_utils.py b/test/test_utils.py index 740dc8b7..f12bb51a 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -29,6 +29,8 @@ TEST_DIRECTORY = os.path.dirname(os.path.abspath(__file__)) PLZ_ROOT_DIRECTORY = os.path.abspath( os.path.join(os.path.dirname(os.path.abspath(__file__)), '..')) +COVERAGE_RESULTS_DIRECTORY = os.path.join(TEST_DIRECTORY, 'coverage', + 'results') DATA_DIRECTORY = f'{PLZ_ROOT_DIRECTORY}/test_cache/' REDIS_DATA_DIRECTORY = f'{DATA_DIRECTORY}/redis_data/' @@ -255,8 +257,22 @@ def stop_all_clis(): def stop_controller(): + if running_with_coverage(): + # Unless we interrupt the server before stopping, coverage won't write + # the report + execute_command( + ['docker', 'kill', '--signal=INT', CONTROLLER_CONTAINER]) + execute_command([ + 'docker', 'container', 'cp', + f'{CONTROLLER_CONTAINER}:/src/controller.coverage', + os.path.join(COVERAGE_RESULTS_DIRECTORY, 'controller.coverage') + ]) docker_compose('stop') - docker_compose('logs') + # When we are running with coverage don't print the logs. They will be + # printed in the run without coverage. If we print both times, the whole + # output of the tests is too big for CircleCI + if not running_with_coverage(): + docker_compose('logs') docker_compose('down', '--volumes') @@ -287,3 +303,7 @@ def docker_compose(*args): f'--file={os.path.join(TEST_DIRECTORY, "docker-compose.yml")}', *args ], env=env) + + +def running_with_coverage(): + return os.environ.get('RUN_WITH_COVERAGE', '') != ''