From 75fe71b1917a37ef00b354b32e42288243b56258 Mon Sep 17 00:00:00 2001 From: "James H. Nguyen" Date: Mon, 14 Apr 2025 00:52:13 -0700 Subject: [PATCH 1/3] Fix: Restore accurate test coverage reporting in CI pipeline --- .coveragerc | 3 --- .github/workflows/python-ci.yml | 16 ++++++++-------- pr_description.md | 23 ----------------------- scripts/run_coverage_ci.sh | 2 +- 4 files changed, 9 insertions(+), 35 deletions(-) delete mode 100644 pr_description.md diff --git a/.coveragerc b/.coveragerc index fd4586d..0eabdcd 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,8 +1,5 @@ [run] source = src.cli_code -include = - */src/cli_code/tools/test_runner.py - */src/cli_code/models/gemini.py omit = */.rules/* */.venv/* diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 5bf4202..89526fc 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -87,19 +87,19 @@ jobs: run: | # Set CI variables export CI_TEST_TIMEOUT=120 - export CI_EXIT_ON_TEST_FAILURE=0 + export CI_EXIT_ON_TEST_FAILURE=1 # Run directly without using bash to make error handling clearer bash -ex ./scripts/run_coverage_ci.sh || echo "Coverage generation had errors but we'll continue" - # Generate dummy coverage if needed - if [ ! -f "coverage.xml" ]; then - echo "Creating placeholder coverage.xml file" - echo '/Users/runner/work/cli-code/cli-code/src' > coverage.xml + # Extract actual coverage percentage from XML file + if [ -f "coverage.xml" ]; then + COVERAGE=$(python -c "import xml.etree.ElementTree as ET; tree = ET.parse('coverage.xml'); root = tree.getroot(); line_rate = float(root.attrib['line-rate'])*100; print('{:.2f}%'.format(line_rate))") + echo "percentage=$COVERAGE" >> $GITHUB_OUTPUT + else + echo "Warning: No coverage.xml file was generated" + echo "percentage=0.00%" >> $GITHUB_OUTPUT fi - - # Set a fixed coverage percentage for PR comment - echo "percentage=85.00%" >> $GITHUB_OUTPUT - name: Comment PR with code coverage if: github.event_name == 'pull_request' diff --git a/pr_description.md b/pr_description.md deleted file mode 100644 index 5833d7a..0000000 --- a/pr_description.md +++ /dev/null @@ -1,23 +0,0 @@ -## Overview -This PR fixes the code coverage regression by updating the import statements in test files and using improved test versions. - -## Changes -- Updated import statements in test files to use direct imports from `src.cli_code` instead of `cli_code` to ensure proper coverage tracking -- Updated test scripts to use the improved test files from the `test_dir/improved` directory -- Added BaseTool tests in coverage scripts to improve coverage of the base tool class -- Fixed failing assertions in the Gemini model tests -- Updated tools coverage script to include all the necessary tool tests - -## Test Results -- Tools coverage increased to 95.26% overall -- Individual components show excellent coverage: - - 100% coverage for directory_tools, quality_tools, task_complete_tool, and test_runner - - 98.65% coverage for summarizer_tool - - 96.70% coverage for tree_tool - - 89.83% coverage for file_tools - - 87.50% coverage for base tool class - -## Why It's Needed -The code coverage had regressed due to import paths not being correctly set for coverage tracking. These changes restore and improve the coverage levels while ensuring all tests pass reliably. - -Fixes the coverage regression issues previously identified. \ No newline at end of file diff --git a/scripts/run_coverage_ci.sh b/scripts/run_coverage_ci.sh index 0ef9bd0..0368a1e 100755 --- a/scripts/run_coverage_ci.sh +++ b/scripts/run_coverage_ci.sh @@ -10,7 +10,7 @@ echo "Starting coverage generation for CI..." mkdir -p coverage_html # Set environment variables for CI -export CI_EXIT_ON_TEST_FAILURE=0 # Don't exit on test failures in CI +export CI_EXIT_ON_TEST_FAILURE=1 # Exit on test failures to ensure code quality export CI_TEST_TIMEOUT=60 # Default timeout # Special handling for GitHub Actions environment From 5b2ed7ab73d57ea0ac6834e17114ecf5217c604f Mon Sep 17 00:00:00 2001 From: "James H. Nguyen" Date: Mon, 14 Apr 2025 00:59:33 -0700 Subject: [PATCH 2/3] Fix: Improve coverage reporting in SonarCloud --- .coveragerc | 5 +++++ .github/workflows/python-ci.yml | 4 ++++ scripts/run_coverage_ci.sh | 1 + sonar-project.properties | 3 +++ 4 files changed, 13 insertions(+) diff --git a/.coveragerc b/.coveragerc index 0eabdcd..674d879 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,5 +1,9 @@ [run] source = src.cli_code +include = + */src/cli_code/*.py + */src/cli_code/*/*.py + */src/cli_code/*/*/*.py omit = */.rules/* */.venv/* @@ -10,6 +14,7 @@ omit = */venv/* */.pytest_cache/* */site-packages/* + */src/cli_code/models/ollama.py # Match the exclusion in sonar-project.properties [report] exclude_lines = diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 89526fc..27df5c9 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -142,6 +142,8 @@ jobs: -Dsonar.sourceEncoding=UTF-8 -Dsonar.verbose=true -Dsonar.scm.provider=git + -Dsonar.coverage.jacoco.xmlReportPaths=coverage.xml + -Dsonar.newCode.referenceBranch=origin/main # Add specific PR properties based on GitHub context - name: SonarCloud PR Analysis @@ -158,6 +160,8 @@ jobs: -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} -Dsonar.pullrequest.branch=${{ github.head_ref }} -Dsonar.pullrequest.base=${{ github.base_ref }} + -Dsonar.python.coverage.reportPaths=coverage.xml + -Dsonar.coverage.jacoco.xmlReportPaths=coverage.xml - name: Report SonarCloud Results run: | diff --git a/scripts/run_coverage_ci.sh b/scripts/run_coverage_ci.sh index 0368a1e..7173573 100755 --- a/scripts/run_coverage_ci.sh +++ b/scripts/run_coverage_ci.sh @@ -257,6 +257,7 @@ run_test_group "remaining" \ echo "Generating final coverage report..." | tee -a "$SUMMARY_LOG" python -m pytest \ --cov=src.cli_code \ + --cov-append \ --cov-report=xml:coverage.xml \ --cov-report=html:coverage_html \ --cov-report=term diff --git a/sonar-project.properties b/sonar-project.properties index 7e970a8..3bac4dd 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -17,6 +17,9 @@ sonar.python.coverage.reportPaths=coverage.xml # Configure test coverage exclusions sonar.coverage.exclusions=test_dir/**/*,tests/**/*,src/cli_code/models/ollama.py +# Force SonarCloud to see all files as new code to get proper coverage metrics +sonar.newCode.referenceBranch=origin/main + # Specify Python version sonar.python.version=3.11 From a5ad789d81f41cf13dd426c1e575f990475d9210 Mon Sep 17 00:00:00 2001 From: "James H. Nguyen" Date: Mon, 14 Apr 2025 01:02:56 -0700 Subject: [PATCH 3/3] Address PR review feedback: add explanatory comments and fix CI_EXIT_ON_TEST_FAILURE handling --- .coveragerc | 4 +++- scripts/run_coverage_ci.sh | 5 ++--- sonar-project.properties | 5 +++++ 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.coveragerc b/.coveragerc index 674d879..711f269 100644 --- a/.coveragerc +++ b/.coveragerc @@ -14,7 +14,9 @@ omit = */venv/* */.pytest_cache/* */site-packages/* - */src/cli_code/models/ollama.py # Match the exclusion in sonar-project.properties + # Excluded as it primarily contains integration code with external dependencies + # that is difficult to test without mocking the entire Ollama API + */src/cli_code/models/ollama.py [report] exclude_lines = diff --git a/scripts/run_coverage_ci.sh b/scripts/run_coverage_ci.sh index 7173573..6a78930 100755 --- a/scripts/run_coverage_ci.sh +++ b/scripts/run_coverage_ci.sh @@ -296,9 +296,8 @@ fi echo "Coverage generation for CI completed." | tee -a "$SUMMARY_LOG" -# Determine exit code based on errors and CI environment -# In CI we might want to exit gracefully for some failures -CI_EXIT_ON_TEST_FAILURE=${CI_EXIT_ON_TEST_FAILURE:-1} +# Use the CI_EXIT_ON_TEST_FAILURE value set at the beginning of the script (=1) +# to determine whether to exit with an error code on test failures if [ $FAILED_TESTS -gt 0 -o $TIMED_OUT_TESTS -gt 0 ]; then echo "Test run had $FAILED_TESTS failing tests and $TIMED_OUT_TESTS timed out tests" | tee -a "$SUMMARY_LOG" diff --git a/sonar-project.properties b/sonar-project.properties index 3bac4dd..92a1a87 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -15,9 +15,14 @@ sonar.tests=test_dir sonar.python.coverage.reportPaths=coverage.xml # Configure test coverage exclusions +# ollama.py is excluded as it primarily contains integration code with external dependencies +# that is difficult to test without mocking the entire Ollama API sonar.coverage.exclusions=test_dir/**/*,tests/**/*,src/cli_code/models/ollama.py # Force SonarCloud to see all files as new code to get proper coverage metrics +# Note: We're using this temporarily to establish accurate baseline coverage +# TODO: Remove this line after initial coverage baseline is established to ensure +# future PRs are evaluated correctly for new code coverage sonar.newCode.referenceBranch=origin/main # Specify Python version