From b0804cd188ba7ec811ee83941063478fe46950e8 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 19 Dec 2025 15:21:29 +0100 Subject: [PATCH 1/6] Make GitHub Actions utilities work locally Adds fallback behavior to src/tools/github.ts for local development: - error/warning/notice functions now print to console when not in GHA - withGroup/withGroupAsync print titles directly instead of using workflow commands --- src/tools/github.ts | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/src/tools/github.ts b/src/tools/github.ts index f2d0e6e59a..947bbdc8cd 100644 --- a/src/tools/github.ts +++ b/src/tools/github.ts @@ -61,7 +61,10 @@ export function error( message: string, properties?: AnnotationProperties, ): void { - if (!isGitHubActions()) return; + if (!isGitHubActions()) { + console.log(message); + return; + } const props = properties ? formatProperties(properties) : ""; console.log(`::error${props}::${escapeData(message)}`); } @@ -70,7 +73,10 @@ export function warning( message: string, properties?: AnnotationProperties, ): void { - if (!isGitHubActions()) return; + if (!isGitHubActions()) { + console.log(message); + return; + } const props = properties ? formatProperties(properties) : ""; console.log(`::warning${props}::${escapeData(message)}`); } @@ -79,7 +85,10 @@ export function notice( message: string, properties?: AnnotationProperties, ): void { - if (!isGitHubActions()) return; + if (!isGitHubActions()) { + console.log(message); + return; + } const props = properties ? formatProperties(properties) : ""; console.log(`::notice${props}::${escapeData(message)}`); } @@ -96,6 +105,10 @@ export function endGroup(): void { } export function withGroup(title: string, fn: () => T): T { + if (!isGitHubActions()) { + console.log(title); + return fn(); + } startGroup(title); try { return fn(); @@ -108,6 +121,10 @@ export async function withGroupAsync( title: string, fn: () => Promise, ): Promise { + if (!isGitHubActions()) { + console.log(title); + return await fn(); + } startGroup(title); try { return await fn(); From 39c6578f4a962a9f0c2dae4fa63877a96e5f769d Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 19 Dec 2025 15:22:38 +0100 Subject: [PATCH 2/6] Configure conditional DEBUG logging for GitHub Actions Adds automatic DEBUG level logging when RUNNER_DEBUG=1 is set in GitHub Actions environment. Can be overridden by explicit --log-level or QUARTO_LOG_LEVEL environment variable. --- src/core/log.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/core/log.ts b/src/core/log.ts index 7c6199930e..89ace56f35 100644 --- a/src/core/log.ts +++ b/src/core/log.ts @@ -22,6 +22,7 @@ import { onCleanup } from "./cleanup.ts"; import { execProcess } from "./process.ts"; import { pandocBinaryPath } from "./resources.ts"; import { Block, pandoc } from "./pandoc/json.ts"; +import { isGitHubActions, isVerboseMode } from "../tools/github.ts"; export type LogLevel = "DEBUG" | "INFO" | "WARN" | "ERROR" | "CRITICAL"; export type LogFormat = "plain" | "json-stream"; @@ -99,8 +100,17 @@ export function logOptions(args: Args) { if (logOptions.log) { ensureDirSync(dirname(logOptions.log)); } - logOptions.level = args.ll || args["log-level"] || - Deno.env.get("QUARTO_LOG_LEVEL"); + + // Determine log level with priority: explicit args > QUARTO_LOG_LEVEL > GitHub Actions debug mode + let level = args.ll || args["log-level"] || Deno.env.get("QUARTO_LOG_LEVEL"); + + // Enable DEBUG logging when GitHub Actions debug mode is on (RUNNER_DEBUG=1) + // Can be overridden by explicit --log-level or QUARTO_LOG_LEVEL + if (!level && isGitHubActions() && isVerboseMode()) { + level = "DEBUG"; + } + + logOptions.level = level; logOptions.quiet = args.q || args.quiet; logOptions.format = parseFormat( args.lf || args["log-format"] || Deno.env.get("QUARTO_LOG_FORMAT"), From c4f9638eb4d3a56212872bcb14303c542fc5fbe6 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 19 Dec 2025 15:23:25 +0100 Subject: [PATCH 3/6] Improve playwright test infrastructure - Suppress webServer output (stderr: 'ignore' in playwright.config.ts) - Add respectStreams: true to execProcess for clearer test output - Add environment variables for test flexibility (SKIP_RENDER, SKIP_CLEANOUTPUT) - Improve document rendering logging with relative paths --- tests/integration/playwright-tests.test.ts | 85 ++++++++++++------- .../playwright/playwright.config.ts | 2 + 2 files changed, 57 insertions(+), 30 deletions(-) diff --git a/tests/integration/playwright-tests.test.ts b/tests/integration/playwright-tests.test.ts index 9aba61ac97..4318d9836c 100644 --- a/tests/integration/playwright-tests.test.ts +++ b/tests/integration/playwright-tests.test.ts @@ -16,7 +16,7 @@ import { execProcess } from "../../src/core/process.ts"; import { quartoDevCmd } from "../utils.ts"; import { fail } from "testing/asserts"; import { isWindows } from "../../src/deno_ral/platform.ts"; -import { join } from "../../src/deno_ral/path.ts"; +import { join, relative } from "../../src/deno_ral/path.ts"; import { existsSync } from "../../src/deno_ral/fs.ts"; import * as gha from "../../src/tools/github.ts"; @@ -37,7 +37,6 @@ await initState(); const multiplexServerPath = "integration/playwright/multiplex-server"; const multiplexNodeModules = join(multiplexServerPath, "node_modules"); if (!existsSync(multiplexNodeModules)) { - console.log("Installing multiplex server dependencies..."); await execProcess({ cmd: isWindows ? "npm.cmd" : "npm", args: ["install", "--loglevel=error"], @@ -48,44 +47,66 @@ if (!existsSync(multiplexNodeModules)) { // const promises = []; const fileNames: string[] = []; -const extraOpts = [ - { - pathSuffix: "docs/playwright/embed-resources/issue-11860/main.qmd", - options: ["--output-dir=inner"], - } -] -for (const { path: fileName } of globOutput) { - const input = fileName; - const options: string[] = []; - for (const extraOpt of extraOpts) { - if (fileName.endsWith(extraOpt.pathSuffix)) { - options.push(...extraOpt.options); +// To avoid re-rendering, see QUARTO_PLAYWRIGHT_SKIP_RENDER env var +if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { + console.log("Skipping render of test documents."); +} else { + const extraOpts = [ + { + pathSuffix: "docs/playwright/embed-resources/issue-11860/main.qmd", + options: ["--output-dir=inner"], } - } + ] - // sigh, we have a race condition somewhere in - // mediabag inspection if we don't wait all renders - // individually. This is very slow.. - await execProcess({ - cmd: quartoDevCmd(), - args: ["render", input, ...options], - }); - fileNames.push(fileName); + for (const { path: fileName } of globOutput) { + const input = relative(Deno.cwd(), fileName); + const options: string[] = []; + for (const extraOpt of extraOpts) { + if (fileName.endsWith(extraOpt.pathSuffix)) { + options.push(...extraOpt.options); + } + } + + // sigh, we have a race condition somewhere in + // mediabag inspection if we don't wait all renders + // individually. This is very slow.. + console.log(`Rendering ${input}...`); + const result = await execProcess({ + cmd: quartoDevCmd(), + args: ["render", input, ...options], + stdout: "piped", + stderr: "piped", + }); + + if (!result.success) { + gha.error(`Failed to render ${input}`) + if (result.stdout) console.log(result.stdout); + if (result.stderr) console.error(result.stderr); + throw new Error(`Render failed with code ${result.code}`); + } + + fileNames.push(fileName); + } } Deno.test({ name: "Playwright tests are passing", // currently we run playwright tests only on Linux - ignore: isWindows, + ignore: gha.isGitHubActions() && isWindows, fn: async () => { try { // run playwright const res = await execProcess({ - cmd: isWindows ? "npx.cmd" : "npx", + cmd: isWindows ? "npx.cmd" : "npx", args: ["playwright", "test", "--ignore-snapshots"], cwd: "integration/playwright", - }); + }, + undefined, // stdin + undefined, // mergeOutput + undefined, // stderrFilter + true // respectStreams - write directly to stderr/stdout + ); if (!res.success) { if (gha.isGitHubActions() && Deno.env.get("GITHUB_REPOSITORY") && Deno.env.get("GITHUB_RUN_ID")) { const runUrl = `https://github.com/${Deno.env.get("GITHUB_REPOSITORY")}/actions/runs/${Deno.env.get("GITHUB_RUN_ID")}`; @@ -99,11 +120,15 @@ Deno.test({ } fail("Failed tests with playwright. Look at playwright report for more details.") } - + } finally { - for (const fileName of fileNames) { - cleanoutput(fileName, "html"); - } + // skip cleanoutput if requested + if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_CLEANOUTPUT") === "true" || Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { + console.log("Skipping cleanoutput of test documents."); + } else + for (const fileName of fileNames) { + cleanoutput(fileName, "html"); + } } } }); diff --git a/tests/integration/playwright/playwright.config.ts b/tests/integration/playwright/playwright.config.ts index d300619dd1..07ec7198e6 100644 --- a/tests/integration/playwright/playwright.config.ts +++ b/tests/integration/playwright/playwright.config.ts @@ -101,6 +101,7 @@ export default defineConfig({ url: 'http://127.0.0.1:8080', reuseExistingServer: !isCI, cwd: '../../docs/playwright', + stderr: 'ignore', // Suppress verbose HTTP request logs }, { // Socket.IO multiplex server for RevealJS @@ -109,6 +110,7 @@ export default defineConfig({ reuseExistingServer: !isCI, cwd: './multiplex-server', timeout: 10000, + stderr: 'ignore', // Suppress verbose logs } ], }); From 6e49fede29178d25fee5ea79cdd4a652f16725f4 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 19 Dec 2025 15:24:03 +0100 Subject: [PATCH 4/6] Add workflow-level grouping for test execution - Add ::group::/::endgroup:: around test file execution in test-smokes.yml - Remove QUARTO_LOG_LEVEL: DEBUG from workflow - Remove unnecessary echo statements from run-tests.sh GitHub Actions doesn't support nested groups, so only workflow-level grouping is used. --- .github/workflows/test-smokes.yml | 5 ++++- tests/run-tests.sh | 2 -- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index 4112a1ec6b..2b25801773 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -264,18 +264,19 @@ jobs: env: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - QUARTO_LOG_LEVEL: DEBUG run: | haserror=0 failed_tests=() readarray -t my_array < <(echo '${{ inputs.buckets }}' | jq -rc '.[]') for file in "${my_array[@]}"; do + echo "::group::Testing ${file}" echo ">>> ./run-tests.sh ${file}" # Run tests without -e so we don't exit on first failure set +e shopt -s globstar && ./run-tests.sh "$file" status=$? set -e + echo "::endgroup::" if [ $status -ne 0 ]; then echo "::error title=Test Bucket Failed::Test bucket ${file} failed with exit code ${status}" echo ">>> Error found in test file: ${file}" @@ -305,9 +306,11 @@ jobs: $haserror=$false $failed_tests=@() foreach ($file in ('${{ inputs.buckets }}' | ConvertFrom-Json)) { + Write-Host "::group::Testing ${file}" Write-Host ">>> ./run-tests.ps1 ${file}" ./run-tests.ps1 $file $status=$LASTEXITCODE + Write-Host "::endgroup::" if ($status -ne 0) { Write-Host "::error title=Test Bucket Failed::Test bucket ${file} failed with exit code ${status}" Write-Host ">>> Error found in test file: ${file}" diff --git a/tests/run-tests.sh b/tests/run-tests.sh index 597b67953a..bb0e79e5e4 100755 --- a/tests/run-tests.sh +++ b/tests/run-tests.sh @@ -106,14 +106,12 @@ else TESTS_TO_RUN="" if [[ ! -z "$*" ]]; then for file in "$*"; do - echo $file filename=$(basename "$file") # smoke-all.test.ts works with .qmd, .md and .ipynb but will ignored file starting with _ if [[ $filename =~ ^[^_].*[.]qmd$ ]] || [[ $filename =~ ^[^_].*[.]ipynb$ ]] || [[ $filename =~ ^[^_].*[.]md$ ]]; then SMOKE_ALL_FILES="${SMOKE_ALL_FILES} ${file}" elif [[ $file =~ .*[.]ts$ ]]; then TESTS_TO_RUN="${TESTS_TO_RUN} ${file}" - echo $TESTS_TO_RUN else echo "#### WARNING" echo "Only .ts, or .qmd, .md and .ipynb passed to smoke-all.test.ts are accepted (file starting with _ are ignored)." From 6d679940a578a44a5aea6c7a084c2994b10dd259 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 19 Dec 2025 15:24:25 +0100 Subject: [PATCH 5/6] Add temporary failing tests to verify CI log grouping Adds three intentionally failing tests to validate that grouping improvements work correctly in CI logs: - Playwright test (tests/integration/playwright/tests/) - Unit test (tests/unit/) - Smoke-all document (tests/docs/smoke-all/) These tests should be removed after verifying CI log behavior. --- tests/docs/smoke-all/temporary-failing-test.qmd | 14 ++++++++++++++ .../tests/temporary-failing-test.spec.ts | 8 ++++++++ tests/unit/temporary-failing-test.test.ts | 15 +++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 tests/docs/smoke-all/temporary-failing-test.qmd create mode 100644 tests/integration/playwright/tests/temporary-failing-test.spec.ts create mode 100644 tests/unit/temporary-failing-test.test.ts diff --git a/tests/docs/smoke-all/temporary-failing-test.qmd b/tests/docs/smoke-all/temporary-failing-test.qmd new file mode 100644 index 0000000000..c4ba89bb99 --- /dev/null +++ b/tests/docs/smoke-all/temporary-failing-test.qmd @@ -0,0 +1,14 @@ +--- +format: html +title: Temporary Failing Test +_quarto: + tests: + html: + ensureFileRegexMatches: + - ["THIS_TEXT_WILL_NEVER_EXIST_IN_OUTPUT"] +--- + +TEMPORARY FAILING TEST - FOR CI LOG TESTING ONLY + +This document is intentionally failing to verify grouping behavior in CI logs. +The test expects text that doesn't exist in the output, causing a failure. diff --git a/tests/integration/playwright/tests/temporary-failing-test.spec.ts b/tests/integration/playwright/tests/temporary-failing-test.spec.ts new file mode 100644 index 0000000000..252b2d2d33 --- /dev/null +++ b/tests/integration/playwright/tests/temporary-failing-test.spec.ts @@ -0,0 +1,8 @@ +import { expect, test } from "@playwright/test"; + +// TEMPORARY FAILING TEST - FOR CI LOG TESTING ONLY +// This test is intentionally failing to verify grouping behavior in CI logs +test("temporary failing test for CI verification", async ({ page }) => { + // This will always fail + expect(true).toBe(false); +}); diff --git a/tests/unit/temporary-failing-test.test.ts b/tests/unit/temporary-failing-test.test.ts new file mode 100644 index 0000000000..2b43d84620 --- /dev/null +++ b/tests/unit/temporary-failing-test.test.ts @@ -0,0 +1,15 @@ +/* +* __temporary-failing-test.test.ts +* +* TEMPORARY FAILING TEST - FOR CI LOG TESTING ONLY +* This test is intentionally failing to verify grouping behavior in CI logs +* +*/ + +import { unitTest } from "../test.ts"; +import { assert } from "testing/asserts"; + +unitTest("temporary failing test for CI verification", async () => { + // This will always fail + assert(false, "This is a temporary failing test to verify CI log grouping"); +}); From f664f1c771968d67770d1b10e06cf9d80aafe440 Mon Sep 17 00:00:00 2001 From: Christophe Dervieux Date: Fri, 19 Dec 2025 15:58:39 +0100 Subject: [PATCH 6/6] Revert "Add temporary failing tests to verify CI log grouping" This reverts commit 85ffd0d5be6fb53da2e2a43175e8e90463800bd2. --- tests/docs/smoke-all/temporary-failing-test.qmd | 14 -------------- .../tests/temporary-failing-test.spec.ts | 8 -------- tests/unit/temporary-failing-test.test.ts | 15 --------------- 3 files changed, 37 deletions(-) delete mode 100644 tests/docs/smoke-all/temporary-failing-test.qmd delete mode 100644 tests/integration/playwright/tests/temporary-failing-test.spec.ts delete mode 100644 tests/unit/temporary-failing-test.test.ts diff --git a/tests/docs/smoke-all/temporary-failing-test.qmd b/tests/docs/smoke-all/temporary-failing-test.qmd deleted file mode 100644 index c4ba89bb99..0000000000 --- a/tests/docs/smoke-all/temporary-failing-test.qmd +++ /dev/null @@ -1,14 +0,0 @@ ---- -format: html -title: Temporary Failing Test -_quarto: - tests: - html: - ensureFileRegexMatches: - - ["THIS_TEXT_WILL_NEVER_EXIST_IN_OUTPUT"] ---- - -TEMPORARY FAILING TEST - FOR CI LOG TESTING ONLY - -This document is intentionally failing to verify grouping behavior in CI logs. -The test expects text that doesn't exist in the output, causing a failure. diff --git a/tests/integration/playwright/tests/temporary-failing-test.spec.ts b/tests/integration/playwright/tests/temporary-failing-test.spec.ts deleted file mode 100644 index 252b2d2d33..0000000000 --- a/tests/integration/playwright/tests/temporary-failing-test.spec.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { expect, test } from "@playwright/test"; - -// TEMPORARY FAILING TEST - FOR CI LOG TESTING ONLY -// This test is intentionally failing to verify grouping behavior in CI logs -test("temporary failing test for CI verification", async ({ page }) => { - // This will always fail - expect(true).toBe(false); -}); diff --git a/tests/unit/temporary-failing-test.test.ts b/tests/unit/temporary-failing-test.test.ts deleted file mode 100644 index 2b43d84620..0000000000 --- a/tests/unit/temporary-failing-test.test.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* -* __temporary-failing-test.test.ts -* -* TEMPORARY FAILING TEST - FOR CI LOG TESTING ONLY -* This test is intentionally failing to verify grouping behavior in CI logs -* -*/ - -import { unitTest } from "../test.ts"; -import { assert } from "testing/asserts"; - -unitTest("temporary failing test for CI verification", async () => { - // This will always fail - assert(false, "This is a temporary failing test to verify CI log grouping"); -});