From 59d972fc341577d9f29c965a08763190bcd17236 Mon Sep 17 00:00:00 2001 From: yinjie <2542374168@qq.com> Date: Thu, 31 Jul 2025 00:33:46 +0800 Subject: [PATCH 1/4] fix: ensure set pre record first & log level in con_inout --- traincheck/invariant/consistency_transient_vars.py | 2 +- traincheck/onlinechecker/streamhandler_filesystem.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/traincheck/invariant/consistency_transient_vars.py b/traincheck/invariant/consistency_transient_vars.py index 570ed42f..b0640dff 100644 --- a/traincheck/invariant/consistency_transient_vars.py +++ b/traincheck/invariant/consistency_transient_vars.py @@ -1034,7 +1034,7 @@ def online_check( if input_value != output_value: check_passed = False except (IndexError, KeyError): - logger.warning( + logger.debug( f"Could not find the value to be checked in input or output tensors for the hypothesis {inv}, skipping this function call." ) diff --git a/traincheck/onlinechecker/streamhandler_filesystem.py b/traincheck/onlinechecker/streamhandler_filesystem.py index 310fc70e..1419daf5 100644 --- a/traincheck/onlinechecker/streamhandler_filesystem.py +++ b/traincheck/onlinechecker/streamhandler_filesystem.py @@ -184,6 +184,7 @@ def _set_func_map(self, trace_record): self.pt_map[ptname][func_call_id].args = trace_record["args"] self.pt_map[ptname][func_call_id].kwargs = trace_record["kwargs"] elif trace_type == TraceLineType.FUNC_CALL_POST: + assert self.pt_map[ptname][func_call_id].pre_record is not None self.pt_map[ptname][func_call_id].post_record = trace_record self.pt_map[ptname][func_call_id].return_values = trace_record[ "return_values" From 0eee11c3228b3542fd7efdcc539cddcd01cd27ac Mon Sep 17 00:00:00 2001 From: yinjie <2542374168@qq.com> Date: Thu, 31 Jul 2025 00:35:04 +0800 Subject: [PATCH 2/4] feat: CI/CD for correctness_check --- .github/workflows/correctness_checks.yml | 52 ++++++++++++++++++++++++ README.md | 1 + 2 files changed, 53 insertions(+) create mode 100644 .github/workflows/correctness_checks.yml diff --git a/.github/workflows/correctness_checks.yml b/.github/workflows/correctness_checks.yml new file mode 100644 index 00000000..3574fca3 --- /dev/null +++ b/.github/workflows/correctness_checks.yml @@ -0,0 +1,52 @@ +name: Correctness Checks +on: + push: + branches: + - main + pull_request: + paths: + - '.github/workflows/**' + - 'traincheck/invariant/**' + - 'traincheck/onlinechecker/**' + - 'traincheck/checker_online.py' + - 'traincheck/checker.py' + +permissions: + contents: write + deployments: write + pull-requests: write + +jobs: + correctness-check: + runs-on: self-hosted + + name: Run Correctness Checks + steps: + - uses: actions/checkout@v4 + + - uses: actions/checkout@v4 + name: Checkout TrainCheck-Benchmarks + with: + repository: OrderLab/TrainCheck-Benchmarks + path: benchmarks + lfs: true + + - uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + + - name: Run correctness script + run: | + cd benchmarks/correctness_check + python3 correct_check.py + + - name: Clear check files + run: | + cd benchmarks/correctness_check + rm -rf trace_* + rm -rf traincheck_* diff --git a/README.md b/README.md index 30c8ffad..cc48a52c 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@

TrainCheck: Training with Confidence

[![format and types](https://github.com/OrderLab/traincheck/actions/workflows/pre-commit-checks.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/pre-commit-checks.yml) +[![format and types](https://github.com/OrderLab/traincheck/actions/workflows/correctness_check.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/correctness_check.yml) [![Chat on Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?logo=discord&logoColor=white)](https://discord.gg/ZvYewjsQ9D) From 9bdee18280e0295bf53b7c13b0e7ba76b03057ac Mon Sep 17 00:00:00 2001 From: yinjie <2542374168@qq.com> Date: Thu, 31 Jul 2025 21:36:26 +0800 Subject: [PATCH 3/4] fix: typos in readme for badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cc48a52c..6812c1bb 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@

TrainCheck: Training with Confidence

[![format and types](https://github.com/OrderLab/traincheck/actions/workflows/pre-commit-checks.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/pre-commit-checks.yml) -[![format and types](https://github.com/OrderLab/traincheck/actions/workflows/correctness_check.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/correctness_check.yml) +[![format and types](https://github.com/OrderLab/traincheck/actions/workflows/correctness_checks.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/correctness_checks.yml) [![Chat on Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?logo=discord&logoColor=white)](https://discord.gg/ZvYewjsQ9D) From 4ddfe221ffc5063f3e45c4ab5fcf84555239e65c Mon Sep 17 00:00:00 2001 From: yinjie <2542374168@qq.com> Date: Fri, 1 Aug 2025 11:09:54 +0800 Subject: [PATCH 4/4] feat: sort the tracefile --- traincheck/onlinechecker/streamhandler_filesystem.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/traincheck/onlinechecker/streamhandler_filesystem.py b/traincheck/onlinechecker/streamhandler_filesystem.py index 1419daf5..2b69c5cc 100644 --- a/traincheck/onlinechecker/streamhandler_filesystem.py +++ b/traincheck/onlinechecker/streamhandler_filesystem.py @@ -325,7 +325,7 @@ def run_stream_monitor(traces, trace_folders, checker_data: Checker_data): if trace_folders is not None: for trace_folder in trace_folders: - for file in os.listdir(trace_folder): + for file in sorted(os.listdir(trace_folder)): if file.startswith("trace_") or file.endswith("proxy_log.json"): file_path = os.path.join(trace_folder, file) handler = StreamLogHandler(file_path, checker_data)