Correctness check

.github/workflows/correctness_checks.yml

@@ -0,0 +1,52 @@
+name: Correctness Checks
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    paths:
+      - '.github/workflows/**'
+      - 'traincheck/invariant/**'
+      - 'traincheck/onlinechecker/**'
+      - 'traincheck/checker_online.py'
+      - 'traincheck/checker.py'
+
+permissions:
+  contents: write
+  deployments: write
+  pull-requests: write
+
+jobs:
+  correctness-check:
+    runs-on: self-hosted
+
+    name: Run Correctness Checks
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/checkout@v4
+        name: Checkout TrainCheck-Benchmarks
+        with:
+          repository: OrderLab/TrainCheck-Benchmarks
+          path: benchmarks
+          lfs: true
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .
+
+      - name: Run correctness script
+        run: |
+          cd benchmarks/correctness_check
+          python3 correct_check.py
+
+      - name: Clear check files
+        run: |
+          cd benchmarks/correctness_check
+          rm -rf trace_*
+          rm -rf traincheck_*

README.md

@@ -5,6 +5,7 @@
 <h1>TrainCheck: Training with Confidence</h1>
 
 [![format and types](https://github.com/OrderLab/traincheck/actions/workflows/pre-commit-checks.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/pre-commit-checks.yml)
+[![format and types](https://github.com/OrderLab/traincheck/actions/workflows/correctness_checks.yml/badge.svg)](https://github.com/OrderLab/traincheck/actions/workflows/correctness_checks.yml)
 [![Chat on Discord](https://img.shields.io/badge/Discord-Join%20us-5865F2?logo=discord&logoColor=white)](https://discord.gg/ZvYewjsQ9D)
 
 </div>

traincheck/invariant/consistency_transient_vars.py

@@ -1034,7 +1034,7 @@ def online_check(
                 if input_value != output_value:
                     check_passed = False
             except (IndexError, KeyError):
-                logger.warning(
+                logger.debug(
                     f"Could not find the value to be checked in input or output tensors for the hypothesis {inv}, skipping this function call."
                 )
 

traincheck/onlinechecker/streamhandler_filesystem.py

@@ -184,6 +184,7 @@ def _set_func_map(self, trace_record):
                     self.pt_map[ptname][func_call_id].args = trace_record["args"]
                     self.pt_map[ptname][func_call_id].kwargs = trace_record["kwargs"]
                 elif trace_type == TraceLineType.FUNC_CALL_POST:
+                    assert self.pt_map[ptname][func_call_id].pre_record is not None
                     self.pt_map[ptname][func_call_id].post_record = trace_record
                     self.pt_map[ptname][func_call_id].return_values = trace_record[
                         "return_values"
@@ -324,7 +325,7 @@ def run_stream_monitor(traces, trace_folders, checker_data: Checker_data):
 
     if trace_folders is not None:
         for trace_folder in trace_folders:
-            for file in os.listdir(trace_folder):
+            for file in sorted(os.listdir(trace_folder)):
                 if file.startswith("trace_") or file.endswith("proxy_log.json"):
                     file_path = os.path.join(trace_folder, file)
                     handler = StreamLogHandler(file_path, checker_data)