Skip to content

Commit 75acf3f

Browse files
CopilotMte90
andcommitted
Fix race condition in periodic progress logging
Co-authored-by: Mte90 <403283+Mte90@users.noreply.github.com>
1 parent 77e3a4f commit 75acf3f

File tree

1 file changed

+13
-9
lines changed

1 file changed

+13
-9
lines changed

ai/analyzer.py

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -536,8 +536,8 @@ def analyze_local_path_sync(
536536
total_files = len(file_paths)
537537
logger.info(f"Found {total_files} files to process")
538538

539-
# Thread-safe counter for progress tracking: [count, lock]
540-
processed_count = [0, threading.Lock()]
539+
# Thread-safe counters: [submitted_count, completed_count, lock]
540+
counters = [0, 0, threading.Lock()]
541541

542542
# Process files in chunks to avoid too many futures at once.
543543
CHUNK_SUBMIT = 256
@@ -546,9 +546,9 @@ def analyze_local_path_sync(
546546
futures = []
547547
for f in chunk:
548548
# Increment counter before starting file processing
549-
with processed_count[1]:
550-
processed_count[0] += 1
551-
file_num = processed_count[0]
549+
with counters[2]:
550+
counters[0] += 1
551+
file_num = counters[0]
552552

553553
fut = _EXECUTOR.submit(
554554
_process_file_sync,
@@ -567,6 +567,12 @@ def analyze_local_path_sync(
567567
try:
568568
r = fut.result()
569569

570+
# Increment completed counter and check for periodic logging
571+
with counters[2]:
572+
counters[1] += 1
573+
completed_count = counters[1]
574+
should_log = completed_count % 10 == 0
575+
570576
if isinstance(r, dict):
571577
if r.get("stored"):
572578
file_count += 1
@@ -576,10 +582,8 @@ def analyze_local_path_sync(
576582
skipped_count += 1
577583

578584
# Log periodic progress updates (every 10 files)
579-
with processed_count[1]:
580-
current_processed = processed_count[0]
581-
if current_processed % 10 == 0:
582-
logger.info(f"Progress: {current_processed}/{total_files} files processed ({file_count} stored, {emb_count} with embeddings, {skipped_count} skipped)")
585+
if should_log:
586+
logger.info(f"Progress: {completed_count}/{total_files} files processed ({file_count} stored, {emb_count} with embeddings, {skipped_count} skipped)")
583587
except Exception:
584588
logger.exception("A per-file task failed")
585589

0 commit comments

Comments
 (0)