@@ -536,8 +536,8 @@ def analyze_local_path_sync(
536536 total_files = len (file_paths )
537537 logger .info (f"Found { total_files } files to process" )
538538
539- # Thread-safe counter for progress tracking : [count , lock]
540- processed_count = [0 , threading .Lock ()]
539+ # Thread-safe counters : [submitted_count, completed_count , lock]
540+ counters = [0 , 0 , threading .Lock ()]
541541
542542 # Process files in chunks to avoid too many futures at once.
543543 CHUNK_SUBMIT = 256
@@ -546,9 +546,9 @@ def analyze_local_path_sync(
546546 futures = []
547547 for f in chunk :
548548 # Increment counter before starting file processing
549- with processed_count [ 1 ]:
550- processed_count [0 ] += 1
551- file_num = processed_count [0 ]
549+ with counters [ 2 ]:
550+ counters [0 ] += 1
551+ file_num = counters [0 ]
552552
553553 fut = _EXECUTOR .submit (
554554 _process_file_sync ,
@@ -567,6 +567,12 @@ def analyze_local_path_sync(
567567 try :
568568 r = fut .result ()
569569
570+ # Increment completed counter and check for periodic logging
571+ with counters [2 ]:
572+ counters [1 ] += 1
573+ completed_count = counters [1 ]
574+ should_log = completed_count % 10 == 0
575+
570576 if isinstance (r , dict ):
571577 if r .get ("stored" ):
572578 file_count += 1
@@ -576,10 +582,8 @@ def analyze_local_path_sync(
576582 skipped_count += 1
577583
578584 # Log periodic progress updates (every 10 files)
579- with processed_count [1 ]:
580- current_processed = processed_count [0 ]
581- if current_processed % 10 == 0 :
582- logger .info (f"Progress: { current_processed } /{ total_files } files processed ({ file_count } stored, { emb_count } with embeddings, { skipped_count } skipped)" )
585+ if should_log :
586+ logger .info (f"Progress: { completed_count } /{ total_files } files processed ({ file_count } stored, { emb_count } with embeddings, { skipped_count } skipped)" )
583587 except Exception :
584588 logger .exception ("A per-file task failed" )
585589
0 commit comments