move_bad_runs.py

Go to the documentation of this file.

#!/usr/bin/env python3
 
import os
import shutil
import re
from collections import defaultdict
 
# Configuration
THRESHOLD = 5 * 1024 * 1024 * 1024  # bytes
BAD_DIR = "beam-bad"
 
# Create the bad directory if it does not exist
if not os.path.exists(BAD_DIR):
    os.makedirs(BAD_DIR)
    print(f"Created directory: {BAD_DIR}")
 
# Regex pattern for the filename format:
# run_<label>_<run_number>_ev<digits>_p<digits>.data
# The label can contain underscores or other characters.
pattern = r'^run_(.*)_(\d+)_ev\d+_p\d+\.data$'
 
# Collect all potential data files
data_files = [
    f for f in os.listdir('.')
    if os.path.isfile(f) and f.startswith('run_') and f.endswith('.data')
]
 
# Group files by run_number
run_groups = defaultdict(list)
for f in data_files:
    match = re.match(pattern, f)
    if match:
        label, run_num_str = match.groups()
        run_num = int(run_num_str)
        run_groups[run_num].append(f)
    else:
        print(f"Skipping file: {f} (does not match expected pattern)")
 
# Process each run
moved_count = 0
for run_num, files in sorted(run_groups.items()):
    if not files:
        continue
 
    # Calculate total size in bytes
    total_size = sum(os.path.getsize(f) for f in files)
    num_files = len(files)
 
    if total_size < THRESHOLD:
        print(f"Bad run {run_num}: {num_files} files, {total_size:,} bytes (< {THRESHOLD:,} bytes) → moving to {BAD_DIR}/")
        for f in files:
            shutil.move(f, os.path.join(BAD_DIR, f))
            moved_count += 1
    else:
        print(f"Good run {run_num}: {num_files} files, {total_size:,} bytes")
 
print(f"\nProcessing complete. Moved {moved_count} files to '{BAD_DIR}'.")