remove_raw_files.C

Go to the documentation of this file.

#include "../../../../services/function_set.h"
 
// Remove raw file metadata from UniConDa for the files listed in the input text file,
// as well as run metadata if no raw file metadata remains for the run.
// All files in the list must belong to the same period, otherwise the macro exits with an error.
// Parameters:
//   list_file_path - path to a text file with raw file path per line (empty lines are skipped)
//   isTest         - if true (default), only print what would be removed without changing the Database;
//                    set to false to actually delete the raw file and run metadata
// Returns 0 on success, or a negative code on error (cannot read files / mixed periods).
int remove_raw_files(TString list_file_path, bool isTest = true)
{
    gSystem->ExpandPathName(list_file_path);
 
    list<string> file_list;
    ifstream list_file(list_file_path.Data());
    string s;
    while (getline(list_file, s)) {
        // Skip empty lines or lines with only whitespace
        if (s.find_first_not_of(" \t\r\n") == string::npos)
            continue;
        file_list.push_back(s);
    }
    if (file_list.empty()) {
        cout << "WARNING: There are no files in the list of the input text file: " << list_file_path << endl;
        return 0;
    }
    cout << "Total file records in the input text file: " << file_list.size() << endl << endl;
 
    int period_number = -1, removed_raw_files = 0, removed_runs = 0;
    unordered_set<int> run_list;
    // cycle for all files in the list of the input text file
    for (auto const& cur_file_path : file_list) {
        string file_name = get_file_name_with_ext(cur_file_path);
        // cout << "Current file of the list: " << cur_file_path << endl;
        cout << "Current file of the list: " << file_name << endl;
 
        TObjArray* raw_files = UniRawFile::GetRawFiles(file_name);
        if (raw_files == nullptr)
            return -1;
 
        // remove raw file metadata for the found files
        for (int i = 0; i < raw_files->GetEntries(); i++) {
            UniRawFile* curRawFile = (UniRawFile*)raw_files->At(i);
 
            if (period_number == -1)
                period_number = curRawFile->GetPeriodNumber();
            else {
                if (period_number != curRawFile->GetPeriodNumber()) {
                    cout << "ERROR: there are different period numbers for the files (" << period_number << "<>"
                         << curRawFile->GetPeriodNumber() << "), exiting...";
                    return -2;
                }
            }
            run_list.insert(curRawFile->GetRunNumber());
 
            cout << "Removing raw file: " << curRawFile->GetFilePath() << endl;
            removed_raw_files++;
            if (!isTest)
                UniRawFile::DeleteRawFile(curRawFile->GetFileId());
        }
    }
 
    // remove run metadata if no raw file metadata remains for the run
    for (int current_run : run_list) {
        cout << "Current run: " << current_run << endl;
        TObjArray* raw_files = UniRawFile::GetRawFiles(period_number, current_run);
 
        if ((raw_files != nullptr) and (raw_files->GetEntries() == 0)) {
            cout << "Removing period:run = " << period_number << ":" << current_run << endl;
            removed_runs++;
            if (!isTest)
                UniRun::DeleteRun(period_number, current_run);
        }
    }
 
    cout << endl
         << (isTest ? "Would remove " : "Removed ") << removed_raw_files << " raw files and " << removed_runs << " runs"
         << endl;
 
    return 0;
}
 
// Remove run metadata from UniConDa for the runs listed in the input text file,
// but only for those runs that have no raw file metadata remaining.
// Parameters:
//   bad_run_file  - path to a text file with run number per line
//   period_number - period the runs belong to (default 9)
//   isTest        - if true (default), only print what would be removed without changing the Database;
//                   set to false to actually delete the run metadata
// Returns 0 on success, or a negative code on error (line is not a valid number / out of range).
int remove_runs(TString bad_run_file, int period_number = 9, bool isTest = true)
{
    gSystem->ExpandPathName(bad_run_file);
 
    list<int> run_list;
    ifstream list_file(bad_run_file.Data());
    string s;
    while (getline(list_file, s)) {
        try {
            int i_run_number = std::stoi(s);
            run_list.push_back(i_run_number);
        } catch (const std::invalid_argument&) {
            cerr << "ERROR: Line is not a valid number: \"" << s << "\"\n";
            return -1;
        } catch (const std::out_of_range&) {
            cerr << "Warning: Line " << s << ": number is out of range\n";
            return -2;
        }
    }
    if (run_list.empty()) {
        cout << "WARNING: There are no runs in the list of the input text file: " << bad_run_file << endl;
        return 0;
    }
    cout << "Total run records in the input text file: " << run_list.size() << endl << endl;
 
    int removed_runs = 0;
    // remove run metadata if no raw file metadata remains for the run
    for (int current_run : run_list) {
        cout << "Current run: " << current_run << endl;
        if (UniRun::CheckRunExists(period_number, current_run) < 1)
            continue;
 
        TObjArray* raw_files = UniRawFile::GetRawFiles(period_number, current_run);
        if ((raw_files != nullptr) and (raw_files->GetEntries() == 0)) {
            cout << "Removing period:run = " << period_number << ":" << current_run << endl;
            removed_runs++;
            if (!isTest)
                UniRun::DeleteRun(period_number, current_run);
        }
    }
 
    cout << endl << (isTest ? "Would remove " : "Removed ") << removed_runs << " runs" << endl;
 
    return 0;
}
 
// Find odd run metadata in UniConDa - runs present in the Database but absent in the input text file -
// and remove those that have no raw file metadata remaining. Also reports runs from the list that are
// missing in the Database.
// Parameters:
//   full_run_path - path to a text file with the full list of valid run numbers, one per line
//   period_number - period to check the runs against (default 9)
//   isTest        - if true (default), only print odd and missing runs without changing the Database;
//                   set to false to actually delete the odd run metadata
// Returns 0 on success, or a negative code on error (invalid number / out of range / no runs in the Database).
int remove_odd_runs(TString good_run_file, int period_number = 9, bool isTest = true)
{
    gSystem->ExpandPathName(good_run_file);
 
    // form full run list
    unordered_set<int> run_list;
    ifstream list_file(good_run_file.Data());
    string s;
    while (getline(list_file, s)) {
        try {
            int i_run_number = std::stoi(s);
            run_list.insert(i_run_number);
        } catch (const std::invalid_argument&) {
            cerr << "ERROR: Line is not a valid number: \"" << s << "\"\n";
            return -1;
        } catch (const std::out_of_range&) {
            cerr << "Warning: Line " << s << ": number is out of range\n";
            return -2;
        }
    }
    if (run_list.empty()) {
        cout << "WARNING: There are no runs in the list of the input text file: " << good_run_file << endl;
        return 0;
    }
    cout << "Total run records in the input text file: " << run_list.size() << endl << endl;
 
    // cycle by runs in UniConDa
    UniqueRunNumber* run_numbers;
    int run_count = UniRunPeriod::GetRunNumbers(run_numbers, period_number);
    if (run_count <= 0) {
        cout << "ERROR: runs do not exist in the Database for period " << period_number << endl;
        return -3;
    }
 
    int nfound_runs = 0, removed_runs = 0;
    for (int i = 0; i < run_count; i++) {
        if (run_list.find(run_numbers[i].run_number) != run_list.end())
            continue;
        cout << "Run " << run_numbers[i].run_number << " is not in the list" << endl;
        nfound_runs++;
        if (!isTest) {
            // remove run metadata if no raw file metadata remains for the run
            TObjArray* raw_files = UniRawFile::GetRawFiles(run_numbers[i].period_number, run_numbers[i].run_number);
 
            if ((raw_files != nullptr) and (raw_files->GetEntries() == 0)) {
                cout << "Removing period:run = " << run_numbers[i].period_number << ":" << run_numbers[i].run_number
                     << endl;
                UniRun::DeleteRun(run_numbers[i].period_number, run_numbers[i].run_number);
                removed_runs++;
            }
        }
    }
 
    // find missing runs
    int missing_runs = 0;
    for (int current_run : run_list) {
        if (UniRun::CheckRunExists(period_number, current_run) < 1) {
            cout << "Run " << current_run << " is missing" << endl;
            missing_runs++;
        }
    }
 
    if (nfound_runs + missing_runs > 0)
        cout << endl;
 
    cout << "Found " << nfound_runs << " odd runs" << endl;
    if (!isTest)
        cout << (isTest ? "Would remove " : "Removed ") << removed_runs << " runs" << endl;
    cout << "Found " << missing_runs << " missing runs" << endl;
 
    return 0;
}