FileCatalogue.cxx

Go to the documentation of this file.

#include "FileCatalogue.h"
 
#include "file_db_settings.h"
#include "nlohmann/json.hpp"
 
#include <fstream>
#include <iostream>
#include <map>
#include <sstream>
 
using json = nlohmann::ordered_json;
 
map<int, string> const Errors{{401, "Unauthorized Access (check user login and password)"},
                              {403, "Token invalid or expired"},
                              {404, "Requested Data not found"},
                              {409, "Incorrect Database Request (e.g. violating constraints)"},
                              {422, "Incorrect Input Request"},
                              {500, "Internal Server Error"},
                              {503, "Service Unavaialble"},
                              {504, "Server Timeout"}};
 
string const GetErrorString(int error_id)
{
    auto it = Errors.find(error_id);
    return it != Errors.end() ? it->second : "";
}
 
int GiveMeNumber()
{
    unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
    std::mt19937 engine(seed);
    std::uniform_int_distribution<int> dist(1, 1000000000);
    int randomNum = dist(engine);
    return randomNum;
}
 
FileCondition::FileCondition(searchAttributes attribute, matchCriteria condition)
    : iValueType(0)
    , FileAttribute(attribute)
    , FileMatchCriteria(condition)
{}
 
FileCondition::FileCondition(searchAttributes attribute, matchCriteria condition, int value)
    : iValueType(1)
    , iValue(value)
    , FileAttribute(attribute)
    , FileMatchCriteria(condition)
{}
 
FileCondition::FileCondition(searchAttributes attribute, matchCriteria condition, unsigned int value)
    : iValueType(2)
    , uiValue(value)
    , FileAttribute(attribute)
    , FileMatchCriteria(condition)
{}
 
FileCondition::FileCondition(searchAttributes attribute, matchCriteria condition, float value)
    : iValueType(3)
    , fValue(value)
    , FileAttribute(attribute)
    , FileMatchCriteria(condition)
{}
 
FileCondition::FileCondition(searchAttributes attribute, matchCriteria condition, string value)
    : iValueType(4)
    , strValue(value)
    , FileAttribute(attribute)
    , FileMatchCriteria(condition)
{}
 
// private default constructor
FileCondition::FileCondition()
    : iValueType(0)
    , FileAttribute(kError)
    , FileMatchCriteria(kEqual)
{}
 
FileCondition::searchAttributes FileCondition::getFileAttribute()
{
    return FileAttribute;
}
 
FileCondition::matchCriteria FileCondition::getMatchCriteria()
{
    return FileMatchCriteria;
}
 
string FileCondition::getValueAsString()
{
    switch (iValueType) {
        case 1:
            return std::to_string(iValue);
        case 2:
            return std::to_string(uiValue);
        case 3:
            return std::to_string(fValue);
        case 4:
            return strValue;
    }
    return "0";
}
 
// File Catalog Constructor
FileCatalogue::FileCatalogue()
{
    curl_global_init(CURL_GLOBAL_DEFAULT);
    curlInstance = curl_easy_init();
    if (!curlInstance) {
        throw runtime_error("CURL initialisation failed");
        return;
    }
 
    ServiceUrl =
        FILE_API_HOST + string("/") + FILE_API_NAME + string("/v") + to_string(FileInfo::Version) + string("/");
    ServiceAccount = FILE_API_USERNAME + string(":") + FILE_API_PASSWORD;
}
 
// File Catalog Destructor
FileCatalogue::~FileCatalogue()
{
    curl_easy_cleanup(curlInstance);
    curl_global_cleanup();
}
 
std::string ReplaceAll(std::string str, const std::string& from, const std::string& to)
{
    size_t start_pos = 0;
    while ((start_pos = str.find(from, start_pos)) != std::string::npos) {
        str.replace(start_pos, from.length(), to);
        start_pos += to.length();   // Handles case where 'to' is a substring of 'from'
    }
    return str;
}
 
int FileCatalogue::GetFileList(vector<unique_ptr<FileCondition>> file_conditions, vector<string>& filepaths)
{
    string conditions = "filepath?";
    for (const auto& cond : file_conditions) {
        switch (cond->getFileAttribute()) {
            case FileCondition::kPeriodNumber:
                conditions += "period_number";
                break;
            case FileCondition::kRunNumber:
                conditions += "run_number";
                break;
            case FileCondition::kRunType:
                conditions += "run_type";
                break;
            case FileCondition::kStartDateTime:
                conditions += "start_datetime";
                break;
            case FileCondition::kEndDateTime:
                conditions += "end_datetime";
                break;
            case FileCondition::kBeamParticle:
                conditions += "beam_particle";
                break;
            case FileCondition::kTargetParticle:
                conditions += "target_particle";
                break;
            case FileCondition::kEnergy:
                conditions += "energy";
                break;
            case FileCondition::kFieldVoltage:
                conditions += "field_voltage";
                break;
            case FileCondition::kStartEvent:
                conditions += "start_event";
                break;
            case FileCondition::kEndEvent:
                conditions += "end_event";
                break;
            case FileCondition::kEventCount:
                conditions += "event_count";
                break;
            case FileCondition::kFileSize:
                conditions += "file_size";
                break;
            case FileCondition::kError:
                return -404;
        };
        switch (cond->getMatchCriteria()) {
            case FileCondition::kLessOrEqual:
                conditions = conditions + "=|" + cond->getValueAsString();
                break;
            case FileCondition::kEqual:
                conditions = conditions + "=" + cond->getValueAsString();
                break;
            case FileCondition::kNotEqual:
                conditions = conditions + "!=" + cond->getValueAsString();
                break;
            case FileCondition::kGreaterOrEqual:
                conditions = conditions + "=" + cond->getValueAsString() + "|";
                break;
            case FileCondition::kLike:
                conditions = conditions + "=*" + cond->getValueAsString() + "*";
        };
        conditions += "&";
    }
    conditions.erase(conditions.size() - 1, 1);   // get rid of last &.
    conditions = ReplaceAll(conditions, string(" "), string("%20"));
    string result;
    result = performRequest(conditions, "", "GET");
    if (result == "ERROR: Failed to authenticate\n") {
        return -401;
    }
    if (result == "ERROR: Can't get files list (DIRAC API ERROR)\n") {
        return -500;
    }
    if (result == "ERROR: Can't get files list\n") {
        return -500;
    }
    if (result.find("ERROR") != std::string::npos)
        return -503;   // presumably server is not available
    result = result.erase(0, 2);
    size_t delim = result.find("\",\"");
    while (delim != string::npos) {
        filepaths.push_back(result.substr(0, delim));
        result.erase(0, delim + 3);
        delim = result.find("\",\"");
    }
    delim = result.find("\"]");
    filepaths.push_back(result.substr(0, delim));
    return 0;
}
 
int FileCatalogue::GetFileInfo(string LFN, FileInfo& file_info)
{
    string conditions = "meta?filepath=" + LFN;
    string res = performRequest(conditions, "", "GET");
    if (res == "ERROR: Failed to authenticate\n") {
        return -401;
    }
    if (res == "ERROR: File not found\n") {
        return -404;
    }
    if (res == "ERROR: Can't get meta (DIRAC API ERROR)\n") {
        return -500;
    }
    if (res == "ERROR: Can't get meta\n") {
        return -500;
    }
    if (res.find("ERROR") != std::string::npos)
        return -503;   // presumably server is not available
    json result = json::parse(res);
    char DTbuf[100];
    std::tm DTtm = {};
    for (auto& [key, value] : result.items()) {
        if (key == "period_number")
            file_info.PeriodNumber = value.get<int>();
        if (key == "run_number")
            file_info.RunNumber = value.get<int>();
        if (key == "run_type")
            file_info.RunType = value.get<int>();
        if (key == "start_datetime") {
            strptime(value.get<string>().c_str(), "%a, %d %b %Y %H:%M:%S GMT", &DTtm);
            strftime(DTbuf, sizeof DTbuf, "%Y-%m-%d %H:%M:%S", &DTtm);   // format from default GMT to YY-MM-DD hh-mm-ss
            file_info.StartDateTime = DTbuf;
        }
        if (key == "end_datetime") {
            strptime(value.get<string>().c_str(), "%a, %d %b %Y %H:%M:%S GMT", &DTtm);
            strftime(DTbuf, sizeof DTbuf, "%Y-%m-%d %H:%M:%S", &DTtm);   // format from default GMT to YY-MM-DD hh-mm-ss
            file_info.EndDateTime = DTbuf;
        }
        if (key == "beam_particle")
            file_info.BeamParticle = value.get<string>();
        if (key == "target_particle")
            file_info.TargetParticle = value.get<string>();
        if (key == "energy")
            file_info.Energy = value.get<float>();
        if (key == "field_voltage")
            file_info.FieldVoltage = value.get<float>();
        if (key == "start_event")
            file_info.StartEvent = value.get<int>();
        if (key == "end_event")
            file_info.EndEvent = value.get<int>();
        if (key == "event_count")
            file_info.EventCount = value.get<int>();
        if (key == "file_size")
            file_info.FileSize = value.get<float>();
    }
    return 0;
}
 
int FileCatalogue::GetDirectoryInfo(string path, string& directory_info)
{
    string conditions = "directorymeta?dirpath=" + path;
    string res = performRequest(conditions, "", "GET");
    if (res == "ERROR: Failed to authenticate\n") {
        return -401;
    }
    if (res == "ERROR: File not found\n") {
        return -404;
    }
    if (res == "ERROR: Can't get meta (DIRAC API ERROR)\n") {
        return -500;
    }
    if (res == "ERROR: Can't get meta\n") {
        return -500;
    }
    if (res.find("ERROR") != std::string::npos)
        return -503;   // presumably server is not available
    directory_info = res;
    return 0;
}
 
int FileCatalogue::GetConsistencyCheckStatus()
{
    string conditions = "checkstatus?";
    string result = performRequest(conditions, "", "GET");
    if (result == "ERROR: Can't read check status file\n")
        return -500;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    result = result.erase(0, result.find("ID=") + 3);
    return stoi(result);
}
 
int FileCatalogue::GetConsistencyCheckResults(int& eos_missing_count,
                                              int& fc_missing_count,
                                              vector<string>& eos_missing,
                                              vector<string>& fc_missing)
{
    string conditions = "checkresults?";
    string result = performRequest(conditions, "", "GET");
    if (result == "ERROR: Can't read check results file\n")
        return -500;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    if (result.find("ERROR") != std::string::npos)
        return -503;   // presumably server is not available
    string line;
    stringstream stream(result);
    getline(stream, line);   // skipping the first line
    while (getline(stream, line)) {
        if (line.find("Addto count:") != string::npos) {
            fc_missing_count = stoi(line.erase(0, 12));
            break;
        }
        fc_missing.push_back(line);
    }
    getline(stream, line);   // skipping the line
    while (getline(stream, line)) {
        if (line.find("Missing count:") != string::npos) {
            eos_missing_count = stoi(line.erase(0, 14));
            break;
        }
        eos_missing.push_back(line);
    }
    return 0;
}
 
int FileCatalogue::GetFile(string LFN, string save_path)
{
    if (save_path.find("root") == 0) {
        // xrd way
        string conditions = "file_to_xrd?LFN=" + LFN + "&XRDURL=" + save_path;
        string res = performRequest(conditions, "", "GET");
        if (res == "ERROR: Failed to authenticate\n")
            return -401;
        if (res == "ERROR: Can't download file from CICC(DIRAC)\n")
            return -500;
        if (res == "ERROR: Can't upload to xrootd path\n")
            return -500;
        return 0;
    } else {
        string conditions = "file?LFN=" + LFN;
        return DownloadFile(conditions, save_path);
    }
}
 
int FileCatalogue::GetPFNs(string LFN, vector<string>& PFNs)
{
    string conditions = "pfn?LFN=" + LFN;
    string res = performRequest(conditions, "", "GET");
    if (res == "ERROR: Failed to authenticate\n") {
        return -401;
    }
    if (res == "ERROR: None found\n") {
        return -404;
    }
    if (res == "ERROR: Can't get PFN\n") {
        return -500;
    }
    if (res.find("ERROR") != std::string::npos)
        return -503;   // presumably server is not available
    try {
        json j = json::parse(res);
        if (!j.is_array()) {
            return -500;
        }
        PFNs = j.get<std::vector<std::string>>();
        return 0;
    } catch (const json::parse_error& e) {
        // std::cerr << "Ошибка парсинга JSON: " << e.what() << std::endl;
        return -500;
    }
    return 0;
}
 
int FileCatalogue::AddFileInfo(string LFN, FileInfo file_metadata)
{
    json fi;
    fi["run_number"] = file_metadata.RunNumber;
    fi["run_type"] = file_metadata.RunType;
    fi["start_datetime"] = file_metadata.StartDateTime;
    fi["end_datetime"] = file_metadata.EndDateTime;
    fi["beam_particle"] = file_metadata.BeamParticle;
    fi["target_particle"] = file_metadata.TargetParticle;
    fi["energy"] = file_metadata.Energy;
    fi["field_voltage"] = file_metadata.FieldVoltage;
    fi["start_event"] = file_metadata.StartEvent;
    fi["end_event"] = file_metadata.EndEvent;
    fi["event_count"] = file_metadata.EventCount;
    fi["file_size"] = file_metadata.FileSize;
    string conditions = "meta?filepath=" + LFN;
    string result = performRequest(conditions, fi.dump(), "PUT");
    if (result == "ERROR: Can't set meta (DIRAC API ERROR)\n" || result == "ERROR: Can't set meta\n")
        return -500;
    if (result == "ERROR: File not found\n")
        return -404;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    return 0;
}
 
int FileCatalogue::AddMetadataField(string field_name, string field_type, bool isDirectoryType)
{
    set<string> valid_meta_types = {"INT", "DATETIME", "FLOAT", "VARCHAR(128)"};
    if (valid_meta_types.find(field_type) == valid_meta_types.end()) {
        cout << " Please, use the valid metadata field type. Like one of this: (INT, DATETIME, FLOAT, VARCHAR(128). "
                "Thanks"
             << endl;
        return -422;
    }
    string conditions = "";
    if (isDirectoryType)
        conditions = "metadatafield_directory?fieldname=" + field_name + "&fieldtype=" + field_type;
    else
        conditions = "metadatafield?fieldname=" + field_name + "&fieldtype=" + field_type;
    string result = performRequest(conditions, "", "POST");
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    if (result == "ERROR: Can't set metadata field (DIRAC API ERROR)\n"
        || result == "ERROR: Can't set metadata field\n")
        return -500;
    return 0;
}
 
size_t appendData(void* contents, size_t size, size_t nmemb, string* data)
{
    size_t realsize = size * nmemb;
    try {
        data->append((char*)contents, realsize);
    } catch (std::bad_alloc& e) {
        // handle memory problem
        return 0;
    }
 
    return realsize;
}
 
int FileCatalogue::AddFileInfo(string json_file_path)
{
    ifstream json_file(json_file_path);
    json full_json = json::parse(json_file);
    auto it = full_json.find("file_path");
    if (it == full_json.end()) {
        cout << " Please include \"file_path\" field with correct filepath to file in your json, thanks " << endl;
        return -422;
    }
    string filepath = it.value();
    full_json.erase("file_path");
    string conditions = "meta?filepath=" + filepath;
    string result = performRequest(conditions, full_json.dump(), "PUT");
 
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    if (result == "ERROR: Can't set meta (DIRAC API ERROR)\n" || result == "ERROR: Can't set meta\n")   // some
        return -500;
    if (result == "ERROR: File not found\n")   // file not found
        return -404;
    return 0;
}
 
int FileCatalogue::AddDirectoryInfo(string json_file_path)
{
    ifstream json_file(json_file_path);
    json full_json = json::parse(json_file);
    auto it = full_json.find("dir_path");
    if (it == full_json.end()) {
        cout << " Please include \"dir_path\" field with correct filepath to file in your json, thanks " << endl;
        return -422;
    }
    string dirpath = it.value();
    full_json.erase("dir_path");
    string conditions = "meta?filepath=" + dirpath;
    string result = performRequest(conditions, full_json.dump(), "PUT");
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    if (result == "ERROR: Can't set meta (DIRAC API ERROR)\n" || result == "ERROR: Can't set meta\n")   // some
        return -500;
    if (result == "ERROR: File not found\n")   // file not found
        return -404;
    return 0;
}
 
int FileCatalogue::AddFile(string input_path, string LFN)
{
    string result;
    string conditions = "file?LFN=" + LFN;
    return UploadFile(conditions, input_path);
}
 
int FileCatalogue::UpdateFileInfo(string LFN, FileInfo file_metadata)
{
    return AddFileInfo(LFN, file_metadata);
}
 
int FileCatalogue::DeleteFileInfo(string LFN)
{
    string conditions = "meta?filepath=" + LFN;
    string result = performRequest(conditions, "", "DELETE");
    if (result == "ERROR: Can't delete metadata (DIRAC API ERROR)\n" || result == "ERROR: Can't delete metadata\n")
        return -500;
    if (result == "ERROR: File not found\n")
        return -404;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    return 0;
}
 
int FileCatalogue::DeleteDirectoryInfo(string path)
{
    string conditions = "directorymeta?dirpath=" + path;
    string result = performRequest(conditions, "", "DELETE");
    if (result == "ERROR: Can't delete metadata (DIRAC API ERROR)\n" || result == "ERROR: Can't delete metadata\n")
        return -500;
    if (result == "ERROR: Directory not found\n")
        return -404;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    return 0;
}
 
int FileCatalogue::DeleteMetadataField(string metadatafield_name)
{
    string conditions = "metadatafield?fieldname=" + metadatafield_name;
    string result = performRequest(conditions, "", "DELETE");
    if (result == "ERROR: Can't delete metadata field (DIRAC API ERROR)\n"
        || result == "ERROR: Can't delete metadata field\n")
        return -500;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    return 0;
}
 
int FileCatalogue::DeleteFile(string LFN)
{
    string conditions = "file?LFN=" + LFN;
    string result = performRequest(conditions, "", "DELETE");
    if (result == "ERROR: Can't delete file from CICC (DIRAC API ERROR)\n"
        || result == "ERROR: Can't delete file from CICC\n")
        return -500;
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    return 0;
}
 
int FileCatalogue::CheckFileInfo(string LFN)
{
    string conditions = "meta?filepath=" + LFN;
    string res = performRequest(conditions, "", "GET");
    if (res == "ERROR: Failed to authenticate\n")
        return -401;
    if (res == "ERROR: File not found\n")
        return -404;
    if (res == "ERROR: Can't get meta (DIRAC API ERROR)\n" || res == "ERROR: Can't get meta\n")
        return -500;
    if (res == "{}\n")
        return -1;
    return 0;
}
 
int FileCatalogue::CheckFileExist(string LFN)
{
    string conditions = "isfileexist?filepath=" + LFN;
    string res = performRequest(conditions, "", "GET");
    if (res == "ERROR: Failed to authenticate\n")
        return -401;
    if (res == "ERROR: Can't check file exist (DIRAC API ERROR)\n" || res == "ERROR: Can't check file exist\n")
        return -500;
    if (res == "FALSE\n")
        return 0;
    if (res == "TRUE\n")
        return 1;
    return -500;
}
 
int FileCatalogue::PrintFileInfo(string LFN)
{
    string conditions = "meta?filepath=" + LFN;
    string res = performRequest(conditions, "", "GET");
    if (res == "ERROR: Failed to authenticate\n") {
        cout << " Sorry, failed to authenticate" << endl;
        return -401;
    }
    if (res == "ERROR: File not found\n") {
        cout << " The file " << LFN << " was not found! " << endl;
        return -2;
    }
    if (res == "ERROR: Can't get meta (DIRAC API ERROR)\n" || res == "ERROR: Can't get meta\n") {
        cout << " Sorry, we got some errors while getting metadata of your file " << endl;
        return -500;
    }
    if (res == "{}\n") {
        cout << " There is no metadata for your file - " << LFN << endl;
        return 0;
    }
    cout << " Here is metadata for your file - " << LFN << ": " << endl << res << endl;
    return 0;
}
 
int FileCatalogue::PrintMetadataFields()
{
    string res = performRequest("metadatafields?", "", "GET");
    if (res == "ERROR: Can't get metadata fields (DIRAC API ERROR)\n" || res == "ERROR: Can't get metadata fields\n")
        return -500;
    if (res == "ERROR: Failed to authenticate\n")
        return -401;
    cout << " Metadatafields list: " << endl << res << endl;
    return 0;
}
 
std::string exec(const char* cmd)
{
    char buffer[128];
    std::string result = "";
    FILE* pipe = popen(cmd, "r");
    if (!pipe)
        return "-1";
    // throw std::runtime_error("popen() failed!");
    try {
        while (fgets(buffer, sizeof buffer, pipe) != NULL) {
            result += buffer;
        }
    } catch (...) {
        pclose(pipe);
        throw;
    }
    pclose(pipe);
    return result;
}
 
int FileCatalogue::FileCatalogRequest(string rest_api_request, vector<string>& filepaths)
{
    vector<string> file_array;
    if (rest_api_request.find("GET") == string::npos) {
        return -422;
    }
    string result;
    try {
        result = exec(rest_api_request.c_str());
    } catch (...) {
        return -500;
    }
    if (result == "ERROR: Failed to authenticate\n")
        return -401;
    // if (result.find("ERROR") != std::string::npos)
    //     return -503;    // presumably server is not available
    result = result.erase(0, 2);
    size_t delim = result.find("\",\"");
    while (delim != string::npos) {
        filepaths.push_back(result.substr(0, delim));
        result.erase(0, delim + 3);
        delim = result.find("\",\"");
    }
    delim = result.find("\"]");
    filepaths.push_back(result.substr(0, delim));
    return 0;
}
 
int FileCatalogue::FileCatalogConsistencyCheck()
{
    int check_id = 0;
    string res = performRequest("consistencycheck?", "", "POST");
    if (res == "ERROR: Can't run consistency check (error from running shell script)\n")
        return -500;
    if (res == "ERROR: Failed to authenticate\n")
        return -401;
    try {
        check_id = stoi(res.erase(0, 21));
    } catch (const std::invalid_argument& e) {
        return -500;
    }
    return check_id;
}
 
int FileCatalogue::FileCatalogStats(int& files_count,
                                    int& files_without_replicas,
                                    int& replicas_count,
                                    int& replicas_without_files)
{
    string res = performRequest("catalogcounters?", "", "GET");
    if (res == "ERROR: Can't get catalog counters (DIRAC API ERROR)\n" || res == "ERROR: Can't get catalog counters\n")
        return -500;
    if (res == "ERROR: Failed to authenticate\n")
        return -401;
    json metrics = json::parse(res);
    for (auto& [key, value] : metrics.items()) {
        if (key == "Files")
            files_count = value.get<int>();
        if (key == "Files w/o Replicas")
            files_without_replicas = value.get<int>();
        if (key == "Replicas")
            replicas_count = value.get<int>();
        if (key == "Replicas w/o Files")
            replicas_without_files = value.get<int>();
    }
    return 0;
}
 
string FileCatalogue::performRequest(string conditions, string json_content, string qType)
{
    CURLcode res;
    string response_string;
 
    conditions = ServiceUrl + conditions;
    const char* curl_url = conditions.c_str();
    curl_easy_setopt(curlInstance, CURLOPT_URL, curl_url);
 
    /* cache the CA cert bundle in memory for a week */
#ifdef CURLOPT_CA_CACHE_TIMEOUT
    curl_easy_setopt(curlInstance, CURLOPT_CA_CACHE_TIMEOUT, 604800L);   // 7 days
#endif
    curl_easy_setopt(curlInstance, CURLOPT_WRITEFUNCTION, appendData);
    curl_easy_setopt(curlInstance, CURLOPT_WRITEDATA, &response_string);
    curl_easy_setopt(curlInstance, CURLOPT_USERNAME, FILE_API_USERNAME);
    curl_easy_setopt(curlInstance, CURLOPT_PASSWORD, FILE_API_PASSWORD);
 
    struct curl_slist* headers = NULL;
    // headers = curl_slist_append(headers, "Accept: application/json");
    headers = curl_slist_append(headers, "Content-Type: application/json");
    // headers = curl_slist_append(headers, "charset: utf-8");
 
    const char* c_json = json_content.c_str();   // what's the point of using nlohmann then?
    const char* curl_customRequest = qType.c_str();
    curl_easy_setopt(curlInstance, CURLOPT_CUSTOMREQUEST, curl_customRequest);
    curl_easy_setopt(curlInstance, CURLOPT_HTTPHEADER, headers);
    curl_easy_setopt(curlInstance, CURLOPT_POSTFIELDS, c_json);   // jsonObj
    /* Perform the request, res gets the return code */
    res = curl_easy_perform(curlInstance);
    /* Check for errors */
    if (res != CURLE_OK) {
        fprintf(stderr, "curl_easy_perform() failed: %s\n", curl_easy_strerror(res));
    }
 
    long response_code;
    curl_easy_getinfo(curlInstance, CURLINFO_RESPONSE_CODE, &response_code);
    if (VerboseLevel > 2) {
        double elapsed;
        curl_easy_getinfo(curlInstance, CURLINFO_TOTAL_TIME, &elapsed);
        char* url;
        curl_easy_getinfo(curlInstance, CURLINFO_EFFECTIVE_URL, &url);
        cout << "DEBUG: CURLINFO_TOTAL_TIME = " << elapsed << ", CURLINFO_EFFECTIVE_URL = " << string(url) << endl;
    }
 
    if (VerboseLevel > 1) {
        double elapsed;
        curl_easy_getinfo(curlInstance, CURLINFO_TOTAL_TIME, &elapsed);
        char* url;
        curl_easy_getinfo(curlInstance, CURLINFO_EFFECTIVE_URL, &url);
        cout << "CURLINFO_TOTAL_TIME = " << elapsed << endl;
    }
    // check the result of the request
    if ((response_code >= 200) && (response_code < 205)) {
        if (VerboseLevel > 0)
            cout << qType << " request has been succesfully processed " << endl;
        if (VerboseLevel > 1)
            cout << "Request was finished with the following response: " << response_string << endl;
    } else {
        cout << "ERROR: Sending request to '" << ServiceUrl << "' has been failed with code = " << response_code
             << " and message: " << endl
             << response_string << endl;
        curl_slist_free_all(headers);
        return "ERROR: error code: " + to_string(response_code);
    }
    curl_slist_free_all(headers);
    return response_string;
}
 
enum State
{
    CHECK_PREFIX,
    WRITE_FILE
};
 
struct WriteData
{
    State state = CHECK_PREFIX;
    std::string buffer;   // накопленные данные для проверки
    std::ofstream* file = nullptr;
    bool error_detected = false;
    std::string error_msg;
};
 
size_t WriteCallback(void* contents, size_t size, size_t nmemb, void* userp)
{
    WriteData* data = static_cast<WriteData*>(userp);
    size_t total = size * nmemb;
    char* chunk = static_cast<char*>(contents);
 
    if (data->error_detected) {
        return total;   // игнорируем остальные данные
    }
 
    if (data->state == CHECK_PREFIX) {
        // Добавляем в буфер
        data->buffer.append(chunk, total);
        // Проверяем, достаточно ли данных для проверки (минимум 5 байт)
        if (data->buffer.size() >= 5) {
            if (data->buffer.compare(0, 5, "ERROR") == 0) {
                data->error_detected = true;
                data->error_msg = data->buffer;
                return total;   // можно вернуть 0, чтобы прервать запись, но проще игнорировать
            } else {
                // Это не ошибка – переключаемся на запись в файл
                data->state = WRITE_FILE;
                if (data->file && data->file->is_open()) {
                    data->file->write(data->buffer.data(), data->buffer.size());
                }
                data->buffer.clear();
            }
        }
        return total;
    }
 
    // WRITE_FILE – просто пишем в файл
    if (data->file && data->file->is_open()) {
        data->file->write(chunk, total);
    }
    return total;
}
 
int FileCatalogue::DownloadFile(string conditions, string savePath)
{
 
    std::ofstream file(savePath, std::ios::binary);
    if (!file.is_open()) {
        curl_easy_cleanup(curlInstance);
        return -500;
    }
    WriteData data;
    data.file = &file;
 
    conditions = ServiceUrl + conditions;
    const char* curl_url = conditions.c_str();
    curl_easy_setopt(curlInstance, CURLOPT_URL, curl_url);
    curl_easy_setopt(curlInstance, CURLOPT_WRITEFUNCTION, WriteCallback);
    curl_easy_setopt(curlInstance, CURLOPT_WRITEDATA, &data);
    curl_easy_setopt(curlInstance, CURLOPT_FAILONERROR, 1L);
    curl_easy_setopt(curlInstance, CURLOPT_USERNAME, FILE_API_USERNAME);
    curl_easy_setopt(curlInstance, CURLOPT_PASSWORD, FILE_API_PASSWORD);
 
    CURLcode res = curl_easy_perform(curlInstance);
    long http_code = 0;
    curl_easy_getinfo(curlInstance, CURLINFO_RESPONSE_CODE, &http_code);
    curl_easy_cleanup(curlInstance);
    file.close();
 
    if (data.error_detected) {
        // Удаляем частично записанный файл (если он был создан)
        std::remove(savePath.c_str());
        if (data.error_msg == "ERROR: Failed to authenticate\n")
            return -401;
        else
            return -500;
    }
 
    if (res != CURLE_OK) {
        cout << "CURL error: " << curl_easy_strerror(res) << endl;
        std::remove(savePath.c_str());
        return -500;
    }
 
    return 0;
}
 
// MIME API появился в libcurl 7.56.0, CURLOPT_XFERINFOFUNCTION – в 7.32.0.
// На старых системах (например, CentOS 7 с libcurl 7.29.0) используются устаревшие аналоги.
#define FC_HAS_CURL_MIME (LIBCURL_VERSION_NUM >= 0x073800)
#define FC_HAS_CURL_XFERINFO (LIBCURL_VERSION_NUM >= 0x072000)
 
#if FC_HAS_CURL_XFERINFO
int progressCallback(void* clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
{
#else
int progressCallback(void* clientp, double dltotal, double dlnow, double ultotal, double ulnow)
{
#endif
    if (ultotal > 0) {
        double percent = (double)ulnow / ultotal * 100;
        std::cout << "\rUploading: " << int(percent) << "%" << std::flush;
    }
    return 0;   // 0 – продолжать, 1 – прервать
}
 
int FileCatalogue::UploadFile(string conditions, string filepath)
{
#if FC_HAS_CURL_MIME
    struct curl_mime* mime = curl_mime_init(curlInstance);
    struct curl_mimepart* part = curl_mime_addpart(mime);
    curl_mime_name(part, "file");                 // имя поля (как в -F "file=...")
    curl_mime_filedata(part, filepath.c_str());   // прикрепляем файл
#else
    struct curl_httppost* formpost = nullptr;
    struct curl_httppost* lastptr = nullptr;
    curl_formadd(&formpost, &lastptr, CURLFORM_COPYNAME, "file",   // имя поля (как в -F "file=...")
                 CURLFORM_FILE, filepath.c_str(),                  // прикрепляем файл
                 CURLFORM_END);
#endif
 
    std::string response;
    conditions = ServiceUrl + conditions;
    const char* curl_url = conditions.c_str();
    curl_easy_setopt(curlInstance, CURLOPT_URL, curl_url);
    curl_easy_setopt(curlInstance, CURLOPT_USERNAME, FILE_API_USERNAME);
    curl_easy_setopt(curlInstance, CURLOPT_PASSWORD, FILE_API_PASSWORD);
#if FC_HAS_CURL_MIME
    curl_easy_setopt(curlInstance, CURLOPT_MIMEPOST, mime);   // POST с multipart
#else
    curl_easy_setopt(curlInstance, CURLOPT_HTTPPOST, formpost);   // POST с multipart
#endif
    curl_easy_setopt(curlInstance, CURLOPT_WRITEFUNCTION, appendData);
    curl_easy_setopt(curlInstance, CURLOPT_WRITEDATA, &response);
    curl_easy_setopt(curlInstance, CURLOPT_FAILONERROR, 1L);   // чтобы HTTP>=400 возвращал ошибку
 
    curl_easy_setopt(curlInstance, CURLOPT_TIMEOUT, 3600L);           // 1 час
    curl_easy_setopt(curlInstance, CURLOPT_LOW_SPEED_LIMIT, 1024L);   // 1 КБ/с минимум
    curl_easy_setopt(curlInstance, CURLOPT_LOW_SPEED_TIME, 60L);      // если медленнее 60 сек – таймаут
 
#if FC_HAS_CURL_XFERINFO
    curl_easy_setopt(curlInstance, CURLOPT_XFERINFOFUNCTION, progressCallback);
#else
    curl_easy_setopt(curlInstance, CURLOPT_PROGRESSFUNCTION, progressCallback);
#endif
    curl_easy_setopt(curlInstance, CURLOPT_NOPROGRESS, 0L);
 
    CURLcode res = curl_easy_perform(curlInstance);
 
    long http_code = 0;
    curl_easy_getinfo(curlInstance, CURLINFO_RESPONSE_CODE, &http_code);
 
#if FC_HAS_CURL_MIME
    curl_mime_free(mime);
#else
    curl_formfree(formpost);
#endif
    curl_easy_cleanup(curlInstance);
 
    if (res != CURLE_OK) {
        return http_code ? -http_code : -500;
    }
 
    // Проверяем, не вернул ли сервер ошибку (сообщение начинается с "ERROR")
    if (response.compare(0, 5, "ERROR") == 0) {
        if (response == "ERROR: Failed to authenticate\n")
            return -401;
        if (response == "ERROR: No file uploaded\n" or response == "ERROR: Empty filename\n")
            return -422;
        return -500;
    }
 
    // Успех
    // std::cout << "Upload successful. Server response: " << response << std::endl;
    return 0;
}