FileCatalogue.h

Go to the documentation of this file.

/** Interface to the File Catalog (through REST API service) to add, update, delete and search for file metadata
 *
 * @author  I.S.Zhironkin // based on K.Gertsenberger template
 * @version 1
 * @since   09.01.24
 *
 */
#ifndef FILECATALOG_H
#define FILECATALOG_H
 
// #include "TObject.h"
 
#include <chrono>
#include <cstdlib>
#include <curl/curl.h>
#include <iostream>
#include <memory>
#include <random>
#include <set>
#include <string>
#include <vector>
using namespace std;
 
/// Class presenting File Info (Metadata) Structure
class FileInfo
{
  public:
    static const int Version = 1;   ///< structure (API) version
 
    int PeriodNumber = -1;   ///< number of experiment session, when the events was collected
    int RunNumber = -1;      ///< number of experiment run, when the events was collected
    int RunType = -1;        ///< run type (1 - physics run, 2 - calibration run, 3 - test run)
 
    string StartDateTime =
        "1970-01-01 00-00-00";   ///< start datetime of the stored event set (YYYY-MM-DD HH-MM-SS format)
    string EndDateTime = "1970-01-01 00-00-00";   ///< end datetime of the stored event set (YYYY-MM-DD HH-MM-SS format)
    string BeamParticle = "";                     ///< beam particle name
    string TargetParticle = "";                   ///< target particle name
 
    float Energy = 0.0;         ///< beam energy
    float FieldVoltage = 0.0;   ///< field voltage
 
    int StartEvent = -1;   ///< event number from which stored event set begins
    int EndEvent = -1;     ///< event number with which event set ends
    int EventCount = -1;   ///< event count stored in the file
 
    float FileSize = -1;   ///< file size in bytes
 
    /// Print all file metadata to the standard output
    void Print() const
    {
        cout << "FileInfo (version " << Version << "):" << endl;
        cout << "  Period:Run  = " << PeriodNumber << ":" << RunNumber << endl;
        cout << "  Run Type    = " << RunType << endl;
        cout << "  Time        = " << StartDateTime << " - " << EndDateTime << endl;
        cout << "  Beam        = " << BeamParticle << endl;
        cout << "  Energy      = " << Energy << " GeV" << endl;
        cout << "  Target      = " << TargetParticle << endl;
        cout << "  MagField    = " << FieldVoltage << " mV" << endl;
        cout << "  Event Range = " << StartEvent << " - " << EndEvent << endl;
        cout << "  Event Count = " << EventCount << endl;
        cout << "  File Size   = " << FileSize << endl;
    }
};
 
/// Class for describing a condition to search for necessary file metadata
class FileCondition
{
  public:
    /// enumeration for attributes, which can be used in searching file metadata
    enum searchAttributes
    {
        kPeriodNumber,
        kRunNumber,
        kRunType,
        kStartDateTime,
        kEndDateTime,
        kBeamParticle,
        kTargetParticle,
        kEnergy,
        kFieldVoltage,
        kStartEvent,
        kEndEvent,
        kEventCount,
        kFileSize,
        kError
    };
    /// enumeration for conditions, which can be used for matching with a specified value
    enum matchCriteria
    {
        kLessOrEqual,
        kEqual,
        kNotEqual,
        kGreaterOrEqual,
        kLike
    };
 
    /*!
     * Create condition for searching file metadata
     * \param[in] attribute Used for searching file metadata
     * \param[in] condition Defining the operation of comparison with the value
     */
    FileCondition(searchAttributes attribute, matchCriteria condition);
    FileCondition(searchAttributes attribute, matchCriteria condition, int value);
    FileCondition(searchAttributes attribute, matchCriteria condition, unsigned int value);
    FileCondition(searchAttributes attribute, matchCriteria condition, float value);
    FileCondition(searchAttributes attribute, matchCriteria condition, string value);
    virtual ~FileCondition() {}
 
    searchAttributes getFileAttribute();
    matchCriteria getMatchCriteria();
    string getValueAsString();
 
  private:
    FileCondition();
 
    // 1 - int, 2 - unsigned int, 3 - float, 4 - TString
    int iValueType;
    int iValue;
    unsigned int uiValue;
    float fValue;
    string strValue;
 
    searchAttributes FileAttribute;
    matchCriteria FileMatchCriteria;
};
 
/// Main Interface Class to work with the File Catalog of the experiment
/// Throws runtime_exception if a critical error occurrs while the constructor
/// Error Codes: -1 - sending command via CURL failed, -401 - unauthorized request, -403 - token invalid or expired,
/// -404 - data not found, -409 - incorrect database request (e.g. violating constraints), -422 - incorrect input
/// request, -500 - internal server error, -503 - service unavaialble, -504 - gateway timeout
 
class FileCatalogue
{
  public:
    FileCatalogue();
    virtual ~FileCatalogue();
 
    /*!
     * Function for searching for file metadata in the File Catalog using specified conditions
     * \param[in] file_conditions A set of conditions to specify criteria for file metadata search
     * \param[out] filepaths Vector of the filepaths to files containing selected file metadata (empty if not found or
     * some errors occured
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int GetFileList(vector<unique_ptr<FileCondition>> file_conditions, vector<string>& filepaths);
 
    /*!
     * Function to get metadata of the file stored at the given location
     * \param[in] LFN Logical File Name of the file
     * \param[out] file_info FileInfo object containing file metadata
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int GetFileInfo(string LFN, FileInfo& file_info);
 
    /*!
     * Function to get metadata of the directory at the given path
     * \param[in] path Path to the directory in the File Catalog
     * \param[out] directory_info String containing metadata fields and it's values
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int GetDirectoryInfo(string path, string& directory_info);
 
    /*!
     * Function to get status of the last successful consistency check. Date and ID.
     * \return ID (>=0) or error_code (<0) in case of failures
     */
    int GetConsistencyCheckStatus();
 
    /*!
     * Function to get result of the last successful consistency check. Presumably up to hundreds megabytes of data
     * \param[out] eos_missing_count number of files that exist in File Catalog but missing on EOS
     * \param[out] fc_missing_count number of files that exist in EOS but missing from File Catalog
     * \param[out] eos_missing paths to files (lfns) in File Catalog that are missing from EOS
     * \param[out] fc_missing paths to files in EOS that are missing from File Catalog
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int GetConsistencyCheckResults(int& eos_missing_count,
                                   int& fc_missing_count,
                                   vector<string>& eos_missing,
                                   vector<string>& fc_missing);
 
    /*!
     * Function to download file from the CICC (Dirac FC)
     * \param[in] LFN LFN of the downloaded file
     * \param[in] save_path path to place where you want to save it. locally or by xrootd URL
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int GetFile(string LFN, string save_path);
 
    /*!
     * Function to get file PFN (Physical File Name)
     * \param[in] LFN LFN of the file
     * \param[out] PFNs Vector of PFNs of the file
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int GetPFNs(string LFN, vector<string>& PFNs);
 
    /*!
     * Function to add metadata for the file with specific LFN
     * \param[in] LFN Logical File Name of the file
     * \param[in] file_metadata 'FileInfo' object containing file metadata
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int AddFileInfo(string LFN, FileInfo file_metadata);
 
    /*!
     * Function to add file metadata to the File Catalog using input JSON file path
     * \param[in] json_file_path path of the JSON file containing a set of file metadata to be stored
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int AddFileInfo(string json_file_path);
 
    /*!
     * Function to add directory metadata to the File Catalog using input JSON directory path. dir_path field for path
     * to directory
     * \param[in] json_file_path path of the JSON file containing a set of directory metadata to be stored
     * \return 0 - if successful, error_code (<0) in case of failures
     */
    int AddDirectoryInfo(string json_file_path);
 
    /*!
     * Function to add new metadata field to the File Catalog
     * \param[in] field_name metadata field name to be added
     * \param[in] field_type metadata field type (INT, DATETIME, FLOAT, VARCHAR(128))
     * \param[in] isDirectoryType which type of field is needed. Directory or file
     * \return 0 - if successfully added, error_code (<0) in case of failures
     */
    int AddMetadataField(string field_name, string field_type, bool isDirectoryType = true);
 
    /*!
     * Function to add file to the File Catalog and upload to CICC
     * \param[in] input_path local path to the file
     * \param[in] LFN LFN of the output file on the CICC and FileCatalogue
     * \return 0 - if successfully added, error_code (<0) in case of failures
     */
    int AddFile(string input_path, string LFN);
 
    /*!
     * Function to update metadata for the file with specific LFN
     * \param[in] LFN Logical File Name of the file
     * \param[in] file_metadata 'FileInfo' object containing file metadata
     * \return 0 - if successfully updated, <0 - an error occured
     */
    int UpdateFileInfo(string LFN, FileInfo file_metadata);
 
    /*!
     * Function to delete metadata for the file with specific LFN
     * \param[in] LFN Logical File Name of the file
     * \return 0 - if successfully deleted, <0 - an error occured
     */
    int DeleteFileInfo(string LFN);
 
    /*!
     * Function to delete metadata for the directory at specific path
     * \param[in] path Path to the directory inside File Catalog
     * \return 0 - if successfully deleted, <0 - an error occured
     */
    int DeleteDirectoryInfo(string path);
 
    /*!
     * Function to delete metadata field from the File Catalog. Either for file or directory
     * \param[in] metadatafield_name Metadatafield name which you want to delete
     * \return 0 - if successfully deleted, <0 - an error occured
     */
    int DeleteMetadataField(string metadatafield_name);
 
    /*!
     * Function to delete file from the File Catalog and CICC
     * \param[in] LFN Logical File Name of the file
     * \return 0 - if successfully deleted, error_code (<0) in case of failures
     */
    int DeleteFile(string LFN);
 
    /*!
     * Function to check metadata for the file with specific LFN
     * \param[in] LFN Logical File Name of the file
     * \return 0 - if file metadata is avaialble, -1 - if file metadata is not set, error_code (<0) in case of
     * failures
     */
    int CheckFileInfo(string LFN);
 
    /*!
     * Function to check whether file exist or not
     * \param[in] LFN Logical File Name of the file
     * \return 1 - if file exist, 0 - if not, <0 - an error occured
     */
    int CheckFileExist(string LFN);
 
    /*!
     * Function to print metadata for the file stored at the given location
     * \param[in] LFN Logical File Name of the file
     * \return 0 - if successfull, -2 - if file was not found at the location,
     * error_code (<0) in case of failures
     */
    int PrintFileInfo(string LFN);
 
    /*!
     * Function to print current metadata fields
     * \return 0 - if succesfull, <0 - an error occured
     */
    int PrintMetadataFields();
 
    /*!
     * Function for searching for files matching specific metadata in the File Catalog using REST API request as an
     * HTTP(s) string
     * \param[in] rest_api_request String HTTP command representing a request using REST API format
     * \param[out] filepaths Vector of the filepath containing selected file metadata (empty if not found or some errors
     * occured)
     * \return 0 - if succesfull, <0 - an error occured
     */
 
    int FileCatalogRequest(string rest_api_request, vector<string>& filepaths);
 
    /*!
     * Function to perform a consistency check in the File Catalog
     * \return check ID if successfull, <0 - an error occured
     */
    int FileCatalogConsistencyCheck();
 
    /*!
     * Function to collect summary statistics on the File Catalog
     * \param[out] files_count Number of files in the File Catalog
     * \param[out] files_without_replicas Number of files without replicas
     * \param[out] replicas_count Number of replicas in the File Catalog
     * \param[out] replicas_without_files Number of replicas without files in the File Catalog
     * \return 0 - if successfull, <0 - an error occured
     */
    int FileCatalogStats(int& files_count,
                         int& files_without_replicas,
                         int& replicas_count,
                         int& replicas_without_files);
 
    /*!
     * Setting Verbose level: 0 - only errors, 1 - normal output, 2 - detailed output, 3 - all with debug info
     * \param[in] verbose_level Verbose level for output information
     */
    void SetVerbose(int verbose_level = 0) { VerboseLevel = verbose_level; }
 
  private:
    CURL* curlInstance;
    string ServiceUrl;
    string ServiceAccount;
 
    int VerboseLevel = 0;
 
    /*!
     * Perfrom common curl read/write shenanigans
     * \param[in] conditions String of conditions/path/commands which goes after URL when perfrorming API request
     * \param[in] json_content String containing json with request as append content
     * \param[in] qType type of request
     * \return Response string from the API server
     */
    string performRequest(string conditions, string json_content, string qType);
 
    // curl call for file download
    int DownloadFile(string conditions, string savePath);
 
    // curl call for file upload
    int UploadFile(string conditions, string filepath);
 
    // ClassDef(FileCatalogue, 1)
};
 
#endif /* !FileCatalog_H */