compare_files.C

Go to the documentation of this file.

// Compare the data content of two CERN ROOT files.
//
// Even "identical" ROOT files differ binary-wise: the file header stores the file
// name and modification time, and every TKey/TBasket stores its creation time.
// Therefore the comparison is done on the deserialized data instead: every key of
// both files is read back, the objects are serialized into memory buffers and
// compared byte by byte; TTrees are compared entry by entry, branch by branch.
// NB: full comparison of user classes requires their dictionaries to be loaded.
//
// If size_only = kTRUE, a coarse comparison is performed instead, which does not
// require any class dictionaries: only the number of entries and the uncompressed
// data size of every branch and of every non-tree object are compared.
// NB: equal sizes do not guarantee equal content (e.g. a changed value of a fixed-size type)
//
// The same data can also be reduced to a content hash of a single file with hash_file(), which is
// invariant against all the metadata ignored by the comparison
// NB: the hash is only as reproducible as the streamers of the stored classes are. A class whose
// serialized form contains addresses or uninitialized memory gives a different hash
//
// Usage: root -l -b -q 'compare_files.C("file1.root", "file2.root")'
//        root -l -b -q 'compare_files.C("file1.root", "file2.root", kTRUE)'
//        root -l -b -q -e '.L compare_files.C' -e 'hash_file("file.root")'
//        root -l -b -q -e '.L compare_files.C' -e 'hash_file("file.root", kTRUE)'
 
#include "TBranchElement.h"
#include "TBufferFile.h"
#include "TClass.h"
#include "TFile.h"
#include "TGeoManager.h"
#include "TH1.h"
#include "TKey.h"
#include "TLeaf.h"
#include "TMD5.h"
#include "TROOT.h"
#include "TSystem.h"
#include "TTree.h"
 
#include <cstring>
#include <iostream>
#include <map>
#include <set>
#include <string>
using namespace std;
 
// Keys holding a geometry (a TGeoManager or a parameter set containing one, e.g. FairGeoParSet)
// build that geometry while they are read, and TGeoManager::Init() deletes the geometry the global
// pointer refers to - which here is the object read from the first file and still needed. Detaching
// the global pointer before every read leaves the previous geometry alone.
void DetachGeometry()
{
    gGeoManager = nullptr;
}
 
// Make ROOT forget the geometries read by this macro before the process ends, so that none of them
// is deleted twice on teardown. They are leaked on purpose: the macro is short-lived, and their
// owners (the parameter sets) may have released them already.
void ForgetGeometries()
{
    gGeoManager = nullptr;
    if (gROOT->GetListOfGeometries() != nullptr)
        gROOT->GetListOfGeometries()->Clear("nodelete");
}
 
// serialize two objects of the same class into memory buffers and compare them byte by byte
bool AreObjectsEqual(void* obj1, void* obj2, TClass* obj_class)
{
    if ((obj1 == nullptr) || (obj2 == nullptr))
        return obj1 == obj2;
 
    // stream via the known class directly: unlike WriteObjectAny, this does not try to detect
    // the actual class via the virtual IsA(), which fails for classes without a dictionary
    TBufferFile buffer1(TBuffer::kWrite);
    TBufferFile buffer2(TBuffer::kWrite);
    obj_class->Streamer(obj1, buffer1);
    obj_class->Streamer(obj2, buffer2);
 
    if (buffer1.Length() != buffer2.Length())
        return false;
    return memcmp(buffer1.Buffer(), buffer2.Buffer(), buffer1.Length()) == 0;
}
 
// compare the number of entries and the uncompressed data size of a branch and all its sub-branches
bool CompareBranchSizes(TBranch* branch1, TBranch* branch2, const TString& obj_path)
{
    bool is_equal = true;
 
    if (branch1->GetEntries() != branch2->GetEntries()) {
        cout << "DIFF " << obj_path << ": branch '" << branch1->GetName() << "' has different number of entries ("
             << branch1->GetEntries() << " vs " << branch2->GetEntries() << ")" << endl;
        is_equal = false;
    } else if (branch1->GetTotBytes() != branch2->GetTotBytes()) {
        cout << "DIFF " << obj_path << ": branch '" << branch1->GetName() << "' has different data size ("
             << branch1->GetTotBytes() << " vs " << branch2->GetTotBytes() << " bytes uncompressed)" << endl;
        is_equal = false;
    }
 
    TObjArray* sub_branches1 = branch1->GetListOfBranches();
    TObjArray* sub_branches2 = branch2->GetListOfBranches();
    if (sub_branches1->GetEntries() != sub_branches2->GetEntries()) {
        cout << "DIFF " << obj_path << ": branch '" << branch1->GetName() << "' has different number of sub-branches ("
             << sub_branches1->GetEntries() << " vs " << sub_branches2->GetEntries() << ")" << endl;
        return false;
    }
    for (Int_t i = 0; i < sub_branches1->GetEntries(); i++) {
        TBranch* sub_branch1 = (TBranch*)sub_branches1->At(i);
        TBranch* sub_branch2 = (TBranch*)sub_branches2->FindObject(sub_branch1->GetName());
        if (sub_branch2 == nullptr) {
            cout << "DIFF " << obj_path << ": branch '" << sub_branch1->GetName() << "' is missing in the second file"
                 << endl;
            is_equal = false;
        } else if (!CompareBranchSizes(sub_branch1, sub_branch2, obj_path))
            is_equal = false;
    }
 
    return is_equal;
}
 
// compare two trees: entry by entry, branch by branch (or only entry counts and branch sizes if size_only)
bool CompareTrees(TTree* tree1, TTree* tree2, const TString& obj_path, bool size_only)
{
    Long64_t entry_count = tree1->GetEntries();
    if (entry_count != tree2->GetEntries()) {
        cout << "DIFF " << obj_path << ": different number of entries (" << entry_count << " vs " << tree2->GetEntries()
             << ")" << endl;
        return false;
    }
 
    // compare the branch structure
    TObjArray* branches1 = tree1->GetListOfBranches();
    Int_t branch_count = branches1->GetEntries();
    if (branch_count != tree2->GetListOfBranches()->GetEntries()) {
        cout << "DIFF " << obj_path << ": different number of branches (" << branch_count << " vs "
             << tree2->GetListOfBranches()->GetEntries() << ")" << endl;
        return false;
    }
    for (Int_t i = 0; i < branch_count; i++) {
        TBranch* branch1 = (TBranch*)branches1->At(i);
        if (tree2->GetBranch(branch1->GetName()) == nullptr) {
            cout << "DIFF " << obj_path << ": branch '" << branch1->GetName() << "' is missing in the second file"
                 << endl;
            return false;
        }
    }
 
    // coarse mode: compare only the entry counts and uncompressed data sizes of all branches
    if (size_only) {
        bool is_equal = true;
        for (Int_t i = 0; i < branch_count; i++) {
            TBranch* branch1 = (TBranch*)branches1->At(i);
            if (!CompareBranchSizes(branch1, tree2->GetBranch(branch1->GetName()), obj_path))
                is_equal = false;
        }
        if (is_equal)
            cout << "OK   " << obj_path << " (TTree): " << entry_count << " entries, same branch sizes ("
                 << tree1->GetTotBytes() << " bytes uncompressed)" << endl;
        return is_equal;
    }
 
    for (Long64_t entry = 0; entry < entry_count; entry++) {
        // read the whole entry in both trees first (branch addresses are allocated by ROOT itself),
        // so that leaf-count branches are up to date before any data are compared
        for (Int_t i = 0; i < branch_count; i++) {
            TBranch* branch1 = (TBranch*)branches1->At(i);
            branch1->GetEntry(entry);
            tree2->GetBranch(branch1->GetName())->GetEntry(entry);
        }
 
        for (Int_t i = 0; i < branch_count; i++) {
            TBranch* branch1 = (TBranch*)branches1->At(i);
            TBranch* branch2 = tree2->GetBranch(branch1->GetName());
 
            // object branches: compare the deserialized objects
            TBranchElement* branch_el1 = dynamic_cast<TBranchElement*>(branch1);
            TBranchElement* branch_el2 = dynamic_cast<TBranchElement*>(branch2);
            if ((branch_el1 != nullptr) && (branch_el2 != nullptr) && (branch_el1->GetObject() != nullptr)
                && (branch_el2->GetObject() != nullptr))
            {
                if (strcmp(branch_el1->GetClassName(), branch_el2->GetClassName()) != 0) {
                    cout << "DIFF " << obj_path << ": branch '" << branch1->GetName() << "' has different classes ("
                         << branch_el1->GetClassName() << " vs " << branch_el2->GetClassName() << ")" << endl;
                    return false;
                }
                TClass* branch_class = TClass::GetClass(branch_el1->GetClassName());
                if (!AreObjectsEqual(branch_el1->GetObject(), branch_el2->GetObject(), branch_class)) {
                    cout << "DIFF " << obj_path << ": branch '" << branch1->GetName() << "' differs at entry " << entry
                         << endl;
                    return false;
                }
                continue;
            }
 
            // branches of basic types: compare the leaf data
            TIter next_leaf(branch1->GetListOfLeaves());
            TLeaf* leaf1;
            while ((leaf1 = (TLeaf*)next_leaf()) != nullptr) {
                TLeaf* leaf2 = branch2->GetLeaf(leaf1->GetName());
                if ((leaf2 == nullptr) || (leaf1->GetLen() != leaf2->GetLen())
                    || (memcmp(leaf1->GetValuePointer(), leaf2->GetValuePointer(),
                               leaf1->GetLen() * leaf1->GetLenType())
                        != 0))
                {
                    cout << "DIFF " << obj_path << ": leaf '" << branch1->GetName() << "/" << leaf1->GetName()
                         << "' differs at entry " << entry << endl;
                    return false;
                }
            }
        }
    }
 
    cout << "OK   " << obj_path << " (TTree): " << entry_count << " entries are identical" << endl;
    return true;
}
 
// compare all keys of two directories (the highest cycle of each key), recursing into subdirectories
bool CompareDirectories(TDirectory* dir1, TDirectory* dir2, const TString& dir_path, bool size_only)
{
    bool is_equal = true;
 
    set<string> key_names1, key_names2;
    TIter next_key1(dir1->GetListOfKeys());
    TKey* key;
    while ((key = (TKey*)next_key1()) != nullptr)
        key_names1.insert(key->GetName());
    TIter next_key2(dir2->GetListOfKeys());
    while ((key = (TKey*)next_key2()) != nullptr)
        key_names2.insert(key->GetName());
 
    for (const string& key_name : key_names1)
        if (key_names2.count(key_name) == 0) {
            cout << "DIFF " << dir_path << key_name << ": present only in the first file" << endl;
            is_equal = false;
        }
    for (const string& key_name : key_names2)
        if (key_names1.count(key_name) == 0) {
            cout << "DIFF " << dir_path << key_name << ": present only in the second file" << endl;
            is_equal = false;
        }
 
    for (const string& key_name : key_names1) {
        if (key_names2.count(key_name) == 0)
            continue;
 
        TKey* key1 = dir1->GetKey(key_name.c_str());
        TKey* key2 = dir2->GetKey(key_name.c_str());
        TString obj_path = dir_path + key_name;
 
        if (strcmp(key1->GetClassName(), key2->GetClassName()) != 0) {
            cout << "DIFF " << obj_path << ": different classes (" << key1->GetClassName() << " vs "
                 << key2->GetClassName() << ")" << endl;
            is_equal = false;
            continue;
        }
 
        TClass* obj_class = TClass::GetClass(key1->GetClassName());
 
        // TProcessID contains only a time-based GUID, i.e. pure metadata
        if ((obj_class != nullptr) && obj_class->InheritsFrom("TProcessID"))
            continue;
 
        if ((obj_class != nullptr) && obj_class->InheritsFrom(TTree::Class())) {
            TTree* tree1 = (TTree*)key1->ReadObj();
            TTree* tree2 = (TTree*)key2->ReadObj();
            if (!CompareTrees(tree1, tree2, obj_path, size_only))
                is_equal = false;
        } else if ((obj_class != nullptr) && obj_class->InheritsFrom(TDirectory::Class())) {
            if (!CompareDirectories(dir1->GetDirectory(key_name.c_str()), dir2->GetDirectory(key_name.c_str()),
                                    obj_path + "/", size_only))
                is_equal = false;
        } else if (size_only) {
            // coarse mode: compare only the uncompressed size of the stored object (no dictionary is required)
            if (key1->GetObjlen() == key2->GetObjlen())
                cout << "OK   " << obj_path << " (" << key1->GetClassName() << "): same size of " << key1->GetObjlen()
                     << " bytes" << endl;
            else {
                cout << "DIFF " << obj_path << " (" << key1->GetClassName() << "): different size ("
                     << key1->GetObjlen() << " vs " << key2->GetObjlen() << " bytes uncompressed)" << endl;
                is_equal = false;
            }
        } else if (obj_class == nullptr) {
            cout << "WARN " << obj_path << ": unknown class " << key1->GetClassName() << ", the key is skipped" << endl;
        } else {
            DetachGeometry();
            void* obj1 = key1->ReadObjectAny(obj_class);
            DetachGeometry();
            void* obj2 = key2->ReadObjectAny(obj_class);
            if (AreObjectsEqual(obj1, obj2, obj_class))
                cout << "OK   " << obj_path << " (" << key1->GetClassName() << ")" << endl;
            else {
                cout << "DIFF " << obj_path << " (" << key1->GetClassName() << "): content differs" << endl;
                is_equal = false;
            }
            if (obj1 != nullptr)
                obj_class->Destructor(obj1);
            if (obj2 != nullptr)
                obj_class->Destructor(obj2);
        }
    }
 
    return is_equal;
}
 
// ------------------------------------------------------------------------------------------------
// Content hash of a single file: exactly the same data that are compared above are fed, in a
// canonical (name-sorted) order, into an MD5 digest, while all metadata are left out. Two files
// with equal data content therefore get the same hash although their bytes on disk differ, i.e.
// the guarantee is: if compare_files() reports IDENTICAL, the hashes of both files are equal.
// The hash of a file written with size_only = kTRUE is a different, coarser one and must never be
// compared with a full content hash.
// ------------------------------------------------------------------------------------------------
 
// tag of the hashing scheme: any change of the code below must change this tag, so that hashes
// produced by different versions of this macro are never compared with each other
static const char* hash_scheme_tag = "bmnroot-hash-1";
 
// feed a text token into the digest (the separator prevents ambiguous concatenations)
void HashText(TMD5& digest, const char* text)
{
    digest.Update((const UChar_t*)text, strlen(text));
    digest.Update((const UChar_t*)"\n", 1);
}
 
// feed a number into the digest as text, so that the hash does not depend on the byte order
void HashNumber(TMD5& digest, Long64_t number)
{
    HashText(digest, Form("%lld", number));
}
 
// feed a length-prefixed memory block into the digest
void HashBlock(TMD5& digest, const void* data, Int_t length)
{
    HashNumber(digest, length);
    if (length > 0)
        digest.Update((const UChar_t*)data, length);
}
 
// feed the serialized representation of an object into the digest (see AreObjectsEqual)
void HashObject(TMD5& digest, void* obj, TClass* obj_class)
{
    if ((obj == nullptr) || (obj_class == nullptr)) {
        HashText(digest, "no-object");
        return;
    }
 
    TBufferFile buffer(TBuffer::kWrite);
    obj_class->Streamer(obj, buffer);
    HashBlock(digest, buffer.Buffer(), buffer.Length());
}
 
// feed the number of entries and the uncompressed data size of a branch and all its sub-branches into the digest
void HashBranchSizes(TMD5& digest, TBranch* branch)
{
    HashText(digest, branch->GetName());
    HashNumber(digest, branch->GetEntries());
    HashNumber(digest, branch->GetTotBytes());
 
    // the sub-branches are hashed in alphabetical order, so that the digest does not depend on their storage order
    map<string, TBranch*> sub_branches;
    TObjArray* sub_branch_list = branch->GetListOfBranches();
    for (Int_t i = 0; i < sub_branch_list->GetEntries(); i++) {
        TBranch* sub_branch = (TBranch*)sub_branch_list->At(i);
        sub_branches[sub_branch->GetName()] = sub_branch;
    }
    HashNumber(digest, sub_branches.size());
    for (const auto& sub_branch : sub_branches)
        HashBranchSizes(digest, sub_branch.second);
}
 
// feed a tree into the digest: entry by entry, branch by branch (or only entry counts and branch sizes if size_only)
void HashTree(TMD5& digest, TTree* tree, bool size_only)
{
    Long64_t entry_count = tree->GetEntries();
    HashNumber(digest, entry_count);
 
    // the branches are hashed in alphabetical order, so that the digest does not depend on their storage order
    map<string, TBranch*> branches;
    TObjArray* branch_list = tree->GetListOfBranches();
    for (Int_t i = 0; i < branch_list->GetEntries(); i++) {
        TBranch* branch = (TBranch*)branch_list->At(i);
        branches[branch->GetName()] = branch;
    }
    HashNumber(digest, branches.size());
 
    // coarse mode: hash only the entry counts and uncompressed data sizes of all branches
    if (size_only) {
        for (const auto& branch : branches)
            HashBranchSizes(digest, branch.second);
        return;
    }
 
    for (Long64_t entry = 0; entry < entry_count; entry++) {
        // read the whole entry first (branch addresses are allocated by ROOT itself),
        // so that leaf-count branches are up to date before any data are hashed
        for (const auto& branch : branches)
            branch.second->GetEntry(entry);
 
        for (const auto& branch : branches) {
            HashText(digest, branch.first.c_str());
 
            // object branches: hash the deserialized objects
            TBranchElement* branch_el = dynamic_cast<TBranchElement*>(branch.second);
            if ((branch_el != nullptr) && (branch_el->GetObject() != nullptr)) {
                HashText(digest, branch_el->GetClassName());
                HashObject(digest, branch_el->GetObject(), TClass::GetClass(branch_el->GetClassName()));
                continue;
            }
 
            // branches of basic types: hash the leaf data
            TIter next_leaf(branch.second->GetListOfLeaves());
            TLeaf* leaf;
            while ((leaf = (TLeaf*)next_leaf()) != nullptr) {
                HashText(digest, leaf->GetName());
                if (leaf->GetValuePointer() == nullptr)
                    HashText(digest, "no-value");
                else
                    HashBlock(digest, leaf->GetValuePointer(), leaf->GetLen() * leaf->GetLenType());
            }
        }
    }
}
 
// feed all keys of a directory (the highest cycle of each key) into the digest, recursing into subdirectories
void HashDirectory(TMD5& digest, TDirectory* dir, const TString& dir_path, bool size_only)
{
    // the keys are hashed in alphabetical order, so that the digest does not depend on their storage order
    set<string> key_names;
    TIter next_key(dir->GetListOfKeys());
    TKey* key;
    while ((key = (TKey*)next_key()) != nullptr)
        key_names.insert(key->GetName());
 
    for (const string& key_name : key_names) {
        TKey* dir_key = dir->GetKey(key_name.c_str());
        TString obj_path = dir_path + key_name;
        TClass* obj_class = TClass::GetClass(dir_key->GetClassName());
 
        // TProcessID contains only a time-based GUID, i.e. pure metadata
        if ((obj_class != nullptr) && obj_class->InheritsFrom("TProcessID"))
            continue;
 
        // objects of unknown classes are skipped by the comparison as well, otherwise files reported
        // as identical could get different hashes
        if ((obj_class == nullptr) && !size_only) {
            cout << "WARN " << obj_path << ": unknown class " << dir_key->GetClassName() << ", the key is not hashed"
                 << endl;
            continue;
        }
 
        HashText(digest, obj_path.Data());
        HashText(digest, dir_key->GetClassName());
 
        if ((obj_class != nullptr) && obj_class->InheritsFrom(TTree::Class()))
            HashTree(digest, (TTree*)dir_key->ReadObj(), size_only);
        else if ((obj_class != nullptr) && obj_class->InheritsFrom(TDirectory::Class()))
            HashDirectory(digest, dir->GetDirectory(key_name.c_str()), obj_path + "/", size_only);
        else if (size_only)
            // coarse mode: hash only the uncompressed size of the stored object (no dictionary is required)
            HashNumber(digest, dir_key->GetObjlen());
        else {
            DetachGeometry();
            void* obj = dir_key->ReadObjectAny(obj_class);
            HashObject(digest, obj, obj_class);
            if (obj != nullptr)
                obj_class->Destructor(obj);
        }
    }
}
 
// compute the hash of the data content of a ROOT file ignoring metadata
// if size_only = kTRUE, the coarser hash of the entry counts and
// uncompressed data sizes is computed instead (no class dictionaries required)
TString hash_file(TString root_file_path, Bool_t size_only = kFALSE)
{
    gSystem->ExpandPathName(root_file_path);
 
    if (gSystem->AccessPathName(root_file_path.Data())) {
        cerr << "Error: file does not exist or is not accessible: " << root_file_path << endl;
        return "";
    }
 
    TFile* root_file = TFile::Open(root_file_path);
    if ((root_file == nullptr) || root_file->IsZombie()) {
        cerr << "Error: could not open ROOT file: " << root_file_path << endl;
        return "";
    }
 
    // do not attach histograms being read to the current directory, so they can be safely deleted
    TH1::AddDirectory(kFALSE);
 
    TMD5 digest;
    HashText(digest, hash_scheme_tag);
    HashText(digest, size_only ? "size-only" : "content");
    HashDirectory(digest, root_file, "/", size_only);
    digest.Final();
 
    TString hash = digest.AsString();
    cout << hash << "  " << root_file_path << (size_only ? "  (sizes only)" : "  (data content)") << endl;
 
    ForgetGeometries();
    root_file->Close();
    return hash;
}
 
// compare the data content of two ROOT files ignoring metadata (timestamps in the file header, TKey and TBasket);
// if size_only = kTRUE, only entry counts and uncompressed data sizes are compared (no class dictionaries required)
void compare_files(TString root_file_path1, TString root_file_path2, Bool_t size_only = kFALSE)
{
    gSystem->ExpandPathName(root_file_path1);
    gSystem->ExpandPathName(root_file_path2);
 
    // check that the files exist and can be opened
    if (gSystem->AccessPathName(root_file_path1.Data())) {
        cerr << "Error: file does not exist or is not accessible: " << root_file_path1 << endl;
        return;
    }
    if (gSystem->AccessPathName(root_file_path2.Data())) {
        cerr << "Error: file does not exist or is not accessible: " << root_file_path2 << endl;
        return;
    }
 
    TFile* root_file1 = TFile::Open(root_file_path1);
    if ((root_file1 == nullptr) || root_file1->IsZombie()) {
        cerr << "Error: could not open ROOT file: " << root_file_path1 << endl;
        return;
    }
    TFile* root_file2 = TFile::Open(root_file_path2);
    if ((root_file2 == nullptr) || root_file2->IsZombie()) {
        cerr << "Error: could not open ROOT file: " << root_file_path2 << endl;
        root_file1->Close();
        return;
    }
 
    // do not attach histograms being read to the current directory, so they can be safely deleted
    TH1::AddDirectory(kFALSE);
 
    bool is_equal = CompareDirectories(root_file1, root_file2, "/", size_only);
 
    cout << endl;
    if (is_equal)
        cout << "The files are IDENTICAL ("
             << (size_only ? "by entry counts and data sizes only" : "data content, ignoring metadata") << ")" << endl;
    else
        cout << "The files are DIFFERENT" << endl;
 
    ForgetGeometries();
    root_file1->Close();
    root_file2->Close();
}