BmnFileReaderZStd.cxx

Go to the documentation of this file.

// headers from this target
#include "BmnFileReaderZStd.h"
 
#include "BmnDefinitions.h"
// std:: headers
#include <cassert>
#include <filesystem>
// FairLogger headers
#include <fairlogger/Logger.h>
// the main Zstd header
#ifdef BMNCOMPRESSION_WITH_ZSTD
#include <zstd.h>
#else
// If we build without the library, it would be find to use #pragma on the entire object, without push
// and pop, but it's nothing too important for us here - just for simplicity I'm going with this def
DISABLE_UNUSED_PARAM_WARNING
#endif   // BMNCOMPRESSION_WITH_ZSTD
 
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
// ================== BmnFileReaderZStd public methods ==================
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
 
// ========== BmnIFileReaderInterface ==========
 
BmnFileReaderZStd::BmnFileReaderZStd(const std::string& fileName)
    : BmnIFileReader(BmnFileType::ZStd, fileName)
    , fZstdStream(nullptr)
    , fArchiveReadManager(*this)
    , fNumberOfReadBytes(0ull)
    , fLastReadReturnValue(0ull)
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    // Keeping it hidden so ZStd would be PRIVATE in CMake
    static_assert(ZSTD_VERSION_MAJOR >= 1 && ZSTD_VERSION_MINOR > 2,
                  "You are trying to use an old version of ZStd - in that version `ZSTD_DStream` was not the same as "
                  "`ZSTD_DCtx_s`!");
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
BmnFileReaderZStd::BmnFileReaderZStd(std::string&& fileName)
    : BmnIFileReader(BmnFileType::ZStd, std::move(fileName))
    , fZstdStream(nullptr)
    , fArchiveReadManager(*this)
    , fNumberOfReadBytes(0ull)
    , fLastReadReturnValue(0ull)
{}
 
BmnFileReaderZStd::BmnFileReaderZStd(const std::filesystem::path& filePath)
    : BmnIFileReader(BmnFileType::ZStd, filePath)
    , fZstdStream(nullptr)
    , fArchiveReadManager(*this)
    , fNumberOfReadBytes(0ull)
    , fLastReadReturnValue(0ull)
{}
 
BmnFileReaderZStd::BmnFileReaderZStd(std::filesystem::path&& filePath)
    : BmnIFileReader(BmnFileType::ZStd, std::move(filePath))
    , fZstdStream(nullptr)
    , fArchiveReadManager(*this)
    , fNumberOfReadBytes(0ull)
    , fLastReadReturnValue(0ull)
{}
 
BmnFileReaderZStd::~BmnFileReaderZStd()
{
    BmnFileReaderZStd::Close();
}
 
void BmnFileReaderZStd::Open(const BmnBinaryFileProcessingOptions processingFlags)
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    /*
    While we actually don't need so many comments and checks here, we are still performing them just to ensure
    one doesn't make an accidental mistake - it might be easier to catch it
    */
    if (fZstdStream) {
        LOG(error) << "ZStd stream is already valid! The purpose of the BMN ZStd classes is to compress/decompress "
                      "huge files - if you however\n"
                      "really need to support compressing/decompressing many small files (frames), please let us know "
                      "and we will allow it along\n"
                      "with dictionaries (preferred for small files)\n";
        return;
    }
    if (!PerformStandardChecksBeforeOpening(processingFlags))
        return;
    fArchiveReadManager.Open(fFilePath, std::ios::in | std::ios::binary);
    fZstdStream = ZSTD_createDStream();
#else
    throw std::runtime_error{"Trying to use ZSTD while BMN is not compiled with it!"};
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
void BmnFileReaderZStd::Close() noexcept
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    if (!fZstdStream)
        return;
    assert(fArchiveReadManager.IsOpen());
    fLastReadReturnValue = ZSTD_freeDStream(fZstdStream);
    if (ZSTD_isError(fLastReadReturnValue)) {
        LOG(fatal) << "Failed to free ZStd stream - possible huge memory leak! Failure reason: "
                   << ZSTD_getErrorName(fLastReadReturnValue) << '\n';
        std::terminate();
    }
    fZstdStream = nullptr;
    fArchiveReadManager.Close();
    fNumberOfReadBytes = 0ull;
    fLastReadReturnValue = 0ull;
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
int64_t BmnFileReaderZStd::Read(void* dataToFill, const int64_t sizeToFill)
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    if (ZSTD_isError(fLastReadReturnValue))
        return 0ll;
    return fArchiveReadManager.Read(dataToFill, sizeToFill);
#else
    throw std::runtime_error{"Trying to use ZSTD while BMN is not compiled with it!"};
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
int64_t BmnFileReaderZStd::Tell() const noexcept
{
    return fNumberOfReadBytes + fArchiveReadManager.GetCurrentOutputBufferPosition()
           - fArchiveReadManager.GetTotalOutputBufferFilledSize();
}
 
void BmnFileReaderZStd::Seek(int64_t value)
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    if (value < 0ll || (size_t)value == fNumberOfReadBytes)
        return;
    if (ZSTD_isError(fLastReadReturnValue)) {
        LOG(error) << "Before calling `BmnFileReaderZStd::Seek` ensure to clear bad flags!\n";
        return;
    }
    if (!fZstdStream) {
        LOG(error) << "Before calling `BmnFileReaderZStd::Seek` ensure to open the stream!\n";
        return;
    }
    if (!fArchiveReadManager.IsOpen()) {
        LOG(error) << "Reading file missing while calling `BmnFileReaderZStd::Seek`\n";
        return;
    }
    if ((size_t)value < (fNumberOfReadBytes - fArchiveReadManager.GetTotalOutputBufferFilledSize())) {
        Close();
        Open(BmnBinaryFileProcessingOptions::ReadBinary);
    }
    // in theory I could check IsOpen again for unexpected errors, but....
    fArchiveReadManager.MoveToDecompressingLocation(value);
#else
    throw std::runtime_error{"Trying to use ZSTD while BMN is not compiled with it!"};
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
bool BmnFileReaderZStd::IsGood() const noexcept
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    return fZstdStream && !ZSTD_isError(fLastReadReturnValue) && fArchiveReadManager.IsGood();
#else
    return false;
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
bool BmnFileReaderZStd::IsEOF() const noexcept
{
    return fArchiveReadManager.IsEOF();
}
 
void BmnFileReaderZStd::ClearStateFlags()
{
    // I don't like this method much
    fLastReadReturnValue = 0ull;
    fArchiveReadManager.Clear();
}
 
bool BmnFileReaderZStd::IsOpen() const noexcept
{
    return fArchiveReadManager.IsOpen() && fZstdStream;
}
 
// ========== BmnIArchiveLib Interface ==========
 
BmnFileStreamResults BmnFileReaderZStd::Uncompress(const void* source,
                                                   const size_t sourceTotalSize,
                                                   size_t& numberOfReadBytes,
                                                   void* output,
                                                   const size_t outputTotalSize,
                                                   size_t& numberOfSetOutputBytes)
{
    if (!fZstdStream)
        return BmnFileStreamResults::FileNotOpen;
#ifdef BMNCOMPRESSION_WITH_ZSTD
    if (ZSTD_isError(fLastReadReturnValue))
        return BmnFileStreamResults::TryingToReadBadStream;
    ZSTD_outBuffer outBuffer{output, outputTotalSize, 0ull};
    ZSTD_inBuffer inBuffer{source, sourceTotalSize, 0ull};
    fLastReadReturnValue = ZSTD_decompressStream(fZstdStream, &outBuffer, &inBuffer);
    if (ZSTD_isError(fLastReadReturnValue))
        return BmnFileStreamResults::UnexpectedFail;
    numberOfReadBytes = inBuffer.pos;
    numberOfSetOutputBytes = outBuffer.pos;
    fNumberOfReadBytes += outBuffer.pos;
    if (fLastReadReturnValue == 0ull)
        return BmnFileStreamResults::ReachedEOF;
    return BmnFileStreamResults::Success;
#else
    throw std::runtime_error{"Trying to use ZSTD while BMN is not compiled with it!"};
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
std::string BmnFileReaderZStd::GetLastUncompressErrorDescription() const noexcept
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    std::string strToReturn{};
    if (ZSTD_isError(fLastReadReturnValue))
        strToReturn = std::string{ZSTD_getErrorName(fLastReadReturnValue)};
    return strToReturn;
#else
    return std::string{"BMN NOT BUILT WITH ZSTD"};
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
bool BmnFileReaderZStd::JumpsAllowed() const noexcept
{
    return false;
}
 
void BmnFileReaderZStd::JumpTo(size_t)
{
    /*
    Most compression libraries (including this one), don't like (read allow) jumping over the files and random
    reads. In Zstd, it would be possible to implement it, but only at the compression level. In other words, we
    would need to write the data in independent blocks or something similar. As we don't know what kind of a
    compressed file we are currently reading, we cannot rely on that and we must read all the data to get to the
    needed location - which is also what we currently need in BMN. If for some reason one needs to take some
    other approach and allow fast random access, inherit this class and override this method
    */
    LOG(warning) << "Calling `BmnFileReaderZStd::JumpTo` while the type doesn't allow jumping!\n";
}
 
size_t BmnFileReaderZStd::GetNumberOfDecompressedBytes() const noexcept
{
    return fNumberOfReadBytes;
}
 
bool BmnFileReaderZStd::HasReachedTheEndOfStream() const noexcept
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    if (ZSTD_isError(fLastReadReturnValue))
        return true;
    return fLastReadReturnValue == 0;
#else
    return true;
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
bool BmnFileReaderZStd::IsStreamInGoodState() const noexcept
{
#ifdef BMNCOMPRESSION_WITH_ZSTD
    return !ZSTD_isError(fLastReadReturnValue);
#else
    return false;
#endif   // BMNCOMPRESSION_WITH_ZSTD
}
 
#ifndef BMNCOMPRESSION_WITH_ZSTD
RESTORE_UNUSED_PARAM_WARNING
#endif   // BMNCOMPRESSION_WITH_ZSTD