BmnReaderCacheManager.h

Go to the documentation of this file.

#pragma once
#ifndef BmnReaderCacheManager_h
#define BmnReaderCacheManager_h
 
// headers from this target
#include "BmnIArchiveLibRead.h"
// std:: headers
#include <array>
#include <cassert>
#include <cstring>
#include <filesystem>
#include <fstream>
// FairLogger headers
#include <fairlogger/Logger.h>
 
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
// ================== BmnReaderCacheManager class ==================
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
 
/*
 * This class should be used for easier management of various compression formats. That is, it
 * handles input-file reads, keeps the needed data in cache for optimal (fast) reads in subsequent
 * read calls while still relying on an owner class that inherits from `BmnIArchiveLibRead`.
 */
template<size_t SourceBufferSize = 32 * 1024ull, size_t DestinationBufferSize = 64 * 1024ull>
class BmnReaderCacheManager final
{
    static_assert(SourceBufferSize >= 32 * 1024ull && DestinationBufferSize >= 32 * 1024ull,
                  "Don't reduce buffer sizes too much as you will lose efficiency!");
    static_assert(SourceBufferSize <= DestinationBufferSize,
                  "For uncompressing, uncompressed buffers are expected to be bigger than compressed buffers!");
 
  public:
    BmnReaderCacheManager() = delete;
    BmnReaderCacheManager(BmnIArchiveLibRead&);
    ~BmnReaderCacheManager() = default;
    // We also don't want to copy the data
    BmnReaderCacheManager(const BmnReaderCacheManager&) = delete;
    BmnReaderCacheManager& operator=(const BmnReaderCacheManager&) = delete;
    // removing move constructor/assignment as I'm using arrays, i.e., I don't want to jump all over the heap for no
    // reason. One should use the class without any pointers (no need for them), while the owner class should be
    // allocated on the heap
    BmnReaderCacheManager(BmnReaderCacheManager&&) = delete;
    BmnReaderCacheManager& operator==(BmnReaderCacheManager&&) = delete;
 
  public:
    // @return The number of bytes that are read from an input stream and fed into an underlying decompression system
    [[nodiscard]] constexpr size_t GetSourceBufferSize() const noexcept;
 
    // @return The number of bytes that are used for caching decompressed data - the data is used for fast reading
    [[nodiscard]] constexpr size_t GetDestinationBufferSize() const noexcept;
 
    // inline methods - based around a reading stream
 
    [[nodiscard]] inline bool IsOpen() const noexcept;
    [[nodiscard]] inline bool IsGood() const noexcept;
    [[nodiscard]] inline bool IsEOF() const noexcept;
    [[nodiscard]] inline int64_t Tell() noexcept;
    inline void Clear();
    // inline methods - some getters
 
    [[nodiscard]] inline size_t GetCurrentOutputBufferPosition() const noexcept;
    [[nodiscard]] inline size_t GetTotalOutputBufferFilledSize() const noexcept;
    [[nodiscard]] inline size_t GetCurrentInputBufferPosition() const noexcept;
    [[nodiscard]] inline size_t GetTotalInputBufferFilledSize() const noexcept;
 
  public:
    /*
     * A method for opening an input stream from a compressed file.
     * @param inputFile A path to the file that should be opened
     * @param inputOpenMode Appropriate std::ios::openmode flags related to the reading (mostly, those should be
     * `std::ios::in | std::ios::binary`)
     */
    void Open(const std::filesystem::path& inputFile, const std::ios::openmode inputOpenMode);
 
    /*
     * Safely closes a stream and removes unnecessary data so a new stream can be opened
     */
    void Close();
 
    /*
     * Handles reading of a next data chunk. Before calling using decompression calls, it will forward
     * all unused data from the caching buffer.
     * @param dataToFill A pointer to the data that should be filled in this method
     * @param sizeToFill A number of bytes that should be filled in `dataToFill`
     * @return A number of successfully filled bytes. If `sizeOfFill != returnSize`, the stream either reached the end,
     * or a fatal error has occurred
     */
    [[nodiscard]] int64_t Read(void* dataToFill, int64_t sizeToFill);
 
    /*
     * Seeks a new reading points in an input file
     * @param value A new reading point from an input file
     * @warning While using archives, it can be dangerous trying to access file bytes randomly
     */
    void Seek(int64_t value);
 
    /*
     * A method that helps to move to a specific place in the reading file. This method must move
     * the output stream forward. If backward move is needed, the owner should handle it, mostly by
     * reopening the stream.
     * @param value A value in an output (decompressed) stream to which one wants to jump
     * @warning This method may not be efficient and it would be best if jumping is not used while
     *          decompressing. Still, it can be used efficiently if one has compressed data with proper tactics
     */
    void MoveToDecompressingLocation(int64_t value);
 
  private:
    /*
     * Handles extraction and decompression of the next data chunk. The decompression should be performed by
     * the owner
     */
    [[nodiscard]] bool UncompressNextDataChunk();
 
  private:
    std::array<uint8_t, SourceBufferSize> fArchiveSourceBuffer;
    std::array<uint8_t, DestinationBufferSize> fArchiveOutputBuffer;
    BmnIArchiveLibRead& fOwnerArchiveReference;
    std::ifstream fFileBeingProcessed;
    size_t fCurrentOutputBufferPosition;
    size_t fTotalOutputBufferFilledSize;
    size_t fCurrentInputBufferPosition;
    size_t fTotalInputBufferFilledSize;
    /*
     * keeping it as signed so `0 - 1` wouldn't go to
     * `std::numeric_limits<size_t>::max()` and cause some unexpected error
     */
    int64_t fRemainingFileSizeToRead;
};
 
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
// ================== BmnReaderCacheManager inline methods ==================
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
 
template<size_t src, size_t dst>
constexpr size_t BmnReaderCacheManager<src, dst>::GetSourceBufferSize() const noexcept
{
    return src;
}
 
template<size_t src, size_t dst>
constexpr size_t BmnReaderCacheManager<src, dst>::GetDestinationBufferSize() const noexcept
{
    return dst;
}
 
template<size_t src, size_t dst>
bool BmnReaderCacheManager<src, dst>::IsOpen() const noexcept
{
    return fFileBeingProcessed.is_open();
}
 
template<size_t src, size_t dst>
bool BmnReaderCacheManager<src, dst>::IsGood() const noexcept
{
    return fFileBeingProcessed.good();
}
 
template<size_t src, size_t dst>
bool BmnReaderCacheManager<src, dst>::IsEOF() const noexcept
{
    return fCurrentOutputBufferPosition == fTotalOutputBufferFilledSize && fRemainingFileSizeToRead == 0ll
           && fCurrentInputBufferPosition == fTotalInputBufferFilledSize
           && fOwnerArchiveReference.HasReachedTheEndOfStream();
}
 
template<size_t src, size_t dst>
int64_t BmnReaderCacheManager<src, dst>::Tell() noexcept
{
    if (!fFileBeingProcessed.is_open() || !fFileBeingProcessed.good())
        return -1ll;
    // tellg is not const o.O
    return fFileBeingProcessed.tellg() - (int64_t)(fTotalOutputBufferFilledSize - fCurrentOutputBufferPosition);
}
 
template<size_t src, size_t dst>
void BmnReaderCacheManager<src, dst>::Clear()
{
    fFileBeingProcessed.clear();
}
 
template<size_t src, size_t dst>
size_t BmnReaderCacheManager<src, dst>::GetCurrentOutputBufferPosition() const noexcept
{
    return fCurrentOutputBufferPosition;
}
 
template<size_t src, size_t dst>
size_t BmnReaderCacheManager<src, dst>::GetTotalOutputBufferFilledSize() const noexcept
{
    return fTotalOutputBufferFilledSize;
}
 
template<size_t src, size_t dst>
size_t BmnReaderCacheManager<src, dst>::GetCurrentInputBufferPosition() const noexcept
{
    return fCurrentInputBufferPosition;
}
 
template<size_t src, size_t dst>
size_t BmnReaderCacheManager<src, dst>::GetTotalInputBufferFilledSize() const noexcept
{
    return fTotalInputBufferFilledSize;
}
 
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
// ================== BmnReaderCacheManager public methods ==================
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
 
template<size_t src, size_t dst>
BmnReaderCacheManager<src, dst>::BmnReaderCacheManager(BmnIArchiveLibRead& anOwnerArchive)
    : fArchiveSourceBuffer()
    , fArchiveOutputBuffer()
    , fOwnerArchiveReference(anOwnerArchive)
    , fFileBeingProcessed()
    , fCurrentOutputBufferPosition(0ull)
    , fTotalOutputBufferFilledSize(0ull)
    , fCurrentInputBufferPosition(0ull)
    , fTotalInputBufferFilledSize(0ull)
    , fRemainingFileSizeToRead(-1ll)
{}
 
template<size_t src, size_t dst>
void BmnReaderCacheManager<src, dst>::Open(const std::filesystem::path& inputFile,
                                           const std::ios::openmode inputOpenMode)
{
    try {
        if (!std::filesystem::exists(inputFile) || !std::filesystem::is_regular_file(inputFile)) {
            LOG(error) << "`BmnReaderCacheManager<src, dst>::Open` expects an actual existing file! Passed path: "
                       << inputFile << '\n';
            return;
        }
        if (fFileBeingProcessed.is_open()) {
            LOG(warning) << "You are about to open a new file in `BmnReaderCacheManager<src, dst>::Open` while the "
                            "previous one wasn't closed - implicitly calling `Close()`!\n";
        }
        Close();
        fRemainingFileSizeToRead = (int64_t)std::filesystem::file_size(inputFile);
    } catch (const std::bad_alloc& errorDescription) {
        LOG(error) << "Filesystem failure! Reason: " << errorDescription.what() << '\n';
        return;
    } catch (const std::filesystem::filesystem_error& errorDescription) {
        LOG(error) << "Filesystem failure! Reason: " << errorDescription.what() << '\n';
        return;
    }
    fFileBeingProcessed.open(inputFile, inputOpenMode);
}
 
template<size_t src, size_t dst>
void BmnReaderCacheManager<src, dst>::Close()
{
    if (!fFileBeingProcessed.is_open())
        return;
    fCurrentOutputBufferPosition = 0ull;
    fTotalOutputBufferFilledSize = 0ull;
    fCurrentInputBufferPosition = 0ull;
    fTotalInputBufferFilledSize = 0ull;
    fRemainingFileSizeToRead = -1ll;
    fFileBeingProcessed.close();
}
 
template<size_t tSrcSize, size_t tDstSize>
int64_t BmnReaderCacheManager<tSrcSize, tDstSize>::Read(void* dataToFill, int64_t sizeToFill)
{
    if (!fFileBeingProcessed.is_open()) {
        LOG(error) << "Calling `BmnReaderCacheManager::Read` while the input file is not ready for it!\n";
        return 0ll;
    }
    // As we work with decompression here, we won't be able to know how many raw bytes will be obtained
    // from N compressed bytes (even if we knew the compression ratio, it'd still be just an average).
    // Thus, no matter what, we will be always reading the data into the output buffer, and only then
    // copy them into `dataToFill`
    if (sizeToFill <= 0ll) {
        LOG(error) << "Calling `BmnReaderCacheManager::Read` with `sizeToFill < 0`!\n";
        return 0ll;
    }
    size_t dataToFillOffset = 0ull;
    auto fillTheBuffer = [this, &dataToFillOffset, dataToFill, &sizeToFill]() {
        // we might encounter a few extra operations here for large `sizeToFill`, but it doesn't matter
        size_t unreadBuffer = fTotalOutputBufferFilledSize - fCurrentOutputBufferPosition;
        size_t bytesToCopy = unreadBuffer > (size_t)sizeToFill ? (size_t)sizeToFill : unreadBuffer;
        std::memcpy((uint8_t*)dataToFill + dataToFillOffset, fArchiveOutputBuffer.data() + fCurrentOutputBufferPosition,
                    bytesToCopy);
        fCurrentOutputBufferPosition += bytesToCopy;
        if (fTotalOutputBufferFilledSize == fCurrentOutputBufferPosition) {
            fCurrentOutputBufferPosition = 0ull;
            fTotalOutputBufferFilledSize = 0ull;
        }
        sizeToFill -= (int64_t)bytesToCopy;
        dataToFillOffset += bytesToCopy;
    };
    if (fTotalOutputBufferFilledSize - fCurrentOutputBufferPosition > 0)
        fillTheBuffer();
    while (sizeToFill > 0ll && !IsEOF()) {
        // at this point we will either skip the loop as `sizeToFill == 0` or `fTotalOutputBufferFilledSize ==
        // fCurrentOutputBufferPosition`
        if (!UncompressNextDataChunk()) {
            LOG(error) << "Encountered an error while uncompressing in `BmnReaderCacheManager::Read`... breaking "
                          "unhandled read!\nError reason: "
                       << fOwnerArchiveReference.GetLastUncompressErrorDescription() << '\n';
            return 0ll;
        }
        // and we keep filling till we are done
        fillTheBuffer();
    }
    return (int64_t)dataToFillOffset;
}
 
template<size_t src, size_t dst>
void BmnReaderCacheManager<src, dst>::Seek(int64_t value)
{
    fFileBeingProcessed.seekg(value);
}
 
template<size_t src, size_t dst>
void BmnReaderCacheManager<src, dst>::MoveToDecompressingLocation(int64_t value)
{
    if (!fFileBeingProcessed.is_open()) {
        LOG(warning) << "Calling `BmnReaderCacheManager::MoveToDecompressingLocation` while the input file is not "
                        "ready for it!\n";
        return;
    }
    assert(value > 0ll);
    assert(fTotalInputBufferFilledSize + value >= fOwnerArchiveReference.GetNumberOfDecompressedBytes());
    while (true) {
        if ((size_t)value <= fOwnerArchiveReference.GetNumberOfDecompressedBytes()) {
            fCurrentOutputBufferPosition =
                fTotalOutputBufferFilledSize + value - fOwnerArchiveReference.GetNumberOfDecompressedBytes();
            return;
        }
        if (IsEOF())
            return;
        if (!UncompressNextDataChunk())
            return;
    }
}
 
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
// ================== BmnReaderCacheManager private methods ==================
// o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o o
 
template<size_t tSrcSize, size_t tDstSize>
bool BmnReaderCacheManager<tSrcSize, tDstSize>::UncompressNextDataChunk()
{
    if (!fFileBeingProcessed.good()) {
        // we can expect that different underlying libraries handle EOF differently, so we need to add
        // `IsStreamInGoodState` as well
        if (fFileBeingProcessed.eof() && fCurrentInputBufferPosition == fTotalInputBufferFilledSize
            && !fOwnerArchiveReference.IsStreamInGoodState())
        {
            LOG(error) << "Calling `BmnReaderCacheManager::UncompressNextDataChunk` while the stream is not good!\n";
            return false;
        }
    }
    // Here we must be very carful as some libraries might not consume entire input buffer while uncompressing,
    // i.e., the compressed data might be split into segments (packets). In that case, libraries might need entire
    // segments or they won't process the data at all
    bool shouldReadMoreStreamData = false;
    if (fCurrentInputBufferPosition != fTotalInputBufferFilledSize) {
        // if we used only a small portion of the input buffer, there's no need to move it around the memory and load
        // more... it would have reduced the code efficiency
        if (fCurrentInputBufferPosition != 0
            && (double)fCurrentInputBufferPosition > 0.75 * (double)fTotalInputBufferFilledSize)
        {
            shouldReadMoreStreamData = true;
            // using memmove here as we might encounter some overlapping while it internally handles such cases if
            // needed. if no overlapping, it will practically memcpy
            size_t nextMax = fTotalInputBufferFilledSize - fCurrentInputBufferPosition;
            std::memmove(fArchiveSourceBuffer.data(),
                         (uint8_t*)fArchiveSourceBuffer.data() + fCurrentInputBufferPosition, nextMax);
            fTotalInputBufferFilledSize = nextMax;
            fCurrentInputBufferPosition = 0ull;
        }
    } else {
        fCurrentInputBufferPosition = 0ull;
        fTotalInputBufferFilledSize = 0ull;
        shouldReadMoreStreamData = true;
    }
    if (!fFileBeingProcessed.eof() && shouldReadMoreStreamData) {
        // Now we read the raw data from the stream
        fFileBeingProcessed.read(reinterpret_cast<char*>(fArchiveSourceBuffer.data()) + fTotalInputBufferFilledSize,
                                 (int64_t)(tSrcSize - fTotalInputBufferFilledSize));
        // std::ofstream internally checks the file size and stops reading when EOF is reached, even when we ask
        // additional bytes -> `gcount()`
        fTotalInputBufferFilledSize += (size_t)fFileBeingProcessed.gcount();
        fRemainingFileSizeToRead -= fFileBeingProcessed.gcount();
    }
    size_t noOfUsedBytes = 0ull;
    BmnFileStreamResults decompressStatus =
        fOwnerArchiveReference.Uncompress(fArchiveSourceBuffer.data() + fCurrentInputBufferPosition,
                                          fTotalInputBufferFilledSize - fCurrentInputBufferPosition, noOfUsedBytes,
                                          fArchiveOutputBuffer.data(), tDstSize, fTotalOutputBufferFilledSize);
    fCurrentInputBufferPosition += noOfUsedBytes;
    if (decompressStatus == BmnFileStreamResults::Success || decompressStatus == BmnFileStreamResults::ReachedEOF) {
        return true;
    }
    LOG(error) << "Encountered an error while uncompressing in `BmnReaderCacheManager::Read`... breaking unhandled "
                  "read!\nError reason: "
               << fOwnerArchiveReference.GetLastUncompressErrorDescription() << '\n';
    return false;
}
 
#endif   // BmnReaderCacheManager_h