Repository: ClickHouse/ClickHouse
File: src/Storages/MergeTree/IMergeTreeDataPart.cpp (C++)

Code before:
}

void IMergeTreeDataPart::removeIndexMarksFromCache(MarkCache * index_mark_cache) const
{
    if (!index_mark_cache)
        return;

    auto metadata_snapshot = storage.getInMemoryMetadataPtr();
    const auto & secondary_indices = metadata_snapshot->getSecondaryIndices();
    if (secondary_indices.empty())
        return;

    for (const auto & index_description : secondary_indices)
    {
        auto skip_index = MergeTreeIndexFactory::instance().get(index_description);
        auto index_name = skip_index->getFileName();
        auto index_format = skip_index->getDeserializedFormat(checksums, index_name);

        if (!index_format)
            continue;

        for (const auto & substream : index_format.substreams)
        {
            auto stream_name = getStreamNameOrHash(index_name + substream.suffix, substream.extension, checksums);
            if (!stream_name)
                continue;

            auto marks_file = index_granularity_info.getMarksFilePath(*stream_name);
            auto key = MarkCache::hash(getDataPartStorage().getDiskName() + ":" + (fs::path(getRelativePathOfActivePart()) / marks_file).string());
            index_mark_cache->remove(key);
        }
    }
}

void IMergeTreeDataPart::clearCaches()
{
    if (cleared_data_in_caches.exchange(true) || is_duplicate)
        return;

    /// Remove index and marks from the cache, because otherwise the cache will grow to its maximum size
    /// even if the overall index size is much less.
    removeMarksFromCache(storage.getContext()->getMarkCache().get());
    removeIndexMarksFromCache(storage.getContext()->getIndexMarkCache().get());
    removeIndexFromCache(storage.getPrimaryIndexCache().get());

    /// Remove from other caches of secondary indexes
    removeFromVectorIndexCache(storage.getContext()->getVectorSimilarityIndexCache().get());
}

bool IMergeTreeDataPart::mayStoreDataInCaches() const
{

Review comment (performance):
I wonder why we leaked the internals of mark loader from here
https://github.com/ClickHouse/ClickHouse/blob/696d605f71f224939e1110c5254ae1ac589ac56a/src/Storages/MergeTree/MergeTreeMarksLoader.cpp#L234

But at least this PR does not make it worse; it was copied many times in `MergeTreeDataPart*::removeMarksFromCache` already.

Code after:
{
    if (!index_mark_cache)
        return;

    auto metadata_snapshot = storage.getInMemoryMetadataPtr();
    const auto & secondary_indices = metadata_snapshot->getSecondaryIndices();
    if (secondary_indices.empty())
        return;

    for (const auto & index_description : secondary_indices)
    {
        auto skip_index = MergeTreeIndexFactory::instance().get(index_description);
        auto index_name = skip_index->getFileName();
        auto index_format = skip_index->getDeserializedFormat(checksums, index_name);

        if (!index_format)
            continue;

        for (const auto & substream : index_format.substreams)
        {
            auto stream_name = getStreamNameOrHash(index_name + substream.suffix, substream.extension, checksums);
            if (!stream_name)
                continue;

            auto marks_file = index_granularity_info.getMarksFilePath(*stream_name);
            auto key = MarkCache::hash(getDataPartStorage().getDiskName() + ":" + (fs::path(getRelativePathOfActivePart()) / marks_file).string());
            index_mark_cache->remove(key);
        }
    }
}

void IMergeTreeDataPart::clearCaches()
{
    if (cleared_data_in_caches.exchange(true) || is_duplicate)
        return;

    /// Remove index and marks from the cache, because otherwise the cache will grow to its maximum size
    /// even if the overall index size is much less.
    removeMarksFromCache(storage.getContext()->getMarkCache().get());
    removeIndexMarksFromCache(storage.getContext()->getIndexMarkCache().get());
    removeIndexFromCache(storage.getPrimaryIndexCache().get());

    /// Remove from other caches of secondary indexes
    removeFromVectorIndexCache(storage.getContext()->getVectorSimilarityIndexCache().get());
}

bool IMergeTreeDataPart::mayStoreDataInCaches() const
{
    if (cleared_data_in_caches)
        return false;