Task: Find duplicate files


C++: #include<iostream>
#include<string>
#include<boost/filesystem.hpp>
#include<boost/format.hpp>
#include<boost/iostreams/device/mapped_file.hpp>
#include<optional>
#include<algorithm>
#include<iterator>
#include<execution>
#include"dependencies/xxhash.hpp" 


template<typename  T, typename V, typename F>
size_t for_each_adjacent_range(T begin, T end, V getvalue, F callback) {
    size_t partitions = 0;
    while (begin != end) {
        auto const& value = getvalue(*begin);
        auto current = begin;
        while (++current != end && getvalue(*current) == value);
        callback(begin, current, value);
        ++partitions;
        begin = current;
    }
    return partitions;
}

namespace bi = boost::iostreams;
namespace fs = boost::filesystem;

struct file_entry {
public:
    explicit file_entry(fs::directory_entry const & entry) 
        : path_{entry.path()}, size_{fs::file_size(entry)}
    {}
    auto size() const { return size_; }
    auto const& path() const { return path_; }
    auto get_hash() {
        if (!hash_)
            hash_ = compute_hash();
        return *hash_;
    }
private:
    xxh::hash64_t compute_hash() {
        bi::mapped_file_source source;
        source.open<fs::wpath>(this->path());
        if (!source.is_open()) {
            std::cerr << "Cannot open " << path() << std::endl;
            throw std::runtime_error("Cannot open file");
        }
        xxh::hash_state64_t hash_stream;
        hash_stream.update(source.data(), size_);
        return hash_stream.digest();
    }
private:
    fs::wpath path_;
    uintmax_t size_;
    std::optional<xxh::hash64_t> hash_;
};

using vector_type = std::vector<file_entry>;
using iterator_type = vector_type::iterator;

auto find_files_in_dir(fs::wpath const& path, vector_type& file_vector, uintmax_t min_size = 1) {
    size_t found = 0, ignored = 0;
    if (!fs::is_directory(path)) {
        std::cerr << path << " is not a directory!" << std::endl;
    }
    else {
        std::cerr << "Searching " << path << std::endl;

        for (auto& e : fs::recursive_directory_iterator(path)) {
            ++found;
            if (fs::is_regular_file(e) && fs::file_size(e) >= min_size)
                file_vector.emplace_back(e);
            else ++ignored;
        }
    }
    return std::make_tuple(found, ignored);
}

int main(int argn, char* argv[])
{
    vector_type files;
    for (auto i = 1; i < argn; ++i) {
        fs::wpath path(argv[i]);
        auto [found, ignored] = find_files_in_dir(path, files);
        std::cerr << boost::format{
            "  %1$6d files found\n"
            "  %2$6d files ignored\n"
            "  %3$6d files added\n" } % found % ignored % (found - ignored) 
            << std::endl;
    }

    std::cerr << "Found " << files.size() << " regular files" << std::endl;
    
    std::sort(std::execution::par_unseq, files.begin(), files.end()
        , [](auto const& a, auto const& b) { return a.size() > b.size(); }
    );
    for_each_adjacent_range(
        std::begin(files)
        , std::end(files)
        , [](vector_type::value_type const& f) { return f.size(); }
        , [](auto start, auto end, auto file_size) {
            
            size_t nr_of_files = std::distance(start, end);
            if (nr_of_files > 1) {
                
                std::sort(start, end, [](auto& a, auto& b) { 
                    auto const& ha = a.get_hash();
                    auto const& hb = b.get_hash();
                    auto const& pa = a.path();
                    auto const& pb = b.path();
                    return std::tie(ha, pa) < std::tie(hb, pb); 
                    });
                for_each_adjacent_range(
                    start
                    , end
                    , [](vector_type::value_type& f) { return f.get_hash(); }
                    , [file_size](auto hstart, auto hend, auto hash) {
                        
                        
                        size_t hnr_of_files = std::distance(hstart, hend);
                        if (hnr_of_files > 1) {
                            std::cout << boost::format{ "%1$3d files with hash %3$016x and size %2$d\n" } 
                                % hnr_of_files % file_size % hash;
                            std::for_each(hstart, hend, [hash, file_size](auto& e) {
                                std::cout << '\t' << e.path() << '\n';
                                }
                            );
                        }
                    }
                );
            }
        }
    );
    
    return 0;
}




Java: import java.io.*;
import java.nio.*;
import java.nio.file.*;
import java.nio.file.attribute.*;
import java.security.*;
import java.util.*;

public class DuplicateFiles {
    public static void main(String[] args) {
        if (args.length != 2) {
            System.err.println("Directory name and minimum file size are required.");
            System.exit(1);
        }
        try {
            findDuplicateFiles(args[0], Long.parseLong(args[1]));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void findDuplicateFiles(String directory, long minimumSize)
        throws IOException, NoSuchAlgorithmException {
        System.out.println("Directory: '" + directory + "', minimum size: " + minimumSize + " bytes.");
        Path path = FileSystems.getDefault().getPath(directory);
        FileVisitor visitor = new FileVisitor(path, minimumSize);
        Files.walkFileTree(path, visitor);
        System.out.println("The following sets of files have the same size and checksum:");
        for (Map.Entry<FileKey, Map<Object, List<String>>> e : visitor.fileMap_.entrySet()) {
            Map<Object, List<String>> map = e.getValue();
            if (!containsDuplicates(map))
                continue;
            List<List<String>> fileSets = new ArrayList<>(map.values());
            for (List<String> files : fileSets)
                Collections.sort(files);
            Collections.sort(fileSets, new StringListComparator());
            FileKey key = e.getKey();
            System.out.println();
            System.out.println("Size: " + key.size_ + " bytes");
            for (List<String> files : fileSets) {
                for (int i = 0, n = files.size(); i < n; ++i) {
                    if (i > 0)
                        System.out.print(" = ");
                    System.out.print(files.get(i));
                }
                System.out.println();
            }
        }
    }

    private static class StringListComparator implements Comparator<List<String>> {
        public int compare(List<String> a, List<String> b) {
            int len1 = a.size(), len2 = b.size();
            for (int i = 0; i < len1 && i < len2; ++i) {
                int c = a.get(i).compareTo(b.get(i));
                if (c != 0)
                    return c;
            }
            return Integer.compare(len1, len2);
        }
    }

    private static boolean containsDuplicates(Map<Object, List<String>> map) {
        if (map.size() > 1)
            return true;
        for (List<String> files : map.values()) {
            if (files.size() > 1)
                return true;
        }
        return false;
    }

    private static class FileVisitor extends SimpleFileVisitor<Path> {
        private MessageDigest digest_;
        private Path directory_;
        private long minimumSize_;
        private Map<FileKey, Map<Object, List<String>>> fileMap_ = new TreeMap<>();

        private FileVisitor(Path directory, long minimumSize) throws NoSuchAlgorithmException {
            directory_ = directory;
            minimumSize_ = minimumSize;
            digest_ = MessageDigest.getInstance("MD5");
        }

        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
            if (attrs.size() >= minimumSize_) {
                FileKey key = new FileKey(file, attrs, getMD5Sum(file));
                Map<Object, List<String>> map = fileMap_.get(key);
                if (map == null)
                    fileMap_.put(key, map = new HashMap<>());
                List<String> files = map.get(attrs.fileKey());
                if (files == null)
                    map.put(attrs.fileKey(), files = new ArrayList<>());
                Path relative = directory_.relativize(file);
                files.add(relative.toString());
            }
            return FileVisitResult.CONTINUE;
        }

        private byte[] getMD5Sum(Path file) throws IOException {
            digest_.reset();
            try (InputStream in = new FileInputStream(file.toString())) {
                byte[] buffer = new byte[8192];
                int bytes;
                while ((bytes = in.read(buffer)) != -1) {
                    digest_.update(buffer, 0, bytes);
                }
            }
            return digest_.digest();
        }
    }

    private static class FileKey implements Comparable<FileKey> {
        private byte[] hash_;
        private long size_;

        private FileKey(Path file, BasicFileAttributes attrs, byte[] hash) throws IOException {
            size_ = attrs.size();
            hash_ = hash;
        }

        public int compareTo(FileKey other) {
            int c = Long.compare(other.size_, size_);
            if (c == 0)
                c = hashCompare(hash_, other.hash_);
            return c;
        }
    }

    private static int hashCompare(byte[] a, byte[] b) {
        int len1 = a.length, len2 = b.length;
        for (int i = 0; i < len1 && i < len2; ++i) {
            int c = Byte.compare(a[i], b[i]);
            if (c != 0)
                return c;
        }
        return Integer.compare(len1, len2);
    }
}