Task: Find duplicate files




Go: package main

import (
    "fmt"
    "crypto/md5"
    "io/ioutil"
    "log"
    "os"
    "path/filepath"
    "sort"
    "time"
)

type fileData struct {
    filePath string
    info     os.FileInfo
}

type hash [16]byte

func check(err error) {
    if err != nil {
        log.Fatal(err)
    }
}

func checksum(filePath string) hash {
    bytes, err := ioutil.ReadFile(filePath)
    check(err)
    return hash(md5.Sum(bytes))
}

func findDuplicates(dirPath string, minSize int64) [][2]fileData {
    var dups [][2]fileData
    m := make(map[hash]fileData)
    werr := filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
        if err != nil {
            return err
        }
        if !info.IsDir() && info.Size() >= minSize {
            h := checksum(path)
            fd, ok := m[h]
            fd2 := fileData{path, info}
            if !ok {
                m[h] = fd2
            } else {
                dups = append(dups, [2]fileData{fd, fd2})
            }
        }
        return nil
    })
    check(werr)
    return dups
}

func main() {
    dups := findDuplicates(".", 1)
    fmt.Println("The following pairs of files have the same size and the same hash:\n")
    fmt.Println("File name                 Size      Date last modified")
    fmt.Println("==========================================================")
    sort.Slice(dups, func(i, j int) bool {
        return dups[i][0].info.Size() > dups[j][0].info.Size() 
    })
    for _, dup := range dups {
        for i := 0; i < 2; i++ {
            d := dup[i]
            fmt.Printf("%-20s  %8d    %v\n", d.filePath, d.info.Size(), d.info.ModTime().Format(time.ANSIC))
        }
        fmt.Println()
    }
}




Python: from __future__ import print_function
import os
import hashlib
import datetime

def FindDuplicateFiles(pth, minSize = 0, hashName = "md5"):
    knownFiles = {}

    
    for root, dirs, files in os.walk(pth):
        for fina in files:
            fullFina = os.path.join(root, fina)
            isSymLink = os.path.islink(fullFina)
            if isSymLink:
                continue 
            si = os.path.getsize(fullFina)
            if si < minSize:
                continue
            if si not in knownFiles:
                knownFiles[si] = {}
            h = hashlib.new(hashName)
            h.update(open(fullFina, "rb").read())
            hashed = h.digest()
            if hashed in knownFiles[si]:
                fileRec = knownFiles[si][hashed]
                fileRec.append(fullFina)
            else:
                knownFiles[si][hashed] = [fullFina]

    
    sizeList = list(knownFiles.keys())
    sizeList.sort(reverse=True)
    for si in sizeList:
        filesAtThisSize = knownFiles[si]
        for hashVal in filesAtThisSize:
            if len(filesAtThisSize[hashVal]) < 2:
                continue
            fullFinaLi = filesAtThisSize[hashVal]
            print ("=======Duplicate=======")
            for fullFina in fullFinaLi:
                st = os.stat(fullFina)
                isHardLink = st.st_nlink > 1 
                infoStr = []
                if isHardLink:
                    infoStr.append("(Hard linked)")
                fmtModTime = datetime.datetime.utcfromtimestamp(st.st_mtime).strftime('%Y-%m-%dT%H:%M:%SZ')
                print (fmtModTime, si, os.path.relpath(fullFina, pth), " ".join(infoStr))

if __name__=="__main__":

    FindDuplicateFiles('/home/tim/Dropbox', 1024*1024)