Repository: pmd/pmd (Java)

Issue:
[core] CPD: `--skip-duplicate-files` has no effect (7.0.0 regression)
**Affects PMD Version:** 

7.0.0

**Description:**

CPD reports duplication for identical files even though `--skip-duplicate-files` is enabled.
This also happens via Gradle (`CPDConfiguration.setSkipDuplicates(true)`).

**Code Sample demonstrating the issue:**

```java
/*
 * Copyright 2019 Andreas Schmid
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package de.aaschmid.test;

import static org.junit.Assert.assertTrue;

public class Test {

    public testCpd() {
        assertTrue(true);
    }
}
```

**Steps to reproduce:**

```
$ cat a/Test.java
(as above)
$ cat b/Test.java
(as above)
$ diff a/Test.java b/Test.java
(no output => no difference)
$ pmd-bin-7.0.0/bin/pmd cpd --skip-duplicate-files --minimum-tokens=10 a/ b/
Found a 6 line (15 tokens) duplication in the following files: 
Starting at line 20 of /tmp/pmd/a/Test.java
Starting at line 20 of /tmp/pmd/b/Test.java

public class Test {

    public testCpd() {
        assertTrue(true);
    }
}
```

This is not an issue with 6.55.0:

```
$ ./pmd-bin-6.55.0/bin/run.sh cpd --minimum-tokens 10 --skip-duplicate-files --dir a/ --dir b/
Skipping /tmp/pmd/b/Test.java since it appears to be a duplicate file and --skip-duplicate-files is set
```

**Running PMD through:**

CLI, Gradle


Discussion: I can confirm, that this is broken now. The flag is set on `CPDConfiguration` but never used.

In PMD 6, the implementation is here:
https://github.com/pmd/pmd/blob/d4b99bb047c832f4b0b7153dc91b23f6b8d19565/pmd-compat6/src/main/java/net/sourceforge/pmd/cpd/CPD.java#L57-L66

Note, that PMD was actually only comparing the simple file name and the file size, but **not the content** of the files.




Fix (patch):
diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md
index 74833440cb6..1133b736e11 100644
--- a/docs/pages/release_notes.md
+++ b/docs/pages/release_notes.md
@@ -28,6 +28,7 @@ This is a {{ site.pmd.release_type }} release.
 ### 🐛 Fixed Issues
 * core
   * [#494](https://github.com/pmd/pmd/issues/494): \[core] Adopt JApiCmp to enforce control over API changes
+  * [#4942](https://github.com/pmd/pmd/issues/4942): \[core] CPD: `--skip-duplicate-files` has no effect (7.0.0 regression)
 * cli
   * [#4791](https://github.com/pmd/pmd/issues/4791): \[cli] Could not find or load main class
   * [#4913](https://github.com/pmd/pmd/issues/4913): \[cli] cpd-gui closes immediately
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/FileCollectionUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/FileCollectionUtil.java
index 09ac693b500..6fef5fab9f9 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/FileCollectionUtil.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/FileCollectionUtil.java
@@ -6,16 +6,21 @@
 
 import java.io.IOException;
 import java.io.Reader;
+import java.io.UncheckedIOException;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.sql.SQLException;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import net.sourceforge.pmd.AbstractConfiguration;
+import net.sourceforge.pmd.cpd.CPDConfiguration;
 import net.sourceforge.pmd.lang.document.FileCollector;
 import net.sourceforge.pmd.lang.document.FileId;
 import net.sourceforge.pmd.lang.document.InternalApiBridge;
@@ -36,6 +41,35 @@ private FileCollectionUtil() {
 
     }
 
+    public static void collectFiles(CPDConfiguration cpdConfiguration, FileCollector collector) {
+        if (cpdConfiguration.isSkipDuplicates()) {
+            final Set<String> alreadyAddedFileNamesWithSize = new HashSet<>();
+            collector.setFileFilter(fileId -> {
+                Path path = Paths.get(fileId.getAbsolutePath());
+                if (!Files.isRegularFile(path)) {
+                    // file is not a simple file, maybe inside a ZIP archive
+                    // don't filter these out
+                    LOG.debug("Path {} is not a regular file, skipping in fileFilter", path);
+                    return true;
+                }
+
+                try {
+                    String signature = path.getFileName() + "_" + Files.size(path);
+                    if (!alreadyAddedFileNamesWithSize.add(signature)) {
+                        LOG.info("Skipping {} since it appears to be a duplicate file and --skip-duplicate-files is set",
+                                path);
+                        return false;
+                    }
+                } catch (IOException e) {
+                    throw new UncheckedIOException(e);
+                }
+                return true;
+            });
+        }
+
+        collectFiles((AbstractConfiguration) cpdConfiguration, collector);
+    }
+
     public static void collectFiles(AbstractConfiguration configuration, FileCollector collector) {
         if (configuration.getSourceEncoding() != null) {
             collector.setCharset(configuration.getSourceEncoding());
@@ -77,7 +111,7 @@ public static void collectFiles(FileCollector collector, List<Path> filePaths) {
             try {
                 addRoot(collector, rootLocation);
             } catch (IOException e) {
-                collector.getReporter().errorEx("Error collecting " + rootLocation, e);
+                collector.getReporter().errorEx("Error collecting {0}", new Object[]{ rootLocation }, e);
             }
         }
     }
@@ -85,7 +119,7 @@ public static void collectFiles(FileCollector collector, List<Path> filePaths) {
     public static void collectFileList(FileCollector collector, Path fileList) {
         LOG.debug("Reading file list {}.", fileList);
         if (!Files.exists(fileList)) {
-            collector.getReporter().error("No such file {}", fileList);
+            collector.getReporter().error("No such file {0}", fileList);
             return;
         }
 
@@ -93,7 +127,7 @@ public static void collectFileList(FileCollector collector, Path fileList) {
         try {
             filePaths = FileUtil.readFilelistEntries(fileList);
         } catch (IOException e) {
-            collector.getReporter().errorEx("Error reading {}", new Object[] { fileList }, e);
+            collector.getReporter().errorEx("Error reading {0}", new Object[] { fileList }, e);
             return;
         }
         collectFiles(collector, filePaths);
@@ -135,7 +169,7 @@ public static void collectDB(FileCollector collector, URI uri) {
                     String source = IOUtil.readToString(sourceCode);
                     collector.addSourceFile(FileId.fromPathLikeString(falseFilePath), source);
                 } catch (SQLException ex) {
-                    collector.getReporter().warnEx("Cannot get SourceCode for {}  - skipping ...",
+                    collector.getReporter().warnEx("Cannot get SourceCode for {0}  - skipping ...",
                                                    new Object[] { falseFilePath },
                                                    ex);
                 }
@@ -143,7 +177,7 @@ public static void collectDB(FileCollector collector, URI uri) {
         } catch (ClassNotFoundException e) {
             collector.getReporter().errorEx("Cannot get files from DB - probably missing database JDBC driver", e);
         } catch (Exception e) {
-            collector.getReporter().errorEx("Cannot get files from DB - ''{}''", new Object[] { uri }, e);
+            collector.getReporter().errorEx("Cannot get files from DB - ''{0}''", new Object[] { uri }, e);
         }
     }
 }
diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java
index fcbbdf1de15..ccd668156ac 100644
--- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java
+++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java
@@ -29,6 +29,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
+import java.util.function.Predicate;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -60,6 +61,7 @@ public final class FileCollector implements AutoCloseable {
     private final FileId outerFsPath;
     private boolean closed;
     private boolean recursive = true;
+    private Predicate<FileId> fileFilter = file -> true;
 
     // construction
 
@@ -206,6 +208,12 @@ && addFileImpl(TextFile.builderForCharSeq(sourceContents, fileId, version)
 
     private boolean addFileImpl(TextFile textFile) {
         LOG.trace("Adding file {} (lang: {}) ", textFile.getFileId().getAbsolutePath(), textFile.getLanguageVersion().getTerseName());
+
+        if (!fileFilter.test(textFile.getFileId())) {
+            LOG.trace("File was skipped due to fileFilter...");
+            return false;
+        }
+
         if (allFilesToProcess.add(textFile)) {
             return true;
         }
@@ -377,6 +385,17 @@ public void setCharset(Charset charset) {
         this.charset = Objects.requireNonNull(charset);
     }
 
+    /**
+     * Sets an additional filter that is being called before adding the
+     * file to the list.
+     *
+     * @param fileFilter the filter should return {@code true} if the file
+     *                      should be collected and analyzed.
+     * @throws NullPointerException if {@code fileFilter} is {@code null}.
+     */
+    public void setFileFilter(Predicate<FileId> fileFilter) {
+        this.fileFilter = Objects.requireNonNull(fileFilter);
+    }
 
     // filtering