Repository: hashgraph/hedera-mirror-node
Issue: Record file hash mismatch with previous

**Detailed Description**
`File Hash Mismatch with previous` error still occurs even after the fix for #157.

**Actual Behavior**
1. Run record downloader
2. Run record parser
Wait until hash mismatch occurs.

**Expected Behavior**
Records download, verified and moved to valid with no errors.

**Environment:**
- Environment: staging
- Java: OpenJDK 11.0.4
- Version: 0.1.0

**Additional Context**
Logs when it happened (reverse order):
```console
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,896 WARN  [main ] c.h.d.RecordFileDownloader File Hash Mismatch with previous : /var/lib/mirror-node/recordstreams/valid/2019-09-03T20_43_40.116069Z.rcd
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,896 TRACE [main ] c.h.d.ApplicationStatus Returning application status for : RECORD_HASH_MISMATCH_BYPASS_UNTIL_AFTER,
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,895 TRACE [main ] c.h.d.ApplicationStatus Getting application status for : RECORD_HASH_MISMATCH_BYPASS_UNTIL_AFTER
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,894 TRACE [main ] c.h.p.RecordFileParser Read previous file hash a092fd020847f6b2206b29e7d50283c0f820e4244691eed75e4b19c301d7c97c0a7f5d1f0a24c227c84e6ee67b06b033 for file /var/lib/mirror-node/recordstreams/valid/2019-09-03T20_43_40.116069Z.rcd
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,893 TRACE [main ] c.h.d.ApplicationStatus Returning application status for : LAST_VALID_DOWNLOADED_RECORD_FILE_HASH, d8b70aabf07659c3fd05492f3c27b50539e6535a51e6c3e3f747ae3a26a388ddc4eb47e2fffcbc7ff33fcbe436bc1573
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,893 TRACE [main ] c.h.d.ApplicationStatus Returning application status for : LAST_VALID_DOWNLOADED_RECORD_FILE, 2019-09-03T20_42_00.092453Z.rcd
Sep 03 22:00:36 psql-test-02 java[8100]: 2019-09-03 22:00:36,893 DEBUG [main ] c.h.d.RecordFileDownloader Verified signature file matches at least 2/3 of nodes: 2019-09-03T20_43_40.116069Z.rcd_sig
```



Changed files:
src/test/java/com/hedera/downloader/RecordFileDownloaderTestV1.java, src/main/java/com/hedera/downloader/RecordFileDownloader.java, src/test/java/com/hedera/downloader/RecordFileDownloaderTestV2.java

diff --git a/src/main/java/com/hedera/downloader/RecordFileDownloader.java b/src/main/java/com/hedera/downloader/RecordFileDownloader.java
index 0470d34b2..5c7e570e6 100644
--- a/src/main/java/com/hedera/downloader/RecordFileDownloader.java
+++ b/src/main/java/com/hedera/downloader/RecordFileDownloader.java
@@ -28,6 +28,7 @@ import com.hedera.utilities.Utility;
 
 import lombok.extern.log4j.Log4j2;
 import org.apache.commons.codec.binary.Hex;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.tuple.Pair;
 
 import java.io.File;
@@ -39,6 +40,7 @@ import java.util.stream.Stream;
 @Log4j2
 public class RecordFileDownloader extends Downloader {
 
+	private static final String EMPTY_HASH = Hex.encodeHexString(new byte[48]);
 	private final String validDir = ConfigLoader.getDefaultParseDir(OPERATION_TYPE.RECORDS);
 	private final String tmpDir = ConfigLoader.getDefaultTmpDir(OPERATION_TYPE.RECORDS);
 
@@ -65,7 +67,6 @@ public class RecordFileDownloader extends Downloader {
 
 			// Verify signature files and download .rcd files of valid signature files
 			verifySigsAndDownloadRecordFiles(sigFilesMap);
-			verifyValidRecordFiles(validDir);
 		} catch (Exception e) {
 			log.error("Error downloading and verifying new record files", e);
 		}
@@ -73,56 +74,28 @@ public class RecordFileDownloader extends Downloader {
 
 	/**
 	 * Verify the .rcd files to see if the file Hash matches prevFileHash
-	 * @param fileToCheck
 	 * @throws Exception 
 	 */
-	private void verifyValidRecordFiles(String validDir) throws Exception {
-		String lastValidRcdFileName =  applicationStatus.getLastValidDownloadedRecordFileName();
-		String lastValidRcdFileHash = applicationStatus.getLastValidDownloadedRecordFileHash();
-
-		try (Stream<Path> pathStream = Files.walk(Paths.get(validDir))) {
-			List<String> fileNames = pathStream.filter(p -> Utility.isRecordFile(p.toString()))
-					.filter(p -> lastValidRcdFileName.isEmpty() ||
-							fileNameComparator.compare(p.toFile().getName(), lastValidRcdFileName) > 0)
-					.sorted(pathComparator)
-					.map(p -> p.toString()).collect(Collectors.toList());
-
-			String newLastValidRcdFileName = lastValidRcdFileName;
-			String newLastValidRcdFileHash = lastValidRcdFileHash;
-
-			for (String rcdName : fileNames) {
-				if (Utility.checkStopFile()) {
-					log.info("Stop file found, stopping");
-					break;
-				}
-				String prevFileHash = RecordFileParser.readPrevFileHash(rcdName);
-				if (prevFileHash == null) {
-					log.warn("Doesn't contain valid prevFileHash: {}", rcdName);
-					break;
-				}
-				if (newLastValidRcdFileHash.isEmpty() ||
-						newLastValidRcdFileHash.equals(prevFileHash) ||
-						prevFileHash.equals(Hex.encodeHexString(new byte[48]))) {
-					newLastValidRcdFileHash = Utility.bytesToHex(Utility.getFileHash(rcdName));
-					newLastValidRcdFileName = new File(rcdName).getName();
-				} else if (applicationStatus.getBypassRecordHashMismatchUntilAfter().compareTo(new File(rcdName).getName()) > 0) {
-					newLastValidRcdFileName = new File(rcdName).getName();
-					newLastValidRcdFileHash = Utility.bytesToHex(Utility.getFileHash(rcdName));
-				} else {
-					log.warn("File Hash Mismatch with previous : {}", rcdName);
-					break;
-				}
-			}
-
-			if (!newLastValidRcdFileName.equals(lastValidRcdFileName)) {
-				applicationStatus.updateLastValidDownloadedRecordFileHash(newLastValidRcdFileHash);
-				applicationStatus.updateLastValidDownloadedRecordFileName(newLastValidRcdFileName);
-			}
+	private boolean verifyHashChain(File recordFile) throws Exception {
+		String recordPath = recordFile.getAbsolutePath();
+		String lastValidRecordFileHash = applicationStatus.getLastValidDownloadedRecordFileHash();
+		String bypassMismatch = StringUtils.defaultIfBlank(applicationStatus.getBypassRecordHashMismatchUntilAfter(), "");
+		String prevFileHash = RecordFileParser.readPrevFileHash(recordPath);
+
+		if (prevFileHash == null) {
+			log.warn("Doesn't contain valid previous file hash: {}", recordPath);
+			return false;
+		}
 
-		} catch (Exception ex) {
-			log.error("Failed to verify record files in {}", validDir, ex);
+		if (StringUtils.isBlank(lastValidRecordFileHash) || lastValidRecordFileHash.equals(prevFileHash) ||
+				EMPTY_HASH.equals(prevFileHash) || bypassMismatch.compareTo(recordFile.getName()) > 0) {
+			return true;
 		}
+
+		log.warn("File Hash Mismatch with previous: {}, expected {}, got {}", recordFile.getName(), lastValidRecordFileHash, prevFileHash);
+		return false;
 	}
+
 	/**
 	 *  For each group of signature Files with the same file name:
 	 *  (1) verify that the signature files are signed by corresponding node's PublicKey;
@@ -147,7 +120,7 @@ public class RecordFileDownloader extends Downloader {
 			}
 			boolean valid = false;
 			List<File> sigFiles = sigFilesMap.get(fileName);
-			
+
 			// If the number of sigFiles is not greater than 2/3 of number of nodes, we don't need to verify them
 			if (sigFiles == null || !Utility.greaterThanSuperMajorityNum(sigFiles.size(), nodeAccountIds.size())) {
 				log.warn("Signature file count for {} does not exceed 2/3 of nodes", fileName);
@@ -162,18 +135,29 @@ public class RecordFileDownloader extends Downloader {
 					break;
 				}
 
-				Pair<Boolean, File> rcdFileResult = downloadFile(DownloadType.RCD, validSigFile, tmpDir);
-				File rcdFile = rcdFileResult.getRight();
-				if (rcdFile != null && Utility.hashMatch(validSigFile, rcdFile)) {
-					// move the file to the valid directory
-					File fTo = new File(validDir + "/" + rcdFile.getName());					
-					if (moveFile(rcdFile, fTo)) {
-						log.debug("Verified signature file matches at least 2/3 of nodes: {}", fileName);
-						valid = true;
-						break;
+				try {
+					Pair<Boolean, File> rcdFileResult = downloadFile(DownloadType.RCD, validSigFile, tmpDir);
+					File rcdFile = rcdFileResult.getRight();
+					if (rcdFile != null && Utility.hashMatch(validSigFile, rcdFile)) {
+						if (verifyHashChain(rcdFile)) {
+							// move the file to the valid directory
+							String name = rcdFile.getName();
+							String hash = Utility.bytesToHex(Utility.getFileHash(rcdFile.getAbsolutePath()));
+							File validFile = Paths.get(validDir, name).toFile();
+
+							if (moveFile(rcdFile, validFile)) {
+								log.debug("Verified signature file matches at least 2/3 of nodes: {}", fileName);
+								applicationStatus.updateLastValidDownloadedRecordFileHash(hash);
+								applicationStatus.updateLastValidDownloadedRecordFileName(name);
+								valid = true;
+								break;
+							}
+						}
+					} else if (rcdFile != null) {
+						log.warn("Hash of {} doesn't match the hash contained in the signature file. Will try to download a record file with same timestamp from other nodes", validSigFile);
 					}
-				} else if (rcdFile != null) {
-					log.warn("Hash of {} doesn't match the hash contained in the signature file. Will try to download a record file with same timestamp from other nodes", rcdFile);
+				} catch (Exception e) {
+					log.error("Unable to verify signature {}", validSigFile, e);
 				}
 			}
 
diff --git a/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV1.java b/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV1.java
index 263d61bb4..781d1fe70 100644
--- a/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV1.java
+++ b/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV1.java
@@ -72,9 +72,6 @@ public class RecordFileDownloaderTestV1 {
 
         s3 = S3Mock.create(8001, s3Path.toString());
         s3.start();
-
-        when(applicationStatus.getLastValidDownloadedRecordFileName()).thenReturn("");
-        when(applicationStatus.getLastValidDownloadedRecordFileHash()).thenReturn("");
     }
 
     @AfterEach
@@ -163,13 +160,11 @@ public class RecordFileDownloaderTestV1 {
     void hashMismatchWithPrevious() throws Exception {
         when(applicationStatus.getLastValidDownloadedRecordFileName()).thenReturn("2019-07-01T14:12:00.000000Z.rcd");
         when(applicationStatus.getLastValidDownloadedRecordFileHash()).thenReturn("123");
-        when(applicationStatus.getBypassRecordHashMismatchUntilAfter()).thenReturn("");
         fileCopier.copy();
         downloader.download();
         assertThat(Files.walk(validPath))
                 .filteredOn(p -> !p.toFile().isDirectory())
-                .hasSize(2)
-                .allMatch(p -> Utility.isRecordFile(p.toString()));
+                .hasSize(0);
     }
 
     @Test
diff --git a/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV2.java b/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV2.java
index 39bbf0974..5576a3811 100644
--- a/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV2.java
+++ b/src/test/java/com/hedera/downloader/RecordFileDownloaderTestV2.java
@@ -72,9 +72,6 @@ public class RecordFileDownloaderTestV2 {
 
         s3 = S3Mock.create(8001, s3Path.toString());
         s3.start();
-
-        when(applicationStatus.getLastValidDownloadedRecordFileName()).thenReturn("");
-        when(applicationStatus.getLastValidDownloadedRecordFileHash()).thenReturn("");
     }
 
     @AfterEach