Repository: hashgraph/hedera-mirror-node
Issue: Don't process partial files

Due to the nature of downloading to files and processing the files in separate processes, it's possible that processing could occur on a partially downloaded file. Whether it's polling for new files manually or using java's WatcherService, it's a possibility that the S3 downloader created the file, wrote some bytes but has not finished writing all bytes by the time the file is picked up by the parsers (especially for larger files).

Possible solutions:
1. Download files to a different directory and move them once complete. Move is an atomic operation.
2. Combine downloader and parser into a single process and use S3's GetObject to download to a ByteArrayOutputStream instead of a file.

Pros to 1 is its quickest.

Pros to 2 is it's a simpler architecture and avoids the penalty of writing files and coordinating processes. Due to the serial nature of processing these file streams, the only benefit of having separate processes is for separation of failures. But parser would be effectively down anyway if there's no new files being downloaded. Con is it's more work and not sure how large files get to be loaded into memory.

Changed files:
src/main/java/com/hedera/configLoader/ConfigLoader.java, src/main/java/com/hedera/downloader/AccountBalancesDownloader.java, src/main/java/com/hedera/downloader/Downloader.java, src/main/java/com/hedera/downloader/EventStreamFileDownloader.java, src/main/java/com/hedera/downloader/RecordFileDownloader.java, src/main/java/com/hedera/utilities/Utility.java

diff --git a/src/main/java/com/hedera/configLoader/ConfigLoader.java b/src/main/java/com/hedera/configLoader/ConfigLoader.java
index 28f64a7ea..492a8d046 100644
--- a/src/main/java/com/hedera/configLoader/ConfigLoader.java
+++ b/src/main/java/com/hedera/configLoader/ConfigLoader.java
@@ -355,6 +355,10 @@ public class ConfigLoader {
 		return "";
 	}
 
+	public static String getDefaultTmpDir(OPERATION_TYPE operation) {
+		return getDefaultParseDir(operation).replace("/valid", "/tmp");
+	}
+
 	public static int getProxyPort() {
 		return proxyPort;
 	}
diff --git a/src/main/java/com/hedera/downloader/AccountBalancesDownloader.java b/src/main/java/com/hedera/downloader/AccountBalancesDownloader.java
index 96dfb9f75..b17abaa6a 100644
--- a/src/main/java/com/hedera/downloader/AccountBalancesDownloader.java
+++ b/src/main/java/com/hedera/downloader/AccountBalancesDownloader.java
@@ -12,13 +12,18 @@ import org.apache.commons.lang3.tuple.Pair;
 import com.google.gson.JsonIOException;
 import com.google.gson.JsonSyntaxException;
 import com.hedera.configLoader.ConfigLoader;
+import com.hedera.configLoader.ConfigLoader.OPERATION_TYPE;
 import com.hedera.signatureVerifier.NodeSignatureVerifier;
 import com.hedera.utilities.Utility;
 
 public class AccountBalancesDownloader extends Downloader {
 
+	private static String validDir = ConfigLoader.getDefaultParseDir(OPERATION_TYPE.BALANCE);
+	private static String tmpDir = ConfigLoader.getDefaultTmpDir(OPERATION_TYPE.BALANCE);
 
 	public AccountBalancesDownloader() {
+		Utility.createDirIfNotExists(validDir);
+		Utility.createDirIfNotExists(tmpDir);
 	}
 
 	public static void main(String[] args) {
@@ -67,8 +72,7 @@ public class AccountBalancesDownloader extends Downloader {
 	 *  return the name of directory which contains valid _Balances.csv files
 	 * @param sigFilesMap
 	 */
-	String verifySigsAndDownloadBalanceFiles(Map<String, List<File>> sigFilesMap) {
-		String validDir = null;
+	private void verifySigsAndDownloadBalanceFiles(Map<String, List<File>> sigFilesMap) {
 		String lastValidBalanceFileName = "";
 		try {
 			lastValidBalanceFileName = ConfigLoader.getLastValidBalanceFileName();
@@ -99,18 +103,21 @@ public class AccountBalancesDownloader extends Downloader {
 							log.info(MARKER, "Stop file found, stopping.");
 							break;
 						}
-						if (validDir == null) {
-							validDir = validSigFile.getParentFile().getParent() + "/valid/";
-						}
-						Pair<Boolean, File> fileResult = downloadBalanceFile(validSigFile, validDir);
+
+						Pair<Boolean, File> fileResult = downloadFile(DownloadType.BALANCE, validSigFile, tmpDir);
 						File file = fileResult.getRight();
-						if (file != null &&
-								Utility.hashMatch(validSigFile, file)) {
-							if (newLastValidBalanceFileName.isEmpty() ||
-									fileNameComparator.compare(newLastValidBalanceFileName, file.getName()) < 0) {
-								newLastValidBalanceFileName = file.getName();
-							}
-							break;
+						if (file != null && Utility.hashMatch(validSigFile, file)) {
+
+							// move the file to the valid directory
+					        File fTo = new File(validDir + file.getName());
+
+					        if (moveFile(file, fTo)) {
+								if (newLastValidBalanceFileName.isEmpty() ||
+										fileNameComparator.compare(newLastValidBalanceFileName, file.getName()) < 0) {
+									newLastValidBalanceFileName = file.getName();
+								}
+								break;
+					        } 
 						} else if (file != null) {
 							log.warn(MARKER, "{}'s Hash doesn't match the Hash contained in valid signature file. Will try to download a balance file with same timestamp from other nodes and check the Hash.", file.getPath());
 						}
@@ -122,17 +129,5 @@ public class AccountBalancesDownloader extends Downloader {
 			ConfigLoader.setLastValidBalanceFileName(newLastValidBalanceFileName);
 			ConfigLoader.saveToFile();
 		}
-		return validDir;
 	}
-
-	Pair<Boolean, File> downloadBalanceFile(File sigFile, String validDir) {
-		String nodeAccountId = Utility.getAccountIDStringFromFilePath(sigFile.getPath());
-		String sigFileName = sigFile.getName();
-		String balanceFileName = sigFileName.replace("_Balances.csv_sig", "_Balances.csv");
-		String s3ObjectKey = "accountBalances/balance" + nodeAccountId + "/" + balanceFileName;
-		String localFileName = validDir + balanceFileName;
-		return saveToLocal(bucketName, s3ObjectKey, localFileName);
-	}
-
-
 }
diff --git a/src/main/java/com/hedera/downloader/Downloader.java b/src/main/java/com/hedera/downloader/Downloader.java
index 733cbf466..411b1c38f 100644
--- a/src/main/java/com/hedera/downloader/Downloader.java
+++ b/src/main/java/com/hedera/downloader/Downloader.java
@@ -1,6 +1,5 @@
 package com.hedera.downloader;
 
-
 import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.SdkClientException;
@@ -16,7 +15,6 @@ import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.transfer.Download;
 import com.amazonaws.services.s3.transfer.TransferManager;
 import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
-import com.google.gson.JsonObject;
 import com.hedera.configLoader.ConfigLoader;
 import com.hedera.configLoader.ConfigLoader.CLOUD_PROVIDER;
 import com.hedera.configLoader.ConfigLoader.OPERATION_TYPE;
@@ -30,11 +28,14 @@ import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.Marker;
 import org.apache.logging.log4j.MarkerManager;
 
+import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Instant;
 import java.util.ArrayList;
@@ -42,7 +43,6 @@ import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
-import java.util.stream.Collectors;
 
 public abstract class Downloader {
 	protected static final Logger log = LogManager.getLogger("downloader");
@@ -149,6 +149,7 @@ public abstract class Downloader {
 	 *  Download all balance .csv files with timestamp later than lastValidBalanceFileName
 	 */
 
+	@Deprecated
 	protected void downloadBalanceFiles() throws IOException {
 		String s3Prefix = null;
 		String fileType = null;
@@ -157,8 +158,7 @@ public abstract class Downloader {
 		s3Prefix = ConfigLoader.getAccountBalanceS3Location();
 		fileType = ".csv";
 		lastValidFileName = ConfigLoader.getLastValidBalanceFileName();
-		saveFilePath = ConfigLoader.getDefaultParseDir(OPERATION_TYPE.BALANCE);
-
+		saveFilePath = ConfigLoader.getDefaultTmpDir(OPERATION_TYPE.BALANCE);
 		
 		// refresh node account ids
 		nodeAccountIds = loadNodeAccountIDs();
@@ -215,8 +215,11 @@ public abstract class Downloader {
 									count++;
 									File file = result.getRight();
 									if (file != null) {
-										String fileName = file.getName();
-										files.add(fileName);
+										// move the file to the valid directory
+								        File fTo = new File(file.getAbsolutePath().replace("/tmp/", "/valid/") + file.getName());
+								        if (moveFile(file, fTo)) {
+											files.add(file.getName());
+										}
 									}
 								} else if (result.getRight() == null) {
 									log.error(MARKER, "File {} failed to download from cloud", s3ObjectKey);
@@ -396,27 +399,6 @@ public abstract class Downloader {
 		return sigFilesMap;
 	}
 
-//	/**
-//	 * return a pair of download result:
-//	 * boolean: download it or not.
-//	 * True means we download it successfully; False means it already exists or we fail to download it;
-//	 * File is the local file
-//	 * @param bucket_name
-//	 * @param s3ObjectKey
-//	 * @return
-//	 */
-//	protected static Pair<Boolean, File> saveToLocal(String bucket_name,
-//			String s3ObjectKey, String filePath) throws IOException {
-//
-//		if (!Utility.createDirIfNotExists(filePath)) {
-//			log.error(MARKER, "{} doesn't exist and we fail to create this directory", filePath);
-//			return null;
-//		}
-//
-//		filePath += s3ObjectKey;
-//		return saveToLocal(bucket_name, s3ObjectKey, filePath);
-//	}
-
 	/**
 	 * return a pair of download result:
 	 * boolean: download it or not.
@@ -429,6 +411,7 @@ public abstract class Downloader {
 	 */
 	protected static Pair<Boolean, File> saveToLocal(String bucket_name,
 		String s3ObjectKey, String localFilepath)  {
+		
 		// ensure filePaths have OS specific separator
 		localFilepath = localFilepath.replace("/", "~");
 		localFilepath = localFilepath.replace("\\\\", "~");
@@ -436,15 +419,7 @@ public abstract class Downloader {
 
         File f = new File(localFilepath).getAbsoluteFile();
 
-		if (f.exists()) {
-			log.info(MARKER, "File exists: " + localFilepath);
-			return Pair.of(false, f);
-		}
 		try {
-            if( ! f.getParentFile().exists() ) {
-                f.getParentFile().mkdirs();
-            }
-			//f.createNewFile();
 			Download download = xfer_mgr.download(bucket_name, s3ObjectKey, f);
 			download.waitForCompletion();
 			if (download.isDone()) {
@@ -455,10 +430,10 @@ public abstract class Downloader {
 				return Pair.of(false, null);
 			}
 		} catch (AmazonServiceException ex) {
-			log.error(MARKER, "Download Fails: {}, Exception: {}", s3ObjectKey, ex);
+			log.error(MARKER, "Download Failed: {}, Exception: {}", s3ObjectKey, ex);
 		} catch (InterruptedException ex) {
-			log.error(MARKER, "Download Fails: {}, Exception: {}", s3ObjectKey, ex);
-        }
+			log.error(MARKER, "Download Failed: {}, Exception: {}", s3ObjectKey, ex);
+		}
 		return Pair.of(false, null);
 	}
 
@@ -517,4 +492,52 @@ public abstract class Downloader {
 		xfer_mgr = TransferManagerBuilder.standard()
 				.withS3Client(s3Client).build();
 	}
+
+	/**
+	 * Moves a file from one location to another
+	 * boolean: true if file moved successfully
+	 * Note: The method doesn't check if source file or destination directory exist to avoid 
+	 * repeated checks that could hurt performance 
+	 * @param sourceFile
+	 * @param destinationFile
+	 * @return boolean
+	 */
+	protected boolean moveFile(File sourceFile, File destinationFile) {
+        try {
+        	// not checking if file exists to help with performance
+        	// assumption is caller has created the destination file folder
+        	Files.move(sourceFile.toPath(), destinationFile.toPath(), REPLACE_EXISTING);
+        	return true;
+        } catch (IOException e) {
+			log.error(MARKER, "File Move from {} to {} Failed: {}, Exception: {}", sourceFile.getAbsolutePath(), destinationFile.getAbsolutePath(), e);
+			return false;
+       }
+    }
+
+	protected Pair<Boolean, File> downloadFile(DownloadType downloadType, File sigFile, String targetDir) {
+		String fileName = "";
+		String s3Prefix = "";
+		
+		String nodeAccountId = Utility.getAccountIDStringFromFilePath(sigFile.getPath());
+		String sigFileName = sigFile.getName();
+		
+		switch (downloadType) {
+			case BALANCE:
+				fileName = sigFileName.replace("_Balances.csv_sig", "_Balances.csv");
+				s3Prefix = ConfigLoader.getAccountBalanceS3Location();
+				break;
+			case EVENT:
+				fileName = sigFileName.replace(".evts_sig", ".evts");
+				s3Prefix = ConfigLoader.getEventFilesS3Location();
+				break;
+			case RCD:
+				fileName = sigFileName.replace(".rcd_sig", ".rcd");
+				s3Prefix =  ConfigLoader.getRecordFilesS3Location();
+				break;
+		}
+		String s3ObjectKey = s3Prefix + nodeAccountId + "/" + fileName;
+		
+		String localFileName = targetDir + "/" + fileName;
+		return saveToLocal(bucketName, s3ObjectKey, localFileName);
+	}
 }
diff --git a/src/main/java/com/hedera/downloader/EventStreamFileDownloader.java b/src/main/java/com/hedera/downloader/EventStreamFileDownloader.java
index f3a91a852..4db1f5172 100644
--- a/src/main/java/com/hedera/downloader/EventStreamFileDownloader.java
+++ b/src/main/java/com/hedera/downloader/EventStreamFileDownloader.java
@@ -3,6 +3,7 @@ package com.hedera.downloader;
 import com.google.gson.JsonIOException;
 import com.google.gson.JsonSyntaxException;
 import com.hedera.configLoader.ConfigLoader;
+import com.hedera.configLoader.ConfigLoader.OPERATION_TYPE;
 import com.hedera.parser.EventStreamFileParser;
 import com.hedera.signatureVerifier.NodeSignatureVerifier;
 import com.hedera.utilities.Utility;
@@ -23,9 +24,13 @@ import java.util.stream.Stream;
 
 public class EventStreamFileDownloader extends Downloader {
 
-	private static String validDir = null;
+	private static String validDir = ConfigLoader.getDefaultParseDir(OPERATION_TYPE.EVENTS);
+	private static String tmpDir = ConfigLoader.getDefaultTmpDir(OPERATION_TYPE.EVENTS);
+	private static File fileValidDir = new File(validDir);
 
 	public EventStreamFileDownloader() {
+		Utility.createDirIfNotExists(validDir);
+		Utility.createDirIfNotExists(tmpDir);
 	}
 	
 	public static void downloadNewEventfiles(EventStreamFileDownloader downloader) {
@@ -50,7 +55,7 @@ public class EventStreamFileDownloader extends Downloader {
 
 			if (validDir != null) {
 //				new Thread(() -> {
-					verifyValidFiles(validDir);
+					verifyValidFiles();
 //				}).start();
 			}
 
@@ -86,7 +91,7 @@ public class EventStreamFileDownloader extends Downloader {
 	 *
 	 * @param validDir
 	 */
-	public static void verifyValidFiles(String validDir) {
+	public static void verifyValidFiles() {
 		String lastValidEventFileName = "";
 		try {
 			lastValidEventFileName = ConfigLoader.getLastValidEventFileName();
@@ -104,11 +109,7 @@ public class EventStreamFileDownloader extends Downloader {
 		}
 
 		String lastValidEventFileName2 = lastValidEventFileName;
-		File validDirFile = new File(validDir);
-		if (!validDirFile.exists()) {
-			return;
-		}
-		try (Stream<Path> pathStream = Files.walk(validDirFile.toPath())) {
+		try (Stream<Path> pathStream = Files.walk(fileValidDir.toPath())) {
 			List<String> fileNames = pathStream.filter(p -> Utility.isEventStreamFile(p.toString()))
 					.filter(p -> lastValidEventFileName2.isEmpty() ||
 							fileNameComparator.compare(p.toFile().getName(), lastValidEventFileName2) > 0)
@@ -159,7 +160,7 @@ public class EventStreamFileDownloader extends Downloader {
 	 *
 	 * @param sigFilesMap
 	 */
-	String verifySigsAndDownloadEventStreamFiles(Map<String, List<File>> sigFilesMap) {
+	private void verifySigsAndDownloadEventStreamFiles(Map<String, List<File>> sigFilesMap) {
 
 		NodeSignatureVerifier verifier = new NodeSignatureVerifier();
 		for (String fileName : sigFilesMap.keySet()) {
@@ -173,14 +174,16 @@ public class EventStreamFileDownloader extends Downloader {
 				List<File> validSigFiles = verifier.verifySignatureFiles(sigFiles);
 				if (validSigFiles != null) {
 					for (File validSigFile : validSigFiles) {
-						if (validDir == null) {
-							validDir = validSigFile.getParentFile().getParent() + "/valid/";
-						}
-						Pair<Boolean, File> fileResult = downloadFile(validSigFile, validDir);
+						Pair<Boolean, File> fileResult = downloadFile(DownloadType.EVENT, validSigFile, tmpDir);
 						File file = fileResult.getRight();
-						if (file != null &&
-								Utility.hashMatch(validSigFile, file)) {
-							break;
+						if (file != null &&	Utility.hashMatch(validSigFile, file)) {
+							// move the file to the valid directory
+					        File fTo = new File(validDir + file.getName());
+
+							if (moveFile(file, fTo)) {
+								break;
+							}
+					        
 						} else if (file != null) {
 							log.warn(MARKER,
 									"{}'s Hash doesn't match the Hash contained in valid signature file. Will try to " +
@@ -195,22 +198,6 @@ public class EventStreamFileDownloader extends Downloader {
 				}
 			}
 		}
-		return validDir;
 	}
 
-	/**
-	 * Download .evts file
-	 *
-	 * @param sigFile
-	 * @param validDir
-	 * @return
-	 */
-	Pair<Boolean, File> downloadFile(File sigFile, String validDir) {
-		String nodeAccountId = Utility.getAccountIDStringFromFilePath(sigFile.getPath());
-		String sigFileName = sigFile.getName();
-		String fileName = sigFileName.replace(".evts_sig", ".evts");
-		String s3ObjectKey = ConfigLoader.getEventFilesS3Location() + nodeAccountId + "/" + fileName;
-		String localFileName = validDir + fileName;
-		return saveToLocal(bucketName, s3ObjectKey, localFileName);
-	}
 }
diff --git a/src/main/java/com/hedera/downloader/RecordFileDownloader.java b/src/main/java/com/hedera/downloader/RecordFileDownloader.java
index 737408e96..669bb612a 100644
--- a/src/main/java/com/hedera/downloader/RecordFileDownloader.java
+++ b/src/main/java/com/hedera/downloader/RecordFileDownloader.java
@@ -3,6 +3,7 @@ package com.hedera.downloader;
 import com.google.gson.JsonIOException;
 import com.google.gson.JsonSyntaxException;
 import com.hedera.configLoader.ConfigLoader;
+import com.hedera.configLoader.ConfigLoader.OPERATION_TYPE;
 import com.hedera.parser.RecordFileParser;
 import com.hedera.signatureVerifier.NodeSignatureVerifier;
 import com.hedera.utilities.Utility;
@@ -23,10 +24,12 @@ import java.util.stream.Stream;
 
 public class RecordFileDownloader extends Downloader {
 
-	private static String validRcdDir = null;
-	private static String s3prefix = "";
+	private static String validDir = ConfigLoader.getDefaultParseDir(OPERATION_TYPE.RECORDS);
+	private static String tmpDir = ConfigLoader.getDefaultTmpDir(OPERATION_TYPE.RECORDS);
 
 	public RecordFileDownloader() {
+		Utility.createDirIfNotExists(validDir);
+		Utility.createDirIfNotExists(tmpDir);
 	}
 
 	public static void downloadNewRecordfiles(RecordFileDownloader downloader) {
@@ -39,9 +42,9 @@ public class RecordFileDownloader extends Downloader {
 			// Verify signature files and download .rcd files of valid signature files
 			downloader.verifySigsAndDownloadRecordFiles(sigFilesMap);
 
-			if (validRcdDir != null) {
+			if (validDir != null) {
 //				new Thread(() -> {
-					verifyValidRecordFiles(validRcdDir);
+					verifyValidRecordFiles(validDir);
 //				}).start();
 			} else {
 			}
@@ -145,12 +148,6 @@ public class RecordFileDownloader extends Downloader {
 		// reload address book and keys
 		NodeSignatureVerifier verifier = new NodeSignatureVerifier();
 
-		validRcdDir = null;
-		s3prefix = ConfigLoader.getRecordFilesS3Location();
-		if (s3prefix.endsWith("/")) {
-			s3prefix = s3prefix.substring(0, s3prefix.length()-2);
-		}
-
 		for (String fileName : sigFilesMap.keySet()) {
 			if (Utility.checkStopFile()) {
 				log.info(MARKER, "Stop file found, stopping");
@@ -169,14 +166,16 @@ public class RecordFileDownloader extends Downloader {
 							log.info(MARKER, "Stop file found, stopping");
 							break;
 						}
-						if (validRcdDir == null) {
-							validRcdDir = validSigFile.getParentFile().getParent() + "/valid/";
-						}
-						Pair<Boolean, File> rcdFileResult = downloadRcdFile(validSigFile, validRcdDir);
+						
+						Pair<Boolean, File> rcdFileResult = downloadFile(DownloadType.RCD, validSigFile, tmpDir);
 						File rcdFile = rcdFileResult.getRight();
-						if (rcdFile != null &&
-								Utility.hashMatch(validSigFile, rcdFile)) {
-							break;
+						if (rcdFile != null && Utility.hashMatch(validSigFile, rcdFile)) {
+							// move the file to the valid directory
+					        File fTo = new File(validDir + "/" + rcdFile.getName());
+
+					        if (moveFile(rcdFile, fTo)) {
+					        	break;
+					        }
 						} else if (rcdFile != null) {
 							log.warn(MARKER, "{}'s Hash doesn't match the Hash contained in valid signature file. Will try to download a rcd file with same timestamp from other nodes and check the Hash.", rcdFile.getPath());
 						}
@@ -186,17 +185,7 @@ public class RecordFileDownloader extends Downloader {
 				}
 			}
 		}
-		return validRcdDir;
-	}
-
-	Pair<Boolean, File> downloadRcdFile(File sigFile, String validRcdDir) {
-		String nodeAccountId = Utility.getAccountIDStringFromFilePath(sigFile.getPath());
-		String sigFileName = sigFile.getName();
-		String rcdFileName = sigFileName.replace(".rcd_sig", ".rcd");
-//		String s3ObjectKey = "recordstreams/record" + nodeAccountId + "/" + rcdFileName;
-		String s3ObjectKey =  s3prefix + nodeAccountId + "/" + rcdFileName;
-//		String localFileName = validRcdDir + rcdFileName;
-		return saveToLocal(bucketName, s3ObjectKey, validRcdDir + rcdFileName);
+		return validDir;
 	}
 
 }
diff --git a/src/main/java/com/hedera/utilities/Utility.java b/src/main/java/com/hedera/utilities/Utility.java
index b67888b66..6f21b2bcc 100644
--- a/src/main/java/com/hedera/utilities/Utility.java
+++ b/src/main/java/com/hedera/utilities/Utility.java
@@ -45,8 +45,8 @@ public class Utility {
 	private static final Marker LOGM_EXCEPTION = MarkerManager.getMarker("EXCEPTION");
   private static final Long SCALAR = 1_000_000_000L;
 
-	private static final byte TYPE_PREV_HASH = 1;       // next 48 bytes are hash384 of previous files
-	private static final byte TYPE_RECORD = 2;          // next data type is transaction and its record
+//	private static final byte TYPE_PREV_HASH = 1;       // next 48 bytes are hash384 of previous files
+//	private static final byte TYPE_RECORD = 2;          // next data type is transaction and its record
 	private static final byte TYPE_SIGNATURE = 3;       // the file content signature, should not be hashed
 	private static final byte TYPE_FILE_HASH = 4;       // next 48 bytes are hash384 of content of corresponding RecordFile