seungrokj commited on
Commit
84c8b87
1 Parent(s): d46bde1
Files changed (2) hide show
  1. chunk_paths.json +1 -0
  2. merge.py +26 -0
chunk_paths.json ADDED
@@ -0,0 +1 @@
 
 
1
+ ["llama-01-of-10.safetensors", "llama-02-of-10.safetensors", "llama-03-of-10.safetensors", "llama-04-of-10.safetensors", "llama-05-of-10.safetensors", "llama-06-of-10.safetensors", "llama-07-of-10.safetensors", "llama-08-of-10.safetensors", "llama-09-of-10.safetensors", "llama-10-of-10.safetensors"]
merge.py ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import math
3
+ import json
4
+
5
+ OUTPUT_FILE_NAME = "llama.safetensors" # Merge output file name
6
+ CHUNK_PATHS_FILE = "chunk_paths.json"
7
+
8
+ def merge(chunk_paths):
9
+ output_path = os.path.join(os.path.dirname(chunk_paths[0]), OUTPUT_FILE_NAME)
10
+
11
+ with open(output_path, "wb") as f_out:
12
+ for filepath in chunk_paths:
13
+ with open(filepath, "rb") as f_in:
14
+ f_out.write(f_in.read())
15
+
16
+ print(f"Merged file saved to {output_path}")
17
+
18
+ if __name__ == "__main__":
19
+
20
+ if os.path.exists(CHUNK_PATHS_FILE):
21
+ with open(CHUNK_PATHS_FILE) as f:
22
+ chunk_paths = json.load(f)
23
+ else:
24
+ chunk_paths = split(main_filepath)
25
+
26
+ merge(chunk_paths)