Hazzzardous commited on
Commit
b549737
1 Parent(s): bbd61dc

Create config.py

Browse files
Files changed (1) hide show
  1. config.py +63 -0
config.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from rwkvstic.agnostic.backends import TORCH, TORCH_QUANT
2
+ import torch
3
+
4
+ quantized = {
5
+ "mode": TORCH_QUANT,
6
+ "runtimedtype": torch.bfloat16,
7
+ "useGPU": torch.cuda.is_available(),
8
+ "chunksize": 32, # larger = more accurate, but more memory
9
+ "target": 100 # your gpu max size, excess vram offloaded to cpu
10
+ }
11
+
12
+ # UNCOMMENT TO SELECT OPTIONS
13
+ # Not full list of options, see https://pypi.org/project/rwkvstic/ and https://huggingface.co/BlinkDL/ for more models/modes
14
+
15
+ # RWKV 1B5 instruct test 1 model
16
+ # Approximate
17
+ # [Vram usage: 6.0GB]
18
+ # [File size: 3.0GB]
19
+
20
+
21
+ config = {
22
+ "path":"https://huggingface.co/BlinkDL/rwkv-4-pile-7b/resolve/main/RWKV-4-Pile-7B-Instruct-test1-20230124.pth",
23
+ "mode":TORCH,
24
+ "runtimedtype":torch.bfloat16,
25
+ "useGPU":torch.cuda.is_available(),
26
+ "dtype":torch.bfloat16
27
+ }
28
+
29
+ title = "RWKV-4 (7B Instruct)"
30
+
31
+ # RWKV 1B5 instruct model quantized
32
+ # Approximate
33
+ # [Vram usage: 1.3GB]
34
+ # [File size: 3.0GB]
35
+
36
+ # config = {
37
+ # "path": "https://huggingface.co/BlinkDL/rwkv-4-pile-1b5/resolve/main/RWKV-4-Pile-1B5-Instruct-test1-20230124.pth",
38
+ # **quantized
39
+ # }
40
+
41
+ # title = "RWKV-4 (1.5b Instruct Quantized)"
42
+
43
+ # RWKV 7B instruct pre-quantized (settings baked into model)
44
+ # Approximate
45
+ # [Vram usage: 7.0GB]
46
+ # [File size: 8.0GB]
47
+
48
+ # config = {
49
+ # "path": "https://huggingface.co/Hazzzardous/RWKV-8Bit/resolve/main/RWKV-4-Pile-7B-Instruct.pqth"
50
+ # }
51
+
52
+ # title = "RWKV-4 (7b Instruct Quantized)"
53
+
54
+ # RWKV 14B quantized (latest as of feb 9)
55
+ # Approximate
56
+ # [Vram usage: 15.0GB]
57
+ # [File size: 28.0GB]
58
+
59
+ # config = {
60
+ # "path": "https://huggingface.co/BlinkDL/rwkv-4-pile-14b/resolve/main/RWKV-4-Pile-14B-20230204-7324.pth"
61
+ # }
62
+
63
+ # title = "RWKV-4 (14b)"