nousr commited on
Commit
be5139b
1 Parent(s): 6b47107

Upload h14-many-gpus-resume/prior_config.json with huggingface_hub

Browse files
h14-many-gpus-resume/prior_config.json ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "prior": {
3
+ "clip": {
4
+ "make": "openclip",
5
+ "model": "ViT-H-14"
6
+ },
7
+ "net": {
8
+ "dim": 1024,
9
+ "depth": 12,
10
+ "num_timesteps": 1000,
11
+ "max_text_len": 77,
12
+ "num_time_embeds": 1,
13
+ "num_image_embeds": 1,
14
+ "num_text_embeds": 1,
15
+ "dim_head": 64,
16
+ "heads": 12,
17
+ "ff_mult": 4,
18
+ "norm_out": true,
19
+ "attn_dropout": 0.05,
20
+ "ff_dropout": 0.05,
21
+ "final_proj": true,
22
+ "normformer": true,
23
+ "rotary_emb": true
24
+ },
25
+ "image_embed_dim": 1024,
26
+ "image_size": 224,
27
+ "image_channels": 3,
28
+ "timesteps": 1000,
29
+ "sample_timesteps": 64,
30
+ "cond_drop_prob": 0.1,
31
+ "loss_type": "l2",
32
+ "predict_x_start": true,
33
+ "beta_schedule": "cosine",
34
+ "condition_on_text_encodings": true
35
+ },
36
+ "data": {
37
+ "batch_size": 350,
38
+ "num_data_points": 2e9,
39
+ "eval_every_seconds": 1600,
40
+ "image_url": "s3://s-laion/vit-h-14-embeddings/img_emb/",
41
+ "meta_url": "s3://s-laion/vit-h-14-embeddings/metadata/",
42
+ "splits": {
43
+ "train": 0.9,
44
+ "val": 2.8e-6,
45
+ "test": 0.0999972
46
+ }
47
+ },
48
+ "train": {
49
+ "epochs": 5,
50
+ "lr": 1.1e-4,
51
+ "wd": 6.02e-2,
52
+ "max_grad_norm": 0.5,
53
+ "use_ema": true,
54
+ "ema_beta": 0.9999,
55
+ "ema_update_after_step": 51,
56
+ "warmup_steps": 50,
57
+ "amp": false,
58
+ "save_every_seconds": 3600,
59
+ "eval_timesteps": [64, 1000],
60
+ "random_seed": 84513
61
+ },
62
+ "tracker": {
63
+ "data_path": ".prior-big-run-resume",
64
+ "overwrite_data_path": true,
65
+ "log": {
66
+ "log_type": "wandb",
67
+ "wandb_entity": "nousr_laion",
68
+ "wandb_project": "h-14-prior",
69
+ "wandb_resume": false,
70
+ "verbose": true
71
+ },
72
+ "save": [
73
+ {
74
+ "save_to": "local",
75
+ "save_type": "checkpoint",
76
+ "save_latest_to": ".prior-big-run-resume/latest_checkpoint.pth",
77
+ "save_best_to": ".prior-big-run-resume/best_checkpoint.pth"
78
+ },
79
+ {
80
+ "save_to": "local",
81
+ "ave_type": "model",
82
+ "save_latest_to": ".prior-big-run-resume/latest_model.pth",
83
+ "save_best_to": ".prior-big-run-resume/best_model.pth"
84
+ },
85
+ {
86
+ "save_to": "huggingface",
87
+ "huggingface_repo": "nousr/dalle2-pytorch",
88
+ "save_meta_to": "h14-many-gpus-resume/",
89
+ "save_latest_to": "h14-many-gpus-resume/latest_model.pth",
90
+ "save_best_to": "h14-many-gpus-resume/best_model.pth",
91
+ "save_type": "model"
92
+ },
93
+ {
94
+ "save_to": "huggingface",
95
+ "huggingface_repo": "nousr/dalle2-pytorch",
96
+ "save_meta_to": "h14-many-gpus-resume/",
97
+ "save_latest_to": "h14-many-gpus-resume/latest_checkpoint.pth",
98
+ "save_best_to": "h14-many-gpus-resume/best_checkpoint.pth",
99
+ "save_type": "checkpoint"
100
+ }
101
+ ],
102
+ "load": {
103
+ "load_from": "local",
104
+ "file_path": ".prior-big-run/latest_checkpoint.pth"
105
+ }
106
+ }
107
+ }
108
+