Hugging Face's logo

weizhiwang
/
unifilter_mllm_pretrain_checkpoints

Model card Files
xet
Community
weizhiwang commited on
Commit
2b086bd
·
verified ·
1 Parent(s): 6bc2659

Upload folder using huggingface_hub

Browse files
Files changed (8) hide show
  1. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-005000-epoch-00-loss=2.0247.pt +3 -0
  2. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-010000-epoch-00-loss=1.9681.pt +3 -0
  3. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-015000-epoch-00-loss=2.0020.pt +3 -0
  4. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-019073-epoch-00-loss=1.8610.pt +3 -0
  5. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/config.json +74 -0
  6. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/config.yaml +67 -0
  7. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7.jsonl +0 -0
  8. obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/run-metrics.jsonl +1 -0
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-005000-epoch-00-loss=2.0247.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6ea404b1a4f73402d4b9b01686c5e1a72f80b41999ec1ab1520508fcb42f622d
3
+ size 15336322587
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-010000-epoch-00-loss=1.9681.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3e1e2bef7774e234a4ca79dd73a28592a80b3dfc2e3b06236c7a710dc3624888
3
+ size 15336322587
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-015000-epoch-00-loss=2.0020.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a91f7a99c2b4817407e079e100cecfdc3fd5c0645f4589049dd7f805a0cfa849
3
+ size 15336322587
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/checkpoints/step-019073-epoch-00-loss=1.8610.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:438ec42470c1625c168c5eee3e4240d11c7968d1433bfe0a8f8260c291d866a9
3
+ size 15336322587
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/config.json ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dataset": {
3
+ "align_stage_components": [
4
+ "download/llava-laion-cc-sbu-558k/chat.json",
5
+ "download/llava-laion-cc-sbu-558k"
6
+ ],
7
+ "dataset_id": "obelics",
8
+ "dataset_resampled": true,
9
+ "dataset_root_dir": "/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/datacomp/datacomp_hq_single_pkl_pil:/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/OBELICS/obelics_hq_single_pkl_pil:/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/synthdog-en/synthdog_single_pkl_pil/",
10
+ "finetune_stage_components": [
11
+ "/fsx-training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/llava/llava_v1_5_mix665k.json",
12
+ "/fsx-training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/llava/data"
13
+ ],
14
+ "max_num_images": 6,
15
+ "min_num_images": 1,
16
+ "train_num_samples": 4882814,
17
+ "type": "obelics",
18
+ "workers": 4
19
+ },
20
+ "model": {
21
+ "align_epochs": 1,
22
+ "align_global_batch_size": 256,
23
+ "align_learning_rate": 0.001,
24
+ "align_lr_scheduler_type": "linear-warmup+cosine-decay",
25
+ "align_max_grad_norm": 1.0,
26
+ "align_max_steps": null,
27
+ "align_per_device_batch_size": 16,
28
+ "align_train_strategy": "fsdp-shard-grad-op",
29
+ "align_warmup_ratio": 0.03,
30
+ "align_weight_decay": 0.0,
31
+ "arch_specifier": "no-align+avgpool",
32
+ "enable_gradient_checkpointing": true,
33
+ "enable_mixed_precision_training": true,
34
+ "finetune_epochs": 1,
35
+ "finetune_global_batch_size": 128,
36
+ "finetune_learning_rate": 2e-05,
37
+ "finetune_lr_scheduler_type": "linear-warmup+cosine-decay",
38
+ "finetune_max_grad_norm": 1.0,
39
+ "finetune_max_steps": null,
40
+ "finetune_per_device_batch_size": 16,
41
+ "finetune_train_strategy": "fsdp-full-shard",
42
+ "finetune_warmup_ratio": 0.03,
43
+ "finetune_weight_decay": 0.1,
44
+ "image_resize_strategy": "resize-naive",
45
+ "llm_backbone_id": "phi3-3b",
46
+ "llm_max_length": 4096,
47
+ "model_id": "phi3-3b-continue-training-hq-20b",
48
+ "pretrain_epochs": 1,
49
+ "pretrain_global_batch_size": 256,
50
+ "pretrain_learning_rate": 5e-05,
51
+ "pretrain_lr_scheduler_type": "linear-warmup+cosine-decay",
52
+ "pretrain_max_grad_norm": 1.0,
53
+ "pretrain_max_steps": 19073,
54
+ "pretrain_per_device_batch_size": 8,
55
+ "pretrain_train_strategy": "fsdp-full-shard",
56
+ "pretrain_warmup_ratio": 0.03,
57
+ "pretrain_weight_decay": 0.01,
58
+ "reduce_in_full_precision": false,
59
+ "type": "one-stage+7b",
60
+ "vision_backbone_id": "siglip-vit-so400m-384px"
61
+ },
62
+ "mount_path": "/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/models",
63
+ "pretrained_checkpoint": null,
64
+ "run_id": "obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7",
65
+ "run_root_dir": "/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/checkpoints/mllm_pretrain_checkpoints",
66
+ "seed": 7,
67
+ "stage": "pretrain",
68
+ "trackers": [
69
+ "jsonl",
70
+ "wandb"
71
+ ],
72
+ "wandb_entity": null,
73
+ "wandb_project": "mmpretrain"
74
+ }
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/config.yaml ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ dataset:
2
+ align_stage_components:
3
+ - download/llava-laion-cc-sbu-558k/chat.json
4
+ - download/llava-laion-cc-sbu-558k
5
+ dataset_id: obelics
6
+ dataset_resampled: true
7
+ dataset_root_dir: /fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/datacomp/datacomp_hq_single_pkl_pil:/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/OBELICS/obelics_hq_single_pkl_pil:/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/synthdog-en/synthdog_single_pkl_pil/
8
+ finetune_stage_components:
9
+ - /fsx-training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/llava/llava_v1_5_mix665k.json
10
+ - /fsx-training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/llava/data
11
+ max_num_images: 6
12
+ min_num_images: 1
13
+ train_num_samples: 4882814
14
+ type: obelics
15
+ workers: 4
16
+ model:
17
+ align_epochs: 1
18
+ align_global_batch_size: 256
19
+ align_learning_rate: 0.001
20
+ align_lr_scheduler_type: linear-warmup+cosine-decay
21
+ align_max_grad_norm: 1.0
22
+ align_max_steps: null
23
+ align_per_device_batch_size: 16
24
+ align_train_strategy: fsdp-shard-grad-op
25
+ align_warmup_ratio: 0.03
26
+ align_weight_decay: 0.0
27
+ arch_specifier: no-align+avgpool
28
+ enable_gradient_checkpointing: true
29
+ enable_mixed_precision_training: true
30
+ finetune_epochs: 1
31
+ finetune_global_batch_size: 128
32
+ finetune_learning_rate: 2.0e-05
33
+ finetune_lr_scheduler_type: linear-warmup+cosine-decay
34
+ finetune_max_grad_norm: 1.0
35
+ finetune_max_steps: null
36
+ finetune_per_device_batch_size: 16
37
+ finetune_train_strategy: fsdp-full-shard
38
+ finetune_warmup_ratio: 0.03
39
+ finetune_weight_decay: 0.1
40
+ image_resize_strategy: resize-naive
41
+ llm_backbone_id: phi3-3b
42
+ llm_max_length: 4096
43
+ model_id: phi3-3b-continue-training-hq-20b
44
+ pretrain_epochs: 1
45
+ pretrain_global_batch_size: 256
46
+ pretrain_learning_rate: 5.0e-05
47
+ pretrain_lr_scheduler_type: linear-warmup+cosine-decay
48
+ pretrain_max_grad_norm: 1.0
49
+ pretrain_max_steps: 19073
50
+ pretrain_per_device_batch_size: 8
51
+ pretrain_train_strategy: fsdp-full-shard
52
+ pretrain_warmup_ratio: 0.03
53
+ pretrain_weight_decay: 0.01
54
+ reduce_in_full_precision: false
55
+ type: one-stage+7b
56
+ vision_backbone_id: siglip-vit-so400m-384px
57
+ mount_path: /fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/models
58
+ pretrained_checkpoint: null
59
+ run_id: obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7
60
+ run_root_dir: /fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/checkpoints/mllm_pretrain_checkpoints
61
+ seed: 7
62
+ stage: pretrain
63
+ trackers:
64
+ - jsonl
65
+ - wandb
66
+ wandb_entity: null
67
+ wandb_project: mmpretrain
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7.jsonl ADDED
The diff for this file is too large to render. See raw diff
 
obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7/run-metrics.jsonl ADDED
@@ -0,0 +1 @@
 
 
1
+ {"hparams": {"dataset": {"align_stage_components": ["download/llava-laion-cc-sbu-558k/chat.json", "download/llava-laion-cc-sbu-558k"], "dataset_id": "obelics", "dataset_resampled": true, "dataset_root_dir": "/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/datacomp/datacomp_hq_single_pkl_pil:/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/OBELICS/obelics_hq_single_pkl_pil:/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/synthdog-en/synthdog_single_pkl_pil/", "finetune_stage_components": ["/fsx-training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/llava/llava_v1_5_mix665k.json", "/fsx-training/shopqa-training-fsx-prod-us-east-1/home/wzwang/data/llava/data"], "max_num_images": 6, "min_num_images": 1, "train_num_samples": 4882814, "type": "obelics", "workers": 4}, "model": {"align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "no-align+avgpool", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 16, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "resize-naive", "llm_backbone_id": "phi3-3b", "llm_max_length": 4096, "model_id": "phi3-3b-continue-training-hq-20b", "pretrain_epochs": 1, "pretrain_global_batch_size": 256, "pretrain_learning_rate": 5e-05, "pretrain_lr_scheduler_type": "linear-warmup+cosine-decay", "pretrain_max_grad_norm": 1.0, "pretrain_max_steps": 19073, "pretrain_per_device_batch_size": 8, "pretrain_train_strategy": "fsdp-full-shard", "pretrain_warmup_ratio": 0.03, "pretrain_weight_decay": 0.01, "reduce_in_full_precision": false, "type": "one-stage+7b", "vision_backbone_id": "siglip-vit-so400m-384px"}, "mount_path": "/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/models", "pretrained_checkpoint": null, "run_id": "obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7", "run_root_dir": "/fsx-Training/shopqa-training-fsx-prod-us-east-1/home/wzwang/checkpoints/mllm_pretrain_checkpoints", "seed": 7, "stage": "pretrain", "trackers": ["jsonl", "wandb"], "wandb_entity": null, "wandb_project": "mmpretrain"}, "run_id": "obelics+phi3-3b-continue-training-hq-20b+stage-pretrain+x7"}