diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt
new file mode 100644
index 0000000000000000000000000000000000000000..bab4be13a36451dfbb10a3b86def1776ef268400
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e556733a87a7fb5b882d07142897e25a279e52f340cf2e737f672d698cac82c1
+size 1795822369
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1a9d46cfa102d30e8dd18a9b60031c55b19d0bb3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_1.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:15,  1.26it/s]
  1%|          | 6/1000 [00:00<01:54,  8.65it/s]
  1%|          | 12/1000 [00:01<00:57, 17.24it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.49it/s]
  2%|▏         | 24/1000 [00:01<00:29, 32.73it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.77it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.60it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.29it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.12it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.20it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.68it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.80it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.51it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.06it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.49it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.85it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.28it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.48it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.79it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.91it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.94it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.92it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.94it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.98it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.69it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.69it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.66it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.59it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.57it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.52it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.36it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.26it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.37it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.32it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.55it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.52it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.44it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.50it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.60it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.64it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.70it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.49it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.56it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.76it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.86it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.06it/s]
 30%|███       | 300/1000 [00:06<00:12, 58.03it/s]
 31%|███       | 306/1000 [00:06<00:11, 57.99it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.06it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.89it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.97it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.07it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.15it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.96it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.02it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.05it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.97it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.63it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.63it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.75it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.91it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.97it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.02it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.03it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.17it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.19it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.99it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.06it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.08it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.17it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.09it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.06it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.07it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.98it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.03it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.11it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.14it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.17it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.06it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.01it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.08it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.98it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.93it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.05it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.04it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.96it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.99it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.99it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.08it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.17it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.00it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.95it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.12it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.06it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.15it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.18it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.10it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.12it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.13it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.16it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.04it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.15it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.10it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.07it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.12it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.10it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.26it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.09it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.19it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.14it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.01it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.07it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.91it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.93it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.99it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.97it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.96it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 58.00it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.10it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.22it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.28it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.27it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.21it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.22it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.16it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.22it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.07it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.14it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.12it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.14it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.23it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.27it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.28it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.26it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.34it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.41it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.33it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.15it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.37it/s]
+2022-12-21,09:37:11 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:46, 107.03it/s]
  1%|          | 512/50176 [00:04<07:16, 113.68it/s]
  2%|▏         | 768/50176 [00:06<07:28, 110.04it/s]
  2%|▏         | 1024/50176 [00:09<08:01, 102.02it/s]
  3%|▎         | 1280/50176 [00:12<08:09, 99.84it/s] 
  3%|▎         | 1536/50176 [00:15<08:59, 90.20it/s]
  4%|▎         | 1792/50176 [00:18<08:41, 92.86it/s]
  4%|▍         | 2048/50176 [00:21<08:32, 93.89it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.58it/s]
  5%|▌         | 2560/50176 [00:26<08:08, 97.50it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.22it/s]
  6%|▌         | 3072/50176 [00:31<08:17, 94.75it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.91it/s]
  7%|▋         | 3584/50176 [00:36<07:42, 100.65it/s]
  8%|▊         | 3840/50176 [00:39<07:59, 96.73it/s] 
  8%|▊         | 4096/50176 [00:42<08:11, 93.71it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.12it/s]
  9%|▉         | 4608/50176 [00:47<07:36, 99.73it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.93it/s]
 10%|█         | 5120/50176 [00:52<07:20, 102.37it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.30it/s]
 11%|█         | 5632/50176 [00:57<07:16, 101.96it/s]
 12%|█▏        | 5888/50176 [00:59<07:14, 101.87it/s]
 12%|█▏        | 6144/50176 [01:02<07:43, 94.98it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.97it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.36it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.60it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.54it/s]
 15%|█▍        | 7424/50176 [01:15<07:08, 99.67it/s]
 15%|█▌        | 7680/50176 [01:18<07:05, 99.84it/s]
 16%|█▌        | 7936/50176 [01:20<06:49, 103.28it/s]
 16%|█▋        | 8192/50176 [01:22<06:48, 102.90it/s]
 17%|█▋        | 8448/50176 [01:25<06:49, 101.90it/s]
 17%|█▋        | 8704/50176 [01:27<06:48, 101.40it/s]
 18%|█▊        | 8960/50176 [01:30<06:47, 101.09it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.73it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.96it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.24it/s]
 20%|█▉        | 9984/50176 [01:40<06:46, 98.76it/s] 
 20%|██        | 10240/50176 [01:43<06:31, 102.07it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.12it/s]
 21%|██▏       | 10752/50176 [01:48<06:28, 101.39it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.16it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.68it/s]
 23%|██▎       | 11520/50176 [01:55<06:08, 104.82it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.61it/s]
 24%|██▍       | 12032/50176 [02:00<06:02, 105.36it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.68it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.83it/s]
 26%|██▌       | 12800/50176 [02:07<05:51, 106.48it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.22it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.63it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.13it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 99.98it/s] 
 28%|██▊       | 14080/50176 [02:19<05:58, 100.62it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.46it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.42it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.97it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.36it/s]
 31%|███       | 15360/50176 [02:32<05:49, 99.67it/s]
 31%|███       | 15616/50176 [02:35<05:37, 102.52it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.79it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.86it/s]
 33%|███▎      | 16384/50176 [02:42<05:27, 103.28it/s]
 33%|███▎      | 16640/50176 [02:45<05:40, 98.35it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.60it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 103.95it/s]
 35%|███▍      | 17408/50176 [02:52<05:18, 103.03it/s]
 35%|███▌      | 17664/50176 [02:55<05:08, 105.39it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.16it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.38it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.62it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.39it/s]
 38%|███▊      | 18944/50176 [03:07<05:02, 103.10it/s]
 38%|███▊      | 19200/50176 [03:10<05:10, 99.68it/s] 
 39%|███▉      | 19456/50176 [03:12<05:13, 97.90it/s]
 39%|███▉      | 19712/50176 [03:15<05:10, 98.01it/s]
 40%|███▉      | 19968/50176 [03:18<05:08, 97.92it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.82it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.60it/s]
 41%|████▏     | 20736/50176 [03:24<04:38, 105.84it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.23it/s]
 42%|████▏     | 21248/50176 [03:30<04:43, 102.11it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.43it/s]
 43%|████▎     | 21760/50176 [03:35<04:36, 102.72it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.58it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 107.77it/s]
 45%|████▍     | 22528/50176 [03:41<04:16, 107.99it/s]
 45%|████▌     | 22784/50176 [03:44<04:24, 103.40it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.65it/s]
 46%|████▋     | 23296/50176 [03:49<04:27, 100.46it/s]
 47%|████▋     | 23552/50176 [03:52<04:32, 97.85it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.78it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.90it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.56it/s]
 49%|████▉     | 24576/50176 [04:02<04:15, 100.33it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.13it/s]
 50%|█████     | 25088/50176 [04:07<03:59, 104.57it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.77it/s]
 51%|█████     | 25600/50176 [04:12<03:54, 104.79it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.61it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.67it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.03it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.37it/s]
 54%|█████▎    | 26880/50176 [04:24<03:40, 105.68it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 105.03it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.43it/s]
 55%|█████▌    | 27648/50176 [04:30<03:22, 111.09it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.44it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.26it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.24it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 101.98it/s]
 58%|█████▊    | 28928/50176 [04:43<03:24, 103.77it/s]
 58%|█████▊    | 29184/50176 [04:46<03:21, 104.10it/s]
 59%|█████▊    | 29440/50176 [04:49<03:30, 98.50it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:29, 97.58it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.10it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.87it/s]
 61%|██████    | 30464/50176 [04:59<03:09, 104.03it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.27it/s]
 62%|██████▏   | 30976/50176 [05:04<03:06, 102.77it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.54it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.51it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.22it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 106.94it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 106.92it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.50it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.36it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.16it/s]
 66%|██████▋   | 33280/50176 [05:26<02:42, 103.72it/s]
 67%|██████▋   | 33536/50176 [05:28<02:40, 103.43it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.58it/s]
 68%|██████▊   | 34048/50176 [05:33<02:28, 108.59it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.07it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.08it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.39it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.88it/s]
 70%|███████   | 35328/50176 [05:50<03:01, 81.62it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.67it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.84it/s]
 72%|███████▏  | 36096/50176 [06:02<03:30, 66.90it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.70it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.88it/s]
 73%|███████▎  | 36864/50176 [06:14<03:16, 67.61it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.68it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.07it/s]
 75%|███████▌  | 37632/50176 [06:20<02:17, 91.55it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.23it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.90it/s]
 77%|███████▋  | 38400/50176 [06:35<03:19, 59.08it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 67.99it/s]
 78%|███████▊  | 38912/50176 [06:42<03:00, 62.24it/s]
 78%|███████▊  | 39168/50176 [06:45<02:38, 69.40it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.43it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.91it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.15it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.53it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.08it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.95it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.74it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.98it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.63it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.10it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.64it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.64it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.54it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.84it/s]
 86%|████████▌ | 43264/50176 [07:22<01:04, 107.85it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.15it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.41it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.48it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.56it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.37it/s]
 89%|████████▉ | 44800/50176 [07:38<00:52, 102.10it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.23it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 100.96it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 103.99it/s]
 91%|█████████▏| 45824/50176 [07:47<00:41, 104.14it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.59it/s]
 92%|█████████▏| 46336/50176 [07:53<00:37, 102.42it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.61it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.79it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.44it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.03it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.62it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.91it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.62it/s]
 96%|█████████▋| 48384/50176 [08:13<00:17, 103.25it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.85it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.56it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.93it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.30it/s]
 99%|█████████▉| 49664/50176 [08:25<00:05, 100.05it/s]
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.69it/s] 
100%|██████████| 50176/50176 [08:29<00:00, 125.05it/s]
100%|██████████| 50176/50176 [08:29<00:00, 98.48it/s] 
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.0824	imagenet-zeroshot-val-top5: 0.2147
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt
new file mode 100644
index 0000000000000000000000000000000000000000..7fbf15d2ea689d9f44fad0358af81e6e978294b8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:72084bdc52bf7efc62684a4d92a1ce685652690b429d5fe5cd6f9f2fb627d95e
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..247325e47ba3dcbf15888cd8c40f5ed94bca8f85
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_10.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:09,  1.27it/s]
  1%|          | 6/1000 [00:00<01:54,  8.68it/s]
  1%|          | 12/1000 [00:01<00:57, 17.30it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.22it/s]
  2%|▏         | 24/1000 [00:01<00:30, 32.51it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.66it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.48it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.26it/s]
  5%|▍         | 48/1000 [00:01<00:19, 50.08it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.18it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.72it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.87it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.66it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.20it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.79it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.14it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.56it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.75it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.97it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.12it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.04it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.97it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.05it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.03it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.04it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.15it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.04it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.89it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.89it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.79it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.74it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.68it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.62it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.39it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.28it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.39it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.61it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.61it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.52it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.57it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.68it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.79it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.82it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.65it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.87it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.96it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.18it/s]
 30%|███       | 300/1000 [00:06<00:12, 58.10it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.08it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.21it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.15it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.02it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.14it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.26it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.04it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 58.00it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.09it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.20it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.09it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.77it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.84it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.94it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.07it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.16it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.14it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.17it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.21it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.11it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.16it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.98it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.13it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.24it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.15it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.13it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.04it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.11it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.01it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.10it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.24it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.16it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.26it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.26it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.16it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.17it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.22it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.10it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.04it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.19it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.96it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.91it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.00it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.18it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.20it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.30it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.13it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.09it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.25it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.20it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.27it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.30it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.28it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.15it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.26it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.24it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.28it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.25it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.27it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.26it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.33it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.40it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.23it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.30it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.23it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.28it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.28it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.01it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.08it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.06it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.11it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.23it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.39it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.42it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.34it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.41it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.37it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.39it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.28it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.33it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.18it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.28it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.28it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.23it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.26it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.21it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.32it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.32it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.33it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.36it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.37it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.46it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.42it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.25it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.46it/s]
+2022-12-21,09:37:11 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:54, 105.19it/s]
  1%|          | 512/50176 [00:04<07:18, 113.27it/s]
  2%|▏         | 768/50176 [00:07<07:31, 109.44it/s]
  2%|▏         | 1024/50176 [00:09<08:01, 102.01it/s]
  3%|▎         | 1280/50176 [00:12<08:10, 99.66it/s] 
  3%|▎         | 1536/50176 [00:15<09:00, 90.03it/s]
  4%|▎         | 1792/50176 [00:18<08:40, 93.03it/s]
  4%|▍         | 2048/50176 [00:21<08:33, 93.69it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.68it/s]
  5%|▌         | 2560/50176 [00:26<08:08, 97.42it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.18it/s]
  6%|▌         | 3072/50176 [00:31<08:17, 94.70it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.84it/s]
  7%|▋         | 3584/50176 [00:36<07:42, 100.64it/s]
  8%|▊         | 3840/50176 [00:39<07:58, 96.79it/s] 
  8%|▊         | 4096/50176 [00:42<08:12, 93.59it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.21it/s]
  9%|▉         | 4608/50176 [00:47<07:37, 99.64it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.84it/s]
 10%|█         | 5120/50176 [00:52<07:20, 102.34it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.30it/s]
 11%|█         | 5632/50176 [00:57<07:16, 102.02it/s]
 12%|█▏        | 5888/50176 [00:59<07:14, 101.89it/s]
 12%|█▏        | 6144/50176 [01:02<07:43, 94.91it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.96it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.35it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.65it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.52it/s]
 15%|█▍        | 7424/50176 [01:15<07:08, 99.68it/s]
 15%|█▌        | 7680/50176 [01:18<07:05, 99.81it/s]
 16%|█▌        | 7936/50176 [01:20<06:49, 103.18it/s]
 16%|█▋        | 8192/50176 [01:22<06:48, 102.89it/s]
 17%|█▋        | 8448/50176 [01:25<06:48, 102.12it/s]
 17%|█▋        | 8704/50176 [01:28<06:48, 101.50it/s]
 18%|█▊        | 8960/50176 [01:30<06:47, 101.03it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.72it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.97it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.26it/s]
 20%|█▉        | 9984/50176 [01:40<06:46, 98.87it/s] 
 20%|██        | 10240/50176 [01:43<06:31, 102.06it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.09it/s]
 21%|██▏       | 10752/50176 [01:48<06:28, 101.36it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.08it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.76it/s]
 23%|██▎       | 11520/50176 [01:55<06:08, 104.94it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.56it/s]
 24%|██▍       | 12032/50176 [02:00<06:01, 105.44it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.51it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.80it/s]
 26%|██▌       | 12800/50176 [02:07<05:51, 106.45it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.19it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.70it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.04it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 100.07it/s]
 28%|██▊       | 14080/50176 [02:19<05:58, 100.57it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.49it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.50it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.84it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.50it/s]
 31%|███       | 15360/50176 [02:32<05:49, 99.66it/s]
 31%|███       | 15616/50176 [02:35<05:36, 102.60it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.62it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.79it/s]
 33%|███▎      | 16384/50176 [02:42<05:27, 103.34it/s]
 33%|███▎      | 16640/50176 [02:45<05:41, 98.34it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.60it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 103.93it/s]
 35%|███▍      | 17408/50176 [02:52<05:17, 103.12it/s]
 35%|███▌      | 17664/50176 [02:55<05:08, 105.34it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.11it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.42it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.63it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.34it/s]
 38%|███▊      | 18944/50176 [03:07<05:03, 103.07it/s]
 38%|███▊      | 19200/50176 [03:10<05:10, 99.66it/s] 
 39%|███▉      | 19456/50176 [03:12<05:13, 97.84it/s]
 39%|███▉      | 19712/50176 [03:15<05:10, 97.97it/s]
 40%|███▉      | 19968/50176 [03:18<05:08, 97.86it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.81it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.61it/s]
 41%|████▏     | 20736/50176 [03:25<04:38, 105.79it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.25it/s]
 42%|████▏     | 21248/50176 [03:30<04:43, 102.17it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.49it/s]
 43%|████▎     | 21760/50176 [03:35<04:36, 102.70it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.69it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 108.04it/s]
 45%|████▍     | 22528/50176 [03:41<04:16, 107.88it/s]
 45%|████▌     | 22784/50176 [03:44<04:25, 103.36it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.54it/s]
 46%|████▋     | 23296/50176 [03:49<04:27, 100.48it/s]
 47%|████▋     | 23552/50176 [03:52<04:31, 97.96it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.88it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.93it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.40it/s]
 49%|████▉     | 24576/50176 [04:02<04:14, 100.41it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.06it/s]
 50%|█████     | 25088/50176 [04:07<03:59, 104.54it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.72it/s]
 51%|█████     | 25600/50176 [04:12<03:54, 104.84it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.54it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.67it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.11it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.25it/s]
 54%|█████▎    | 26880/50176 [04:24<03:40, 105.62it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 105.13it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.31it/s]
 55%|█████▌    | 27648/50176 [04:30<03:22, 111.02it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.43it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.27it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.35it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 102.04it/s]
 58%|█████▊    | 28928/50176 [04:43<03:24, 103.71it/s]
 58%|█████▊    | 29184/50176 [04:46<03:21, 104.20it/s]
 59%|█████▊    | 29440/50176 [04:49<03:30, 98.39it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:30, 97.50it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.18it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.86it/s]
 61%|██████    | 30464/50176 [04:59<03:09, 104.10it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.27it/s]
 62%|██████▏   | 30976/50176 [05:04<03:06, 102.80it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.57it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.36it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.27it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 107.03it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 106.97it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.44it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.30it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.10it/s]
 66%|██████▋   | 33280/50176 [05:26<02:42, 103.80it/s]
 67%|██████▋   | 33536/50176 [05:28<02:41, 103.35it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.53it/s]
 68%|██████▊   | 34048/50176 [05:33<02:28, 108.41it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.11it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.00it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.41it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.89it/s]
 70%|███████   | 35328/50176 [05:50<03:01, 81.65it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.65it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.84it/s]
 72%|███████▏  | 36096/50176 [06:02<03:30, 66.87it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.68it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.88it/s]
 73%|███████▎  | 36864/50176 [06:14<03:16, 67.63it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.62it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.12it/s]
 75%|███████▌  | 37632/50176 [06:20<02:16, 91.64it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.19it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.85it/s]
 77%|███████▋  | 38400/50176 [06:35<03:19, 59.15it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 68.01it/s]
 78%|███████▊  | 38912/50176 [06:42<03:01, 62.22it/s]
 78%|███████▊  | 39168/50176 [06:45<02:38, 69.46it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.40it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.81it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.11it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.47it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.12it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.97it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.89it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.95it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.54it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.02it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.43it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.78it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.61it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.42it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.77it/s]
 86%|████████▌ | 43264/50176 [07:23<01:04, 107.81it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.24it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.43it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.50it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.67it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.36it/s]
 89%|████████▉ | 44800/50176 [07:38<00:52, 101.99it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.16it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 101.07it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 104.01it/s]
 91%|█████████▏| 45824/50176 [07:48<00:41, 104.14it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.53it/s]
 92%|█████████▏| 46336/50176 [07:53<00:37, 102.37it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.66it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.70it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.39it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.07it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.66it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.86it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.69it/s]
 96%|█████████▋| 48384/50176 [08:13<00:17, 103.34it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.83it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.71it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.90it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.20it/s]
 99%|█████████▉| 49664/50176 [08:26<00:05, 99.97it/s] 
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.71it/s]
100%|██████████| 50176/50176 [08:29<00:00, 125.27it/s]
100%|██████████| 50176/50176 [08:29<00:00, 98.47it/s] 
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5273	imagenet-zeroshot-val-top5: 0.8075
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1904268cfaba6b1f52204d4d397d5b6394fda120
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:903f91d7e64c801669f785900efcaa03eaed3f561177264f77e22f744c9cd83a
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8c3212b0170e544c8cbb546628cfa7e3afa75dca
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_11.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<14:16,  1.17it/s]
  1%|          | 6/1000 [00:00<02:02,  8.13it/s]
  1%|          | 12/1000 [00:01<01:00, 16.40it/s]
  2%|▏         | 18/1000 [00:01<00:40, 24.35it/s]
  2%|▏         | 24/1000 [00:01<00:30, 31.55it/s]
  3%|▎         | 30/1000 [00:01<00:25, 37.71it/s]
  4%|▎         | 36/1000 [00:01<00:22, 42.67it/s]
  4%|▍         | 42/1000 [00:01<00:20, 46.55it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.50it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.70it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.29it/s]
  7%|▋         | 66/1000 [00:02<00:17, 54.45it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.27it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.85it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.25it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.63it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.02it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.24it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.54it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.67it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.67it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.62it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.67it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.63it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.64it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.71it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.62it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.32it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.25it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.28it/s]
 18%|█▊        | 180/1000 [00:04<00:14, 57.26it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.24it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.22it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.20it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.01it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.92it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.03it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.03it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.23it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.21it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.13it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.17it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.28it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.19it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.29it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.13it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.26it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.42it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.67it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.65it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.64it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.79it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.70it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.53it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.61it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.73it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.82it/s]
 35%|███▌      | 354/1000 [00:07<00:11, 57.61it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.62it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.67it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.75it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.45it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.20it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.30it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.43it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.61it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.69it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.70it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.70it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.75it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.84it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.85it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.66it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.73it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.78it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.73it/s]
 47%|████▋     | 468/1000 [00:09<00:09, 57.66it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.81it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.67it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.66it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.66it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.56it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.62it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.44it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.61it/s]
 53%|█████▎    | 528/1000 [00:10<00:08, 57.70it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.57it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.71it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.75it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.65it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.68it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.75it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.65it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.56it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.68it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.69it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.60it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.69it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.62it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.63it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.75it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.76it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.85it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.68it/s]
 64%|██████▍   | 642/1000 [00:12<00:06, 57.62it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.76it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.74it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.79it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.83it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.74it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.77it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.77it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.82it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.72it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.79it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.78it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.76it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.71it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.67it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.72it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.71it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.80it/s]
 76%|███████▌  | 756/1000 [00:14<00:04, 57.89it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.73it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.82it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.77it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.80it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.83it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.64it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.63it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.69it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.66it/s]
 82%|████████▏ | 816/1000 [00:15<00:03, 57.66it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.68it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.79it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.92it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.95it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.87it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.93it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.88it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.85it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.79it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.85it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.83it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.71it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.79it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.80it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.79it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.83it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.82it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.62it/s]
 93%|█████████▎| 930/1000 [00:17<00:01, 57.75it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.84it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.84it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.77it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.82it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.81it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.88it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.96it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.94it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.94it/s]
 99%|█████████▉| 990/1000 [00:18<00:00, 57.87it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.06it/s]
100%|██████████| 1000/1000 [00:18<00:00, 54.85it/s]
+2022-12-21,09:37:11 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:17, 114.04it/s]
  1%|          | 512/50176 [00:04<07:03, 117.32it/s]
  2%|▏         | 768/50176 [00:06<07:22, 111.54it/s]
  2%|▏         | 1024/50176 [00:09<07:56, 103.23it/s]
  3%|▎         | 1280/50176 [00:12<08:07, 100.28it/s]
  3%|▎         | 1536/50176 [00:15<08:57, 90.57it/s] 
  4%|▎         | 1792/50176 [00:18<08:39, 93.22it/s]
  4%|▍         | 2048/50176 [00:20<08:31, 94.10it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.70it/s]
  5%|▌         | 2560/50176 [00:25<08:09, 97.37it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.24it/s]
  6%|▌         | 3072/50176 [00:31<08:16, 94.79it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.90it/s]
  7%|▋         | 3584/50176 [00:36<07:43, 100.63it/s]
  8%|▊         | 3840/50176 [00:39<07:58, 96.81it/s] 
  8%|▊         | 4096/50176 [00:42<08:12, 93.64it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.20it/s]
  9%|▉         | 4608/50176 [00:46<07:37, 99.70it/s]
 10%|▉         | 4864/50176 [00:49<07:25, 101.82it/s]
 10%|█         | 5120/50176 [00:51<07:21, 102.16it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.50it/s]
 11%|█         | 5632/50176 [00:56<07:16, 102.03it/s]
 12%|█▏        | 5888/50176 [00:59<07:14, 101.96it/s]
 12%|█▏        | 6144/50176 [01:02<07:44, 94.87it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.91it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.31it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.57it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.49it/s]
 15%|█▍        | 7424/50176 [01:15<07:08, 99.86it/s]
 15%|█▌        | 7680/50176 [01:17<07:05, 99.87it/s]
 16%|█▌        | 7936/50176 [01:20<06:48, 103.41it/s]
 16%|█▋        | 8192/50176 [01:22<06:48, 102.83it/s]
 17%|█▋        | 8448/50176 [01:25<06:49, 101.99it/s]
 17%|█▋        | 8704/50176 [01:27<06:48, 101.40it/s]
 18%|█▊        | 8960/50176 [01:30<06:48, 100.94it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.71it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.95it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.20it/s]
 20%|█▉        | 9984/50176 [01:40<06:46, 98.88it/s] 
 20%|██        | 10240/50176 [01:42<06:31, 101.93it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.13it/s]
 21%|██▏       | 10752/50176 [01:47<06:28, 101.42it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.12it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.79it/s]
 23%|██▎       | 11520/50176 [01:55<06:08, 104.93it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.57it/s]
 24%|██▍       | 12032/50176 [01:59<06:01, 105.45it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.54it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.87it/s]
 26%|██▌       | 12800/50176 [02:07<05:50, 106.49it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.23it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.66it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.07it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 100.10it/s]
 28%|██▊       | 14080/50176 [02:19<05:59, 100.51it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.43it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.50it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.84it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.38it/s]
 31%|███       | 15360/50176 [02:32<05:49, 99.67it/s]
 31%|███       | 15616/50176 [02:35<05:36, 102.63it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.67it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.82it/s]
 33%|███▎      | 16384/50176 [02:42<05:26, 103.38it/s]
 33%|███▎      | 16640/50176 [02:45<05:41, 98.33it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.59it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 103.91it/s]
 35%|███▍      | 17408/50176 [02:52<05:17, 103.20it/s]
 35%|███▌      | 17664/50176 [02:54<05:08, 105.37it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.14it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.37it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.61it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.40it/s]
 38%|███▊      | 18944/50176 [03:07<05:03, 102.98it/s]
 38%|███▊      | 19200/50176 [03:09<05:10, 99.67it/s] 
 39%|███▉      | 19456/50176 [03:12<05:13, 97.87it/s]
 39%|███▉      | 19712/50176 [03:15<05:11, 97.93it/s]
 40%|███▉      | 19968/50176 [03:17<05:08, 97.94it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.85it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.64it/s]
 41%|████▏     | 20736/50176 [03:24<04:38, 105.80it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.24it/s]
 42%|████▏     | 21248/50176 [03:29<04:43, 102.17it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.42it/s]
 43%|████▎     | 21760/50176 [03:34<04:36, 102.72it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.70it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 107.83it/s]
 45%|████▍     | 22528/50176 [03:41<04:15, 108.04it/s]
 45%|████▌     | 22784/50176 [03:44<04:24, 103.37it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.55it/s]
 46%|████▋     | 23296/50176 [03:49<04:27, 100.41it/s]
 47%|████▋     | 23552/50176 [03:52<04:32, 97.85it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.78it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.99it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.49it/s]
 49%|████▉     | 24576/50176 [04:02<04:15, 100.34it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.10it/s]
 50%|█████     | 25088/50176 [04:07<04:00, 104.52it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.71it/s]
 51%|█████     | 25600/50176 [04:11<03:54, 104.81it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.60it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.70it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.12it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.31it/s]
 54%|█████▎    | 26880/50176 [04:23<03:40, 105.77it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 104.99it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.40it/s]
 55%|█████▌    | 27648/50176 [04:30<03:22, 111.15it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.31it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.26it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.26it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 101.92it/s]
 58%|█████▊    | 28928/50176 [04:43<03:24, 103.81it/s]
 58%|█████▊    | 29184/50176 [04:45<03:21, 104.17it/s]
 59%|█████▊    | 29440/50176 [04:48<03:30, 98.48it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:29, 97.57it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.08it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.87it/s]
 61%|██████    | 30464/50176 [04:58<03:09, 104.06it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.35it/s]
 62%|██████▏   | 30976/50176 [05:03<03:06, 102.70it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.53it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.47it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.28it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 106.93it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 107.01it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.49it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.31it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.13it/s]
 66%|██████▋   | 33280/50176 [05:25<02:42, 103.71it/s]
 67%|██████▋   | 33536/50176 [05:28<02:40, 103.43it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.53it/s]
 68%|██████▊   | 34048/50176 [05:32<02:28, 108.51it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.09it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.09it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.40it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.89it/s]
 70%|███████   | 35328/50176 [05:49<03:01, 81.64it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.66it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.83it/s]
 72%|███████▏  | 36096/50176 [06:01<03:30, 66.87it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.71it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.90it/s]
 73%|███████▎  | 36864/50176 [06:13<03:16, 67.63it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.62it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.12it/s]
 75%|███████▌  | 37632/50176 [06:20<02:16, 91.60it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.25it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.88it/s]
 77%|███████▋  | 38400/50176 [06:34<03:19, 59.10it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 67.97it/s]
 78%|███████▊  | 38912/50176 [06:42<03:00, 62.24it/s]
 78%|███████▊  | 39168/50176 [06:44<02:38, 69.46it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.41it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.87it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.13it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.52it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.07it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.93it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.74it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.99it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.62it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.15it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.66it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.66it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.57it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.84it/s]
 86%|████████▌ | 43264/50176 [07:22<01:04, 107.81it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.14it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.38it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.47it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.56it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.36it/s]
 89%|████████▉ | 44800/50176 [07:37<00:52, 102.08it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.24it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 100.89it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 104.01it/s]
 91%|█████████▏| 45824/50176 [07:47<00:41, 104.14it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.63it/s]
 92%|█████████▏| 46336/50176 [07:52<00:37, 102.25it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.74it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.86it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.48it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.08it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.58it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.89it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.59it/s]
 96%|█████████▋| 48384/50176 [08:12<00:17, 103.22it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.91it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.51it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.92it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.32it/s]
 99%|█████████▉| 49664/50176 [08:25<00:05, 100.06it/s]
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.68it/s] 
100%|██████████| 50176/50176 [08:29<00:00, 125.14it/s]
100%|██████████| 50176/50176 [08:29<00:00, 98.51it/s] 
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5318	imagenet-zeroshot-val-top5: 0.8127
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0d23c4f0a1dac9b50718d0bbb6f9567eef22b387
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6b50f9f65b8eaffec9d605452b04e8553836c84d313abf3705a7931ed08b552
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ba6a890fd5ee8bd81af9a743d453bb4dea978b86
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_12.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:26,  1.24it/s]
  1%|          | 6/1000 [00:00<01:56,  8.56it/s]
  1%|          | 12/1000 [00:01<00:57, 17.16it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.44it/s]
  2%|▏         | 24/1000 [00:01<00:29, 32.66it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.77it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.60it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.36it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.20it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.27it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.81it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.92it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.65it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.19it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.77it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.04it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.46it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.59it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.88it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.01it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.02it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.91it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.97it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.03it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.95it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.81it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.81it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.76it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.67it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.64it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.43it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.35it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.46it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.42it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.58it/s]
 23%|██▎       | 234/1000 [00:04<00:14, 51.94it/s]
 24%|██▍       | 240/1000 [00:05<00:14, 53.44it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 54.63it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 55.55it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 56.26it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 56.75it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 56.89it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.13it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.46it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.67it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.94it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.97it/s]
 31%|███       | 306/1000 [00:06<00:11, 57.93it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.11it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.12it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.06it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.91it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.98it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.08it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.18it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.94it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.85it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.93it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.04it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.96it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.66it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.70it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.98it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.05it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 58.05it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.05it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.07it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.17it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.96it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.02it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.98it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.13it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.06it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.04it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.04it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.94it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.00it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.92it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.13it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.01it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.13it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.06it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.07it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.11it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.94it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 58.07it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.99it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.09it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.02it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.13it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.14it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.22it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.05it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.99it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.14it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.10it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.12it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.15it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.17it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.21it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.09it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.18it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.16it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.12it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.13it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.10it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.18it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.27it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 58.11it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.18it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.13it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.17it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.20it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.02it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.07it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.01it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.02it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.83it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.97it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.16it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.24it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.19it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.20it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.21it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.19it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.19it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.07it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.17it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.15it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.12it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.14it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.22it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.25it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.26it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.29it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.36it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.30it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 58.10it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.26it/s]
+2022-12-21,09:37:11 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:42, 108.02it/s]
  1%|          | 512/50176 [00:04<07:12, 114.70it/s]
  2%|▏         | 768/50176 [00:06<07:28, 110.23it/s]
  2%|▏         | 1024/50176 [00:09<08:00, 102.38it/s]
  3%|▎         | 1280/50176 [00:12<08:10, 99.76it/s] 
  3%|▎         | 1536/50176 [00:15<08:59, 90.09it/s]
  4%|▎         | 1792/50176 [00:18<08:39, 93.12it/s]
  4%|▍         | 2048/50176 [00:21<08:32, 93.87it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.67it/s]
  5%|▌         | 2560/50176 [00:26<08:08, 97.49it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.24it/s]
  6%|▌         | 3072/50176 [00:31<08:16, 94.80it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.83it/s]
  7%|▋         | 3584/50176 [00:36<07:43, 100.56it/s]
  8%|▊         | 3840/50176 [00:39<07:58, 96.75it/s] 
  8%|▊         | 4096/50176 [00:42<08:11, 93.73it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.11it/s]
  9%|▉         | 4608/50176 [00:47<07:36, 99.82it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.96it/s]
 10%|█         | 5120/50176 [00:51<07:20, 102.25it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.29it/s]
 11%|█         | 5632/50176 [00:57<07:16, 102.12it/s]
 12%|█▏        | 5888/50176 [00:59<07:15, 101.79it/s]
 12%|█▏        | 6144/50176 [01:02<07:43, 94.98it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.96it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.41it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.69it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.54it/s]
 15%|█▍        | 7424/50176 [01:15<07:09, 99.58it/s]
 15%|█▌        | 7680/50176 [01:18<07:05, 99.81it/s]
 16%|█▌        | 7936/50176 [01:20<06:49, 103.11it/s]
 16%|█▋        | 8192/50176 [01:22<06:47, 102.93it/s]
 17%|█▋        | 8448/50176 [01:25<06:48, 102.08it/s]
 17%|█▋        | 8704/50176 [01:27<06:49, 101.38it/s]
 18%|█▊        | 8960/50176 [01:30<06:47, 101.13it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.75it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.95it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.18it/s]
 20%|█▉        | 9984/50176 [01:40<06:47, 98.75it/s] 
 20%|██        | 10240/50176 [01:42<06:30, 102.16it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.06it/s]
 21%|██▏       | 10752/50176 [01:47<06:28, 101.39it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.19it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.67it/s]
 23%|██▎       | 11520/50176 [01:55<06:09, 104.71it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.63it/s]
 24%|██▍       | 12032/50176 [02:00<06:02, 105.37it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.72it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.79it/s]
 26%|██▌       | 12800/50176 [02:07<05:50, 106.56it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.16it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.65it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.07it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 99.92it/s] 
 28%|██▊       | 14080/50176 [02:19<05:58, 100.69it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.47it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.43it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.88it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.42it/s]
 31%|███       | 15360/50176 [02:32<05:48, 99.78it/s]
 31%|███       | 15616/50176 [02:35<05:37, 102.49it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.66it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.80it/s]
 33%|███▎      | 16384/50176 [02:42<05:27, 103.32it/s]
 33%|███▎      | 16640/50176 [02:45<05:40, 98.38it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.62it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 104.02it/s]
 35%|███▍      | 17408/50176 [02:52<05:18, 102.95it/s]
 35%|███▌      | 17664/50176 [02:55<05:08, 105.42it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.07it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.41it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.68it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.31it/s]
 38%|███▊      | 18944/50176 [03:07<05:02, 103.17it/s]
 38%|███▊      | 19200/50176 [03:10<05:10, 99.70it/s] 
 39%|███▉      | 19456/50176 [03:12<05:14, 97.81it/s]
 39%|███▉      | 19712/50176 [03:15<05:10, 98.02it/s]
 40%|███▉      | 19968/50176 [03:18<05:08, 97.90it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.86it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.51it/s]
 41%|████▏     | 20736/50176 [03:24<04:38, 105.71it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.35it/s]
 42%|████▏     | 21248/50176 [03:30<04:43, 102.11it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.50it/s]
 43%|████▎     | 21760/50176 [03:35<04:36, 102.79it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.59it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 107.90it/s]
 45%|████▍     | 22528/50176 [03:41<04:16, 107.84it/s]
 45%|████▌     | 22784/50176 [03:44<04:24, 103.38it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.58it/s]
 46%|████▋     | 23296/50176 [03:49<04:27, 100.55it/s]
 47%|████▋     | 23552/50176 [03:52<04:32, 97.87it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.78it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.91it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.55it/s]
 49%|████▉     | 24576/50176 [04:02<04:15, 100.33it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.17it/s]
 50%|█████     | 25088/50176 [04:07<03:59, 104.58it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.86it/s]
 51%|█████     | 25600/50176 [04:12<03:54, 104.75it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.64it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.67it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.03it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.36it/s]
 54%|█████▎    | 26880/50176 [04:24<03:40, 105.65it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 104.97it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.44it/s]
 55%|█████▌    | 27648/50176 [04:30<03:23, 110.88it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.48it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.27it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.27it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 101.94it/s]
 58%|█████▊    | 28928/50176 [04:43<03:25, 103.62it/s]
 58%|█████▊    | 29184/50176 [04:46<03:21, 104.01it/s]
 59%|█████▊    | 29440/50176 [04:48<03:30, 98.57it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:30, 97.50it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.14it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.92it/s]
 61%|██████    | 30464/50176 [04:58<03:09, 104.10it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.29it/s]
 62%|██████▏   | 30976/50176 [05:04<03:06, 102.74it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.56it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.56it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.31it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 106.95it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 106.93it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.50it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.31it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.19it/s]
 66%|██████▋   | 33280/50176 [05:26<02:42, 103.71it/s]
 67%|██████▋   | 33536/50176 [05:28<02:40, 103.45it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.63it/s]
 68%|██████▊   | 34048/50176 [05:33<02:28, 108.49it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.13it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.01it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.40it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.87it/s]
 70%|███████   | 35328/50176 [05:50<03:01, 81.63it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.68it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.84it/s]
 72%|███████▏  | 36096/50176 [06:02<03:30, 66.88it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.68it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.86it/s]
 73%|███████▎  | 36864/50176 [06:14<03:16, 67.60it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.67it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.07it/s]
 75%|███████▌  | 37632/50176 [06:20<02:17, 91.54it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.25it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.88it/s]
 77%|███████▋  | 38400/50176 [06:35<03:19, 59.09it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 67.99it/s]
 78%|███████▊  | 38912/50176 [06:42<03:00, 62.26it/s]
 78%|███████▊  | 39168/50176 [06:45<02:38, 69.41it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.44it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.91it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.16it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.52it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.08it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.94it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.75it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.98it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.62it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.00it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.61it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.63it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.48it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.87it/s]
 86%|████████▌ | 43264/50176 [07:22<01:04, 107.89it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.20it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.41it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.53it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.57it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.38it/s]
 89%|████████▉ | 44800/50176 [07:38<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.25it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 100.96it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 104.00it/s]
 91%|█████████▏| 45824/50176 [07:47<00:41, 104.12it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.59it/s]
 92%|█████████▏| 46336/50176 [07:53<00:37, 102.43it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.58it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.75it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.42it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.09it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.58it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.86it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.51it/s]
 96%|█████████▋| 48384/50176 [08:13<00:17, 103.31it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.93it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.61it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.98it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.26it/s]
 99%|█████████▉| 49664/50176 [08:25<00:05, 100.04it/s]
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.65it/s] 
100%|██████████| 50176/50176 [08:29<00:00, 125.14it/s]
100%|██████████| 50176/50176 [08:29<00:00, 98.48it/s] 
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5369	imagenet-zeroshot-val-top5: 0.8153
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt
new file mode 100644
index 0000000000000000000000000000000000000000..867d5954404732653020f67ebb0bab8e93af8d57
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d197318f457dba0d5b00836bbe0f000d94e2348ee341518de4f0fc2f7883029
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9e3a6f58925b7e340ccb50f0d8ec2f5e232bae43
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_13.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:33,  1.58it/s]
  1%|          | 6/1000 [00:00<01:35, 10.46it/s]
  1%|          | 12/1000 [00:00<00:48, 20.31it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.02it/s]
  2%|▏         | 24/1000 [00:01<00:27, 36.07it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.73it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.84it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.07it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.47it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.17it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.39it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.28it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.92it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.40it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.81it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.10it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.47it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.64it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.80it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.78it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.77it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.78it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.85it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.74it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.70it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.64it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.59it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.51it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.34it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.34it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.52it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.50it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.42it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.47it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.52it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.61it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.68it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.48it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.51it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.70it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.82it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.04it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.02it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.00it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.09it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.03it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.85it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.92it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.93it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.02it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.82it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.84it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.94it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.02it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.93it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.61it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.69it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.78it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.91it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.99it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.01it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.03it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.14it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.13it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.90it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.98it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.97it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.10it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.98it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.95it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.96it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.93it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.86it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.98it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.07it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.02it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.15it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.11it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.00it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.99it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.04it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.90it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.85it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.00it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.00it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.91it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.03it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.96it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.96it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.04it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.14it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.96it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.88it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.03it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.00it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.09it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.12it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.07it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.09it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.10it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.14it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.04it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.15it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.13it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.11it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.13it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.12it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.19it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.27it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.09it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.18it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.12it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.15it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.18it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.97it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.95it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.97it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.71it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.76it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.84it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 57.97it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.16it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.22it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.15it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.21it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.16it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.10it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.13it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.00it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.12it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.07it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.09it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.18it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.18it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.21it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.21it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.25it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.27it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.28it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.98it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.01it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.86it/s]
+2022-12-21,09:46:23 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<06:35, 126.10it/s]
  1%|          | 512/50176 [00:04<06:37, 124.94it/s]
  2%|▏         | 768/50176 [00:06<06:46, 121.63it/s]
  2%|▏         | 1024/50176 [00:08<06:47, 120.50it/s]
  3%|▎         | 1280/50176 [00:10<07:06, 114.59it/s]
  3%|▎         | 1536/50176 [00:13<07:36, 106.55it/s]
  4%|▎         | 1792/50176 [00:15<07:25, 108.67it/s]
  4%|▍         | 2048/50176 [00:18<07:16, 110.37it/s]
  5%|▍         | 2304/50176 [00:20<07:04, 112.78it/s]
  5%|▌         | 2560/50176 [00:22<07:01, 112.94it/s]
  6%|▌         | 2816/50176 [00:24<07:10, 110.04it/s]
  6%|▌         | 3072/50176 [00:27<07:12, 108.79it/s]
  7%|▋         | 3328/50176 [00:29<07:08, 109.37it/s]
  7%|▋         | 3584/50176 [00:31<06:54, 112.27it/s]
  8%|▊         | 3840/50176 [00:34<07:18, 105.68it/s]
  8%|▊         | 4096/50176 [00:37<07:36, 100.84it/s]
  9%|▊         | 4352/50176 [00:39<07:28, 102.07it/s]
  9%|▉         | 4608/50176 [00:42<07:14, 104.82it/s]
 10%|▉         | 4864/50176 [00:44<07:06, 106.16it/s]
 10%|█         | 5120/50176 [00:46<07:04, 106.18it/s]
 11%|█         | 5376/50176 [00:49<07:09, 104.33it/s]
 11%|█         | 5632/50176 [00:51<07:10, 103.37it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.62it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.88it/s] 
 13%|█▎        | 6400/50176 [01:00<07:31, 97.00it/s]
 13%|█▎        | 6656/50176 [01:02<07:21, 98.56it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.67it/s]
 14%|█▍        | 7168/50176 [01:07<07:14, 98.98it/s]
 15%|█▍        | 7424/50176 [01:10<07:01, 101.32it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.04it/s]
 16%|█▌        | 7936/50176 [01:14<06:47, 103.63it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.05it/s]
 17%|█▋        | 8448/50176 [01:20<06:51, 101.46it/s]
 17%|█▋        | 8704/50176 [01:22<06:47, 101.68it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.82it/s]
 18%|█▊        | 9216/50176 [01:27<06:43, 101.51it/s]
 19%|█▉        | 9472/50176 [01:30<06:41, 101.39it/s]
 19%|█▉        | 9728/50176 [01:32<06:43, 100.12it/s]
 20%|█▉        | 9984/50176 [01:35<06:43, 99.72it/s] 
 20%|██        | 10240/50176 [01:37<06:28, 102.75it/s]
 21%|██        | 10496/50176 [01:40<06:21, 104.07it/s]
 21%|██▏       | 10752/50176 [01:42<06:28, 101.51it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.79it/s]
 22%|██▏       | 11264/50176 [01:47<06:16, 103.22it/s]
 23%|██▎       | 11520/50176 [01:50<06:09, 104.51it/s]
 23%|██▎       | 11776/50176 [01:52<06:06, 104.85it/s]
 24%|██▍       | 12032/50176 [01:54<06:05, 104.27it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.93it/s]
 25%|██▌       | 12544/50176 [01:59<05:49, 107.54it/s]
 26%|██▌       | 12800/50176 [02:02<05:51, 106.22it/s]
 26%|██▌       | 13056/50176 [02:04<05:47, 106.69it/s]
 27%|██▋       | 13312/50176 [02:06<05:48, 105.85it/s]
 27%|██▋       | 13568/50176 [02:09<06:05, 100.23it/s]
 28%|██▊       | 13824/50176 [02:12<06:08, 98.77it/s] 
 28%|██▊       | 14080/50176 [02:14<06:00, 100.11it/s]
 29%|██▊       | 14336/50176 [02:17<05:57, 100.20it/s]
 29%|██▉       | 14592/50176 [02:19<05:50, 101.59it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.66it/s] 
 30%|███       | 15104/50176 [02:25<05:55, 98.70it/s]
 31%|███       | 15360/50176 [02:27<05:47, 100.13it/s]
 31%|███       | 15616/50176 [02:30<05:37, 102.28it/s]
 32%|███▏      | 15872/50176 [02:32<05:39, 101.18it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.89it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.62it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.06it/s] 
 34%|███▎      | 16896/50176 [02:42<05:30, 100.59it/s]
 34%|███▍      | 17152/50176 [02:45<05:23, 102.23it/s]
 35%|███▍      | 17408/50176 [02:47<05:17, 103.13it/s]
 35%|███▌      | 17664/50176 [02:49<05:03, 107.05it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 108.98it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.76it/s]
 37%|███▋      | 18432/50176 [02:56<04:54, 107.65it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.57it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.67it/s]
 38%|███▊      | 19200/50176 [03:04<05:09, 99.93it/s] 
 39%|███▉      | 19456/50176 [03:07<05:12, 98.24it/s]
 39%|███▉      | 19712/50176 [03:10<05:13, 97.23it/s]
 40%|███▉      | 19968/50176 [03:12<05:11, 96.95it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.37it/s]
 41%|████      | 20480/50176 [03:17<04:48, 103.11it/s]
 41%|████▏     | 20736/50176 [03:19<04:42, 104.37it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.82it/s]
 42%|████▏     | 21248/50176 [03:25<04:41, 102.60it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 103.09it/s]
 43%|████▎     | 21760/50176 [03:29<04:30, 105.19it/s]
 44%|████▍     | 22016/50176 [03:32<04:20, 108.30it/s]
 44%|████▍     | 22272/50176 [03:34<04:17, 108.19it/s]
 45%|████▍     | 22528/50176 [03:36<04:19, 106.40it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.29it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.27it/s] 
 46%|████▋     | 23296/50176 [03:45<04:32, 98.47it/s]
 47%|████▋     | 23552/50176 [03:50<05:45, 77.16it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.02it/s]
 48%|████▊     | 24064/50176 [03:55<04:56, 88.13it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.88it/s]
 49%|████▉     | 24576/50176 [04:00<04:32, 93.92it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.92it/s]
 50%|█████     | 25088/50176 [04:04<04:06, 101.81it/s]
 51%|█████     | 25344/50176 [04:07<03:56, 104.96it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.34it/s]
 52%|█████▏    | 25856/50176 [04:11<03:46, 107.17it/s]
 52%|█████▏    | 26112/50176 [04:14<03:43, 107.51it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.18it/s]
 53%|█████▎    | 26624/50176 [04:18<03:38, 107.85it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.27it/s]
 54%|█████▍    | 27136/50176 [04:23<03:41, 104.05it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.75it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.57it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.64it/s]
 56%|█████▌    | 28160/50176 [04:35<04:00, 91.56it/s]
 57%|█████▋    | 28416/50176 [04:38<03:52, 93.63it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 93.94it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.45it/s]
 58%|█████▊    | 29184/50176 [04:45<03:30, 99.87it/s]
 59%|█████▊    | 29440/50176 [04:48<03:36, 95.62it/s]
 59%|█████▉    | 29696/50176 [04:51<03:34, 95.69it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.61it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.79it/s]
 61%|██████    | 30464/50176 [04:58<03:10, 103.24it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.61it/s]
 62%|██████▏   | 30976/50176 [05:03<03:02, 105.01it/s]
 62%|██████▏   | 31232/50176 [05:06<02:59, 105.45it/s]
 63%|██████▎   | 31488/50176 [05:08<02:52, 108.61it/s]
 63%|██████▎   | 31744/50176 [05:10<02:50, 107.83it/s]
 64%|██████▍   | 32000/50176 [05:13<02:48, 107.94it/s]
 64%|██████▍   | 32256/50176 [05:15<02:43, 109.84it/s]
 65%|██████▍   | 32512/50176 [05:17<02:47, 105.42it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.20it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.51it/s]
 66%|██████▋   | 33280/50176 [05:24<02:36, 107.95it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.61it/s]
 67%|██████▋   | 33792/50176 [05:29<02:32, 107.72it/s]
 68%|██████▊   | 34048/50176 [05:31<02:24, 111.47it/s]
 68%|██████▊   | 34304/50176 [05:33<02:19, 113.92it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.33it/s]
 69%|██████▉   | 34816/50176 [05:39<02:23, 107.21it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.06it/s]
 70%|███████   | 35328/50176 [05:43<02:17, 108.28it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 107.92it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.76it/s]
 72%|███████▏  | 36096/50176 [05:51<02:11, 107.40it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.45it/s]
 73%|███████▎  | 36608/50176 [05:56<02:09, 104.57it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.98it/s]
 74%|███████▍  | 37120/50176 [06:01<02:05, 104.05it/s]
 74%|███████▍  | 37376/50176 [06:03<01:56, 109.52it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.65it/s]
 76%|███████▌  | 37888/50176 [06:07<01:53, 108.00it/s]
 76%|███████▌  | 38144/50176 [06:10<01:51, 108.33it/s]
 77%|███████▋  | 38400/50176 [06:12<01:49, 107.33it/s]
 77%|███████▋  | 38656/50176 [06:15<01:47, 106.68it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.49it/s]
 78%|███████▊  | 39168/50176 [06:19<01:44, 105.63it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.14it/s]
 79%|███████▉  | 39680/50176 [06:24<01:38, 106.64it/s]
 80%|███████▉  | 39936/50176 [06:27<01:34, 107.86it/s]
 80%|████████  | 40192/50176 [06:29<01:34, 106.19it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.85it/s]
 81%|████████  | 40704/50176 [06:34<01:33, 101.16it/s]
 82%|████████▏ | 40960/50176 [06:37<01:27, 105.51it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.40it/s]
 83%|████████▎ | 41472/50176 [06:41<01:22, 105.55it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.36it/s]
 84%|████████▎ | 41984/50176 [06:46<01:18, 104.80it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.98it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.39it/s]
 85%|████████▌ | 42752/50176 [06:53<01:08, 108.28it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.81it/s]
 86%|████████▌ | 43264/50176 [06:58<01:05, 105.53it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.29it/s]
 87%|████████▋ | 43776/50176 [07:03<01:02, 102.79it/s]
 88%|████████▊ | 44032/50176 [07:06<00:59, 104.04it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.92it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 101.06it/s]
 89%|████████▉ | 44800/50176 [07:13<00:52, 102.46it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.09it/s]
 90%|█████████ | 45312/50176 [07:18<00:47, 102.51it/s]
 91%|█████████ | 45568/50176 [07:21<00:43, 106.03it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.93it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.07it/s]
 92%|█████████▏| 46336/50176 [07:28<00:37, 101.22it/s]
 93%|█████████▎| 46592/50176 [07:31<00:34, 105.22it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.18it/s]
 94%|█████████▍| 47104/50176 [07:36<00:30, 101.93it/s]
 94%|█████████▍| 47360/50176 [07:38<00:28, 100.20it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.33it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.63it/s]
 96%|█████████▌| 48128/50176 [07:45<00:19, 104.59it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.18it/s]
 97%|█████████▋| 48640/50176 [07:50<00:14, 104.61it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.50it/s]
 98%|█████████▊| 49152/50176 [07:55<00:09, 103.33it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.29it/s]
 99%|█████████▉| 49664/50176 [08:00<00:05, 102.33it/s]
 99%|█████████▉| 49920/50176 [08:04<00:02, 95.33it/s] 
100%|██████████| 50176/50176 [08:04<00:00, 124.49it/s]
100%|██████████| 50176/50176 [08:04<00:00, 103.52it/s]
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5455	imagenet-zeroshot-val-top5: 0.8192
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e01e7df4f5ab76833dfaf171e2b8622d0df80c7c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:624ac22ec6db08f29577e1c9b768e333b2144e41552220e27e319553c3ee856a
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fc62575b3d25e667d89ebec05656da628e47a8b7
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_14.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:59,  1.51it/s]
  1%|          | 6/1000 [00:00<01:38, 10.13it/s]
  1%|          | 12/1000 [00:00<00:50, 19.61it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.22it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.35it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.08it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.43it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.70it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.13it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.87it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.12it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.07it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.69it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.12it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.60it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.88it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.23it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.38it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.36it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.40it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.42it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.39it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.30it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.17it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.34it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.42it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.41it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.14it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.16it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.11it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.15it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.16it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.11it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.11it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.91it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.85it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.00it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.95it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.16it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.13it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.03it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.13it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.22it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.34it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.40it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.22it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.25it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.42it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.73it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.65it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.63it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.78it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.74it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.58it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.65it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.74it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.84it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.64it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.61it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.67it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.77it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.64it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.33it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.30it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.42it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.55it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.67it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.67it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.70it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.73it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.87it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.87it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.67it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.70it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.75it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.72it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.69it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.80it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.70it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.67it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.66it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.49it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.57it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.47it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.56it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.63it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.74it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.67it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.71it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.75it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.66it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.67it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.73it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.59it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.54it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.69it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.68it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.61it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.73it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.64it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.65it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.76it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.76it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.82it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.48it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.47it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.66it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.64it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.74it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.78it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.71it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.75it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.77it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.80it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.69it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.77it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.76it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.72it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.76it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.72it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.76it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.75it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.85it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.92it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.74it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.84it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.79it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.82it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.84it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.62it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.57it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.65it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.63it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.42it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.51it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.66it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.83it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.90it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.83it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.89it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.84it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.88it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.82it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.87it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 57.85it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.68it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.77it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.76it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.72it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.76it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.74it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.74it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.83it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.86it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.86it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.86it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.91it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.83it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.91it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.70it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.74it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.52it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.46it/s]
+2022-12-21,09:46:23 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:09, 134.94it/s]
  1%|          | 512/50176 [00:03<06:26, 128.53it/s]
  2%|▏         | 768/50176 [00:06<06:39, 123.59it/s]
  2%|▏         | 1024/50176 [00:08<06:45, 121.34it/s]
  3%|▎         | 1280/50176 [00:10<07:04, 115.16it/s]
  3%|▎         | 1536/50176 [00:13<07:35, 106.67it/s]
  4%|▎         | 1792/50176 [00:15<07:24, 108.76it/s]
  4%|▍         | 2048/50176 [00:17<07:15, 110.46it/s]
  5%|▍         | 2304/50176 [00:20<07:04, 112.67it/s]
  5%|▌         | 2560/50176 [00:22<07:01, 113.05it/s]
  6%|▌         | 2816/50176 [00:24<07:08, 110.48it/s]
  6%|▌         | 3072/50176 [00:27<07:12, 108.87it/s]
  7%|▋         | 3328/50176 [00:29<07:08, 109.30it/s]
  7%|▋         | 3584/50176 [00:31<06:54, 112.39it/s]
  8%|▊         | 3840/50176 [00:34<07:19, 105.50it/s]
  8%|▊         | 4096/50176 [00:37<07:37, 100.78it/s]
  9%|▊         | 4352/50176 [00:39<07:29, 101.93it/s]
  9%|▉         | 4608/50176 [00:41<07:13, 105.04it/s]
 10%|▉         | 4864/50176 [00:44<07:07, 106.05it/s]
 10%|█         | 5120/50176 [00:46<07:04, 106.17it/s]
 11%|█         | 5376/50176 [00:49<07:08, 104.62it/s]
 11%|█         | 5632/50176 [00:51<07:11, 103.30it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.67it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.93it/s] 
 13%|█▎        | 6400/50176 [00:59<07:31, 97.05it/s]
 13%|█▎        | 6656/50176 [01:02<07:22, 98.46it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.67it/s]
 14%|█▍        | 7168/50176 [01:07<07:14, 99.02it/s]
 15%|█▍        | 7424/50176 [01:09<07:02, 101.29it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.10it/s]
 16%|█▌        | 7936/50176 [01:14<06:47, 103.60it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.09it/s]
 17%|█▋        | 8448/50176 [01:19<06:51, 101.50it/s]
 17%|█▋        | 8704/50176 [01:22<06:48, 101.48it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.79it/s]
 18%|█▊        | 9216/50176 [01:27<06:44, 101.37it/s]
 19%|█▉        | 9472/50176 [01:30<06:41, 101.44it/s]
 19%|█▉        | 9728/50176 [01:32<06:43, 100.15it/s]
 20%|█▉        | 9984/50176 [01:35<06:42, 99.87it/s] 
 20%|██        | 10240/50176 [01:37<06:29, 102.62it/s]
 21%|██        | 10496/50176 [01:39<06:21, 104.06it/s]
 21%|██▏       | 10752/50176 [01:42<06:29, 101.31it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.85it/s]
 22%|██▏       | 11264/50176 [01:47<06:16, 103.36it/s]
 23%|██▎       | 11520/50176 [01:49<06:10, 104.43it/s]
 23%|██▎       | 11776/50176 [01:52<06:06, 104.89it/s]
 24%|██▍       | 12032/50176 [01:54<06:05, 104.41it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.88it/s]
 25%|██▌       | 12544/50176 [01:59<05:49, 107.56it/s]
 26%|██▌       | 12800/50176 [02:01<05:51, 106.31it/s]
 26%|██▌       | 13056/50176 [02:04<05:47, 106.71it/s]
 27%|██▋       | 13312/50176 [02:06<05:47, 105.95it/s]
 27%|██▋       | 13568/50176 [02:09<06:05, 100.11it/s]
 28%|██▊       | 13824/50176 [02:12<06:08, 98.63it/s] 
 28%|██▊       | 14080/50176 [02:14<06:00, 100.18it/s]
 29%|██▊       | 14336/50176 [02:17<05:57, 100.14it/s]
 29%|██▉       | 14592/50176 [02:19<05:50, 101.48it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.77it/s] 
 30%|███       | 15104/50176 [02:25<05:54, 98.80it/s]
 31%|███       | 15360/50176 [02:27<05:47, 100.11it/s]
 31%|███       | 15616/50176 [02:29<05:37, 102.36it/s]
 32%|███▏      | 15872/50176 [02:32<05:38, 101.20it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.87it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.52it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.05it/s] 
 34%|███▎      | 16896/50176 [02:42<05:31, 100.45it/s]
 34%|███▍      | 17152/50176 [02:45<05:22, 102.29it/s]
 35%|███▍      | 17408/50176 [02:47<05:17, 103.10it/s]
 35%|███▌      | 17664/50176 [02:49<05:03, 107.07it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 108.98it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.83it/s]
 37%|███▋      | 18432/50176 [02:56<04:54, 107.67it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.42it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.70it/s]
 38%|███▊      | 19200/50176 [03:04<05:09, 100.02it/s]
 39%|███▉      | 19456/50176 [03:07<05:12, 98.35it/s] 
 39%|███▉      | 19712/50176 [03:10<05:13, 97.13it/s]
 40%|███▉      | 19968/50176 [03:12<05:11, 96.85it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.39it/s]
 41%|████      | 20480/50176 [03:17<04:48, 103.02it/s]
 41%|████▏     | 20736/50176 [03:19<04:42, 104.36it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.83it/s]
 42%|████▏     | 21248/50176 [03:24<04:41, 102.60it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 103.03it/s]
 43%|████▎     | 21760/50176 [03:29<04:29, 105.42it/s]
 44%|████▍     | 22016/50176 [03:31<04:19, 108.40it/s]
 44%|████▍     | 22272/50176 [03:34<04:18, 108.15it/s]
 45%|████▍     | 22528/50176 [03:36<04:19, 106.52it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.33it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.16it/s] 
 46%|████▋     | 23296/50176 [03:44<04:33, 98.35it/s]
 47%|████▋     | 23552/50176 [03:49<05:44, 77.17it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.03it/s]
 48%|████▊     | 24064/50176 [03:54<04:55, 88.23it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.81it/s]
 49%|████▉     | 24576/50176 [03:59<04:32, 93.94it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.96it/s]
 50%|█████     | 25088/50176 [04:04<04:06, 101.79it/s]
 51%|█████     | 25344/50176 [04:06<03:56, 104.94it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.33it/s]
 52%|█████▏    | 25856/50176 [04:11<03:46, 107.21it/s]
 52%|█████▏    | 26112/50176 [04:13<03:44, 107.37it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.23it/s]
 53%|█████▎    | 26624/50176 [04:18<03:38, 107.89it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.14it/s]
 54%|█████▍    | 27136/50176 [04:23<03:41, 104.07it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.72it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.57it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.50it/s]
 56%|█████▌    | 28160/50176 [04:35<04:00, 91.52it/s]
 57%|█████▋    | 28416/50176 [04:38<03:52, 93.77it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 94.01it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.48it/s]
 58%|█████▊    | 29184/50176 [04:45<03:30, 99.85it/s]
 59%|█████▊    | 29440/50176 [04:48<03:36, 95.73it/s]
 59%|█████▉    | 29696/50176 [04:51<03:34, 95.69it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.72it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.76it/s]
 61%|██████    | 30464/50176 [04:58<03:10, 103.21it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.62it/s]
 62%|██████▏   | 30976/50176 [05:03<03:02, 104.98it/s]
 62%|██████▏   | 31232/50176 [05:05<02:59, 105.43it/s]
 63%|██████▎   | 31488/50176 [05:08<02:51, 108.71it/s]
 63%|██████▎   | 31744/50176 [05:10<02:50, 107.96it/s]
 64%|██████▍   | 32000/50176 [05:12<02:48, 107.91it/s]
 64%|██████▍   | 32256/50176 [05:15<02:43, 109.83it/s]
 65%|██████▍   | 32512/50176 [05:17<02:47, 105.47it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.23it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.54it/s]
 66%|██████▋   | 33280/50176 [05:24<02:36, 107.64it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.59it/s]
 67%|██████▋   | 33792/50176 [05:29<02:32, 107.61it/s]
 68%|██████▊   | 34048/50176 [05:31<02:24, 111.59it/s]
 68%|██████▊   | 34304/50176 [05:33<02:19, 113.81it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.24it/s]
 69%|██████▉   | 34816/50176 [05:38<02:23, 107.19it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.10it/s]
 70%|███████   | 35328/50176 [05:43<02:17, 108.33it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 108.03it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.74it/s]
 72%|███████▏  | 36096/50176 [05:50<02:11, 107.48it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.47it/s]
 73%|███████▎  | 36608/50176 [05:55<02:09, 104.72it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.81it/s]
 74%|███████▍  | 37120/50176 [06:00<02:05, 103.87it/s]
 74%|███████▍  | 37376/50176 [06:02<01:56, 109.54it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.82it/s]
 76%|███████▌  | 37888/50176 [06:07<01:53, 107.89it/s]
 76%|███████▌  | 38144/50176 [06:10<01:51, 108.29it/s]
 77%|███████▋  | 38400/50176 [06:12<01:49, 107.26it/s]
 77%|███████▋  | 38656/50176 [06:15<01:48, 106.64it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.51it/s]
 78%|███████▊  | 39168/50176 [06:19<01:44, 105.57it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.32it/s]
 79%|███████▉  | 39680/50176 [06:24<01:38, 106.78it/s]
 80%|███████▉  | 39936/50176 [06:26<01:34, 107.85it/s]
 80%|████████  | 40192/50176 [06:29<01:33, 106.26it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.81it/s]
 81%|████████  | 40704/50176 [06:34<01:33, 101.17it/s]
 82%|████████▏ | 40960/50176 [06:36<01:27, 105.60it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.24it/s]
 83%|████████▎ | 41472/50176 [06:41<01:22, 105.66it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.30it/s]
 84%|████████▎ | 41984/50176 [06:46<01:18, 104.67it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.87it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.39it/s]
 85%|████████▌ | 42752/50176 [06:53<01:08, 108.34it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.90it/s]
 86%|████████▌ | 43264/50176 [06:58<01:05, 105.68it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.21it/s]
 87%|████████▋ | 43776/50176 [07:03<01:02, 102.71it/s]
 88%|████████▊ | 44032/50176 [07:06<00:59, 104.05it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.89it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 101.01it/s]
 89%|████████▉ | 44800/50176 [07:13<00:52, 102.41it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.25it/s]
 90%|█████████ | 45312/50176 [07:18<00:47, 102.62it/s]
 91%|█████████ | 45568/50176 [07:20<00:43, 106.11it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.85it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.18it/s]
 92%|█████████▏| 46336/50176 [07:28<00:37, 101.07it/s]
 93%|█████████▎| 46592/50176 [07:30<00:34, 105.08it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.00it/s]
 94%|█████████▍| 47104/50176 [07:35<00:30, 102.11it/s]
 94%|█████████▍| 47360/50176 [07:38<00:28, 100.22it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.34it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.69it/s]
 96%|█████████▌| 48128/50176 [07:45<00:19, 104.57it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.11it/s]
 97%|█████████▋| 48640/50176 [07:50<00:14, 104.75it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.40it/s]
 98%|█████████▊| 49152/50176 [07:55<00:09, 103.41it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.11it/s]
 99%|█████████▉| 49664/50176 [08:00<00:04, 102.46it/s]
 99%|█████████▉| 49920/50176 [08:03<00:02, 95.31it/s] 
100%|██████████| 50176/50176 [08:04<00:00, 124.45it/s]
100%|██████████| 50176/50176 [08:04<00:00, 103.55it/s]
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5424	imagenet-zeroshot-val-top5: 0.8184
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d74e059ceac10f834ae936f837381b8039f897dc
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb3731cab5b02ca146c4abcdfb451950a8e498888bfeebdb51f2976c2134da7a
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..64e7fa0667106478e1052eb596c54b92b187e8f2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_15.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:27,  1.45it/s]
  1%|          | 7/1000 [00:00<01:29, 11.16it/s]
  1%|▏         | 13/1000 [00:00<00:48, 20.28it/s]
  2%|▏         | 19/1000 [00:01<00:34, 28.59it/s]
  2%|▎         | 25/1000 [00:01<00:27, 35.55it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.26it/s]
  4%|▎         | 37/1000 [00:01<00:21, 45.66it/s]
  4%|▍         | 43/1000 [00:01<00:19, 48.78it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.04it/s]
  6%|▌         | 55/1000 [00:01<00:17, 52.84it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.31it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.29it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.00it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.60it/s]
  8%|▊         | 85/1000 [00:02<00:16, 57.09it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.28it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.68it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.87it/s]
 11%|█         | 109/1000 [00:02<00:15, 57.96it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 57.90it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.92it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.00it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.06it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.79it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 57.95it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.06it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.08it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.11it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.92it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.89it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.84it/s]
 19%|█▉        | 193/1000 [00:04<00:13, 57.80it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.74it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.60it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.40it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.61it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.83it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.82it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.63it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.76it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.83it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.93it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.90it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.74it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.89it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.81it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 57.94it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.14it/s]
 30%|███       | 301/1000 [00:05<00:12, 58.16it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.15it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.35it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.39it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.23it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.02it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.20it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.29it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.25it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 58.18it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.24it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.29it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.34it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.12it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.95it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 58.01it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.07it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.85it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.03it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.16it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.22it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.25it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.15it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.24it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.18it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.24it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.30it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.25it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.40it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.20it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.31it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.27it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.18it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.23it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.13it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.26it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.24it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.30it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.34it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.37it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.39it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.29it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.34it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.14it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.20it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.29it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.30it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.22it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.32it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.19it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.22it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.37it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.34it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.33it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.26it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.23it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.30it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.29it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.37it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.35it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.39it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.37it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.30it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.31it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.32it/s]
 72%|███████▏  | 715/1000 [00:12<00:04, 58.30it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.31it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.28it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.29it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.34it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.36it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.43it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.46it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.27it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.35it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.35it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.38it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.37it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.23it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.23it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.17it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.22it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.19it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.31it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.47it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.51it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.37it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.40it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.40it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.36it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.39it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.34it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.24it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.34it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.31it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.28it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.36it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.33it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.43it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.48it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.43it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.40it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.55it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.42it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.54it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.65it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.64it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.58it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.46it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 58.26it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.92it/s]
+2022-12-21,09:46:23 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:36, 109.39it/s]
  1%|          | 512/50176 [00:04<07:01, 117.87it/s]
  2%|▏         | 768/50176 [00:06<06:58, 117.94it/s]
  2%|▏         | 1024/50176 [00:08<06:57, 117.62it/s]
  3%|▎         | 1280/50176 [00:11<07:11, 113.30it/s]
  3%|▎         | 1536/50176 [00:13<07:41, 105.31it/s]
  4%|▎         | 1792/50176 [00:16<07:28, 107.87it/s]
  4%|▍         | 2048/50176 [00:18<07:17, 110.04it/s]
  5%|▍         | 2304/50176 [00:20<07:06, 112.21it/s]
  5%|▌         | 2560/50176 [00:22<07:02, 112.70it/s]
  6%|▌         | 2816/50176 [00:25<07:09, 110.17it/s]
  6%|▌         | 3072/50176 [00:27<07:13, 108.72it/s]
  7%|▋         | 3328/50176 [00:30<07:08, 109.35it/s]
  7%|▋         | 3584/50176 [00:32<06:54, 112.38it/s]
  8%|▊         | 3840/50176 [00:34<07:19, 105.43it/s]
  8%|▊         | 4096/50176 [00:37<07:37, 100.75it/s]
  9%|▊         | 4352/50176 [00:40<07:29, 101.95it/s]
  9%|▉         | 4608/50176 [00:42<07:14, 104.96it/s]
 10%|▉         | 4864/50176 [00:44<07:07, 106.07it/s]
 10%|█         | 5120/50176 [00:47<07:03, 106.32it/s]
 11%|█         | 5376/50176 [00:49<07:09, 104.42it/s]
 11%|█         | 5632/50176 [00:52<07:10, 103.35it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.62it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.94it/s] 
 13%|█▎        | 6400/50176 [01:00<07:30, 97.11it/s]
 13%|█▎        | 6656/50176 [01:02<07:21, 98.58it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.71it/s]
 14%|█▍        | 7168/50176 [01:08<07:14, 98.91it/s]
 15%|█▍        | 7424/50176 [01:10<07:02, 101.29it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.02it/s]
 16%|█▌        | 7936/50176 [01:15<06:47, 103.61it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.06it/s]
 17%|█▋        | 8448/50176 [01:20<06:51, 101.52it/s]
 17%|█▋        | 8704/50176 [01:22<06:48, 101.49it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.90it/s]
 18%|█▊        | 9216/50176 [01:27<06:43, 101.39it/s]
 19%|█▉        | 9472/50176 [01:30<06:40, 101.53it/s]
 19%|█▉        | 9728/50176 [01:33<06:43, 100.17it/s]
 20%|█▉        | 9984/50176 [01:35<06:42, 99.84it/s] 
 20%|██        | 10240/50176 [01:38<06:29, 102.54it/s]
 21%|██        | 10496/50176 [01:40<06:20, 104.28it/s]
 21%|██▏       | 10752/50176 [01:43<06:29, 101.33it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.75it/s]
 22%|██▏       | 11264/50176 [01:47<06:16, 103.22it/s]
 23%|██▎       | 11520/50176 [01:50<06:09, 104.52it/s]
 23%|██▎       | 11776/50176 [01:52<06:05, 105.06it/s]
 24%|██▍       | 12032/50176 [01:55<06:05, 104.44it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.82it/s]
 25%|██▌       | 12544/50176 [01:59<05:49, 107.55it/s]
 26%|██▌       | 12800/50176 [02:02<05:51, 106.22it/s]
 26%|██▌       | 13056/50176 [02:04<05:48, 106.58it/s]
 27%|██▋       | 13312/50176 [02:07<05:48, 105.79it/s]
 27%|██▋       | 13568/50176 [02:10<06:05, 100.27it/s]
 28%|██▊       | 13824/50176 [02:12<06:07, 98.80it/s] 
 28%|██▊       | 14080/50176 [02:15<06:00, 100.03it/s]
 29%|██▊       | 14336/50176 [02:17<05:58, 100.10it/s]
 29%|██▉       | 14592/50176 [02:20<05:50, 101.48it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.68it/s] 
 30%|███       | 15104/50176 [02:25<05:55, 98.74it/s]
 31%|███       | 15360/50176 [02:28<05:47, 100.18it/s]
 31%|███       | 15616/50176 [02:30<05:37, 102.31it/s]
 32%|███▏      | 15872/50176 [02:32<05:39, 101.16it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.91it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.56it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.15it/s] 
 34%|███▎      | 16896/50176 [02:43<05:30, 100.61it/s]
 34%|███▍      | 17152/50176 [02:45<05:22, 102.32it/s]
 35%|███▍      | 17408/50176 [02:48<05:17, 103.14it/s]
 35%|███▌      | 17664/50176 [02:50<05:03, 107.06it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 109.03it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.73it/s]
 37%|███▋      | 18432/50176 [02:57<04:54, 107.74it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.54it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.65it/s]
 38%|███▊      | 19200/50176 [03:05<05:10, 99.88it/s] 
 39%|███▉      | 19456/50176 [03:07<05:12, 98.16it/s]
 39%|███▉      | 19712/50176 [03:10<05:13, 97.24it/s]
 40%|███▉      | 19968/50176 [03:13<05:11, 96.84it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.33it/s]
 41%|████      | 20480/50176 [03:17<04:47, 103.17it/s]
 41%|████▏     | 20736/50176 [03:20<04:42, 104.36it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.85it/s]
 42%|████▏     | 21248/50176 [03:25<04:41, 102.69it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 102.93it/s]
 43%|████▎     | 21760/50176 [03:30<04:30, 105.24it/s]
 44%|████▍     | 22016/50176 [03:32<04:19, 108.34it/s]
 44%|████▍     | 22272/50176 [03:34<04:17, 108.29it/s]
 45%|████▍     | 22528/50176 [03:37<04:19, 106.38it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.41it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.27it/s] 
 46%|████▋     | 23296/50176 [03:45<04:33, 98.30it/s]
 47%|████▋     | 23552/50176 [03:50<05:44, 77.24it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.05it/s]
 48%|████▊     | 24064/50176 [03:55<04:56, 88.08it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.81it/s]
 49%|████▉     | 24576/50176 [04:00<04:32, 93.99it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.98it/s]
 50%|█████     | 25088/50176 [04:05<04:06, 101.84it/s]
 51%|█████     | 25344/50176 [04:07<03:56, 104.89it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.28it/s]
 52%|█████▏    | 25856/50176 [04:12<03:46, 107.15it/s]
 52%|█████▏    | 26112/50176 [04:14<03:44, 107.33it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.22it/s]
 53%|█████▎    | 26624/50176 [04:19<03:38, 107.97it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.17it/s]
 54%|█████▍    | 27136/50176 [04:24<03:41, 104.16it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.66it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.59it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.54it/s]
 56%|█████▌    | 28160/50176 [04:36<04:00, 91.52it/s]
 57%|█████▋    | 28416/50176 [04:38<03:52, 93.71it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 94.05it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.45it/s]
 58%|█████▊    | 29184/50176 [04:46<03:30, 99.91it/s]
 59%|█████▊    | 29440/50176 [04:49<03:36, 95.63it/s]
 59%|█████▉    | 29696/50176 [04:51<03:33, 95.86it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.58it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.84it/s]
 61%|██████    | 30464/50176 [04:59<03:10, 103.40it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.59it/s]
 62%|██████▏   | 30976/50176 [05:04<03:03, 104.87it/s]
 62%|██████▏   | 31232/50176 [05:06<02:59, 105.56it/s]
 63%|██████▎   | 31488/50176 [05:08<02:52, 108.53it/s]
 63%|██████▎   | 31744/50176 [05:11<02:50, 107.80it/s]
 64%|██████▍   | 32000/50176 [05:13<02:48, 107.86it/s]
 64%|██████▍   | 32256/50176 [05:15<02:42, 110.02it/s]
 65%|██████▍   | 32512/50176 [05:18<02:47, 105.37it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.18it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.64it/s]
 66%|██████▋   | 33280/50176 [05:25<02:36, 107.65it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.56it/s]
 67%|██████▋   | 33792/50176 [05:30<02:32, 107.62it/s]
 68%|██████▊   | 34048/50176 [05:32<02:24, 111.54it/s]
 68%|██████▊   | 34304/50176 [05:34<02:19, 113.79it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.15it/s]
 69%|██████▉   | 34816/50176 [05:39<02:23, 107.18it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.13it/s]
 70%|███████   | 35328/50176 [05:44<02:16, 108.61it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 108.00it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.68it/s]
 72%|███████▏  | 36096/50176 [05:51<02:11, 107.34it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.51it/s]
 73%|███████▎  | 36608/50176 [05:56<02:09, 104.60it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.83it/s]
 74%|███████▍  | 37120/50176 [06:01<02:05, 103.96it/s]
 74%|███████▍  | 37376/50176 [06:03<01:56, 109.46it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.81it/s]
 76%|███████▌  | 37888/50176 [06:08<01:53, 107.85it/s]
 76%|███████▌  | 38144/50176 [06:10<01:51, 108.36it/s]
 77%|███████▋  | 38400/50176 [06:13<01:49, 107.24it/s]
 77%|███████▋  | 38656/50176 [06:15<01:48, 106.63it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.65it/s]
 78%|███████▊  | 39168/50176 [06:20<01:44, 105.55it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.21it/s]
 79%|███████▉  | 39680/50176 [06:25<01:38, 106.76it/s]
 80%|███████▉  | 39936/50176 [06:27<01:34, 108.03it/s]
 80%|████████  | 40192/50176 [06:29<01:34, 106.19it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.91it/s]
 81%|████████  | 40704/50176 [06:35<01:33, 101.16it/s]
 82%|████████▏ | 40960/50176 [06:37<01:27, 105.46it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.41it/s]
 83%|████████▎ | 41472/50176 [06:42<01:22, 105.51it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.42it/s]
 84%|████████▎ | 41984/50176 [06:47<01:18, 104.67it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.82it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.36it/s]
 85%|████████▌ | 42752/50176 [06:54<01:08, 108.37it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.84it/s]
 86%|████████▌ | 43264/50176 [06:59<01:05, 105.59it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.31it/s]
 87%|████████▋ | 43776/50176 [07:04<01:02, 102.92it/s]
 88%|████████▊ | 44032/50176 [07:06<00:58, 104.21it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.78it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 100.91it/s]
 89%|████████▉ | 44800/50176 [07:14<00:52, 102.36it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.15it/s]
 90%|█████████ | 45312/50176 [07:19<00:47, 102.54it/s]
 91%|█████████ | 45568/50176 [07:21<00:43, 106.08it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.86it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.19it/s]
 92%|█████████▏| 46336/50176 [07:29<00:37, 101.12it/s]
 93%|█████████▎| 46592/50176 [07:31<00:34, 105.08it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.18it/s]
 94%|█████████▍| 47104/50176 [07:36<00:30, 102.05it/s]
 94%|█████████▍| 47360/50176 [07:39<00:28, 100.27it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.36it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.79it/s]
 96%|█████████▌| 48128/50176 [07:46<00:19, 104.47it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.21it/s]
 97%|█████████▋| 48640/50176 [07:51<00:14, 104.58it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.36it/s]
 98%|█████████▊| 49152/50176 [07:56<00:09, 103.35it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.19it/s]
 99%|█████████▉| 49664/50176 [08:01<00:05, 102.35it/s]
 99%|█████████▉| 49920/50176 [08:04<00:02, 95.28it/s] 
100%|██████████| 50176/50176 [08:04<00:00, 124.58it/s]
100%|██████████| 50176/50176 [08:05<00:00, 103.45it/s]
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5514	imagenet-zeroshot-val-top5: 0.8246
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt
new file mode 100644
index 0000000000000000000000000000000000000000..68233a7f9b4c3d641882613d07c66b33b9c0e8f6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:976d520ae21e33924720b48b31f6b4d93b8936143fdbc82efcf0b5b39c23d8df
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..fcaf50d218fc95683e72bd3617783fd4cb9444ac
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_16.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:00,  1.39it/s]
  1%|          | 6/1000 [00:00<01:45,  9.41it/s]
  1%|          | 12/1000 [00:00<00:53, 18.51it/s]
  2%|▏         | 18/1000 [00:01<00:36, 27.00it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.24it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.18it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.69it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.17it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.79it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.70it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.05it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.12it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.84it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.36it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.90it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.20it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.59it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.64it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.91it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.05it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.03it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.99it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.04it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.00it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.04it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.09it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.96it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.80it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.80it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.72it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.66it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.42it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.35it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.52it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.68it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.66it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.56it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.60it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.69it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.80it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.85it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.70it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.86it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.89it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.09it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.06it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.05it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.20it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.19it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.11it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.11it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.20it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.99it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.99it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.07it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.16it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.74it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.79it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.88it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.02it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.07it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.09it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.13it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.14it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.23it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.11it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.02it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.07it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.05it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.08it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.12it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.22it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.25it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.14it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.12it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.04it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.00it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.12it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.04it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.05it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.15it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.15it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.25it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.08it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.02it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.17it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.14it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.22it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.22it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.23it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.12it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.21it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.06it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.98it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.04it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.02it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.07it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.12it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.12it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.21it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.29it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.14it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.23it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.18it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.21it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.24it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.04it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.04it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.01it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.03it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.14it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.28it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.31it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.28it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.24it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.25it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.22it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.09it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.17it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.13it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.22it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.26it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.27it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.31it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.26it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.40it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.22it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.22it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.06it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.66it/s]
+2022-12-21,09:46:23 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:14, 133.40it/s]
  1%|          | 512/50176 [00:04<06:31, 126.77it/s]
  2%|▏         | 768/50176 [00:06<06:42, 122.82it/s]
  2%|▏         | 1024/50176 [00:08<06:46, 121.01it/s]
  3%|▎         | 1280/50176 [00:10<07:05, 114.85it/s]
  3%|▎         | 1536/50176 [00:13<07:37, 106.42it/s]
  4%|▎         | 1792/50176 [00:15<07:24, 108.95it/s]
  4%|▍         | 2048/50176 [00:17<07:15, 110.41it/s]
  5%|▍         | 2304/50176 [00:20<07:04, 112.80it/s]
  5%|▌         | 2560/50176 [00:22<07:01, 112.99it/s]
  6%|▌         | 2816/50176 [00:24<07:08, 110.47it/s]
  6%|▌         | 3072/50176 [00:27<07:13, 108.77it/s]
  7%|▋         | 3328/50176 [00:29<07:07, 109.48it/s]
  7%|▋         | 3584/50176 [00:31<06:55, 112.19it/s]
  8%|▊         | 3840/50176 [00:34<07:19, 105.46it/s]
  8%|▊         | 4096/50176 [00:37<07:36, 100.91it/s]
  9%|▊         | 4352/50176 [00:39<07:29, 101.89it/s]
  9%|▉         | 4608/50176 [00:42<07:13, 105.05it/s]
 10%|▉         | 4864/50176 [00:44<07:06, 106.23it/s]
 10%|█         | 5120/50176 [00:46<07:03, 106.27it/s]
 11%|█         | 5376/50176 [00:49<07:08, 104.59it/s]
 11%|█         | 5632/50176 [00:51<07:11, 103.18it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.56it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.77it/s] 
 13%|█▎        | 6400/50176 [00:59<07:31, 97.06it/s]
 13%|█▎        | 6656/50176 [01:02<07:21, 98.60it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.76it/s]
 14%|█▍        | 7168/50176 [01:07<07:14, 98.91it/s]
 15%|█▍        | 7424/50176 [01:10<07:02, 101.31it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.02it/s]
 16%|█▌        | 7936/50176 [01:14<06:47, 103.56it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.02it/s]
 17%|█▋        | 8448/50176 [01:19<06:50, 101.54it/s]
 17%|█▋        | 8704/50176 [01:22<06:47, 101.66it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.87it/s]
 18%|█▊        | 9216/50176 [01:27<06:44, 101.37it/s]
 19%|█▉        | 9472/50176 [01:30<06:40, 101.52it/s]
 19%|█▉        | 9728/50176 [01:32<06:44, 100.08it/s]
 20%|█▉        | 9984/50176 [01:35<06:42, 99.86it/s] 
 20%|██        | 10240/50176 [01:37<06:29, 102.65it/s]
 21%|██        | 10496/50176 [01:40<06:20, 104.16it/s]
 21%|██▏       | 10752/50176 [01:42<06:28, 101.35it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.76it/s]
 22%|██▏       | 11264/50176 [01:47<06:17, 103.18it/s]
 23%|██▎       | 11520/50176 [01:49<06:09, 104.59it/s]
 23%|██▎       | 11776/50176 [01:52<06:05, 104.97it/s]
 24%|██▍       | 12032/50176 [01:54<06:05, 104.40it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.82it/s]
 25%|██▌       | 12544/50176 [01:59<05:50, 107.48it/s]
 26%|██▌       | 12800/50176 [02:01<05:51, 106.20it/s]
 26%|██▌       | 13056/50176 [02:04<05:48, 106.65it/s]
 27%|██▋       | 13312/50176 [02:06<05:48, 105.84it/s]
 27%|██▋       | 13568/50176 [02:09<06:05, 100.19it/s]
 28%|██▊       | 13824/50176 [02:12<06:08, 98.72it/s] 
 28%|██▊       | 14080/50176 [02:14<06:00, 100.10it/s]
 29%|██▊       | 14336/50176 [02:17<05:57, 100.21it/s]
 29%|██▉       | 14592/50176 [02:19<05:50, 101.57it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.68it/s] 
 30%|███       | 15104/50176 [02:25<05:55, 98.69it/s]
 31%|███       | 15360/50176 [02:27<05:47, 100.14it/s]
 31%|███       | 15616/50176 [02:29<05:37, 102.28it/s]
 32%|███▏      | 15872/50176 [02:32<05:38, 101.21it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.86it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.67it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.06it/s] 
 34%|███▎      | 16896/50176 [02:42<05:30, 100.68it/s]
 34%|███▍      | 17152/50176 [02:45<05:22, 102.29it/s]
 35%|███▍      | 17408/50176 [02:47<05:17, 103.11it/s]
 35%|███▌      | 17664/50176 [02:49<05:03, 107.00it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 109.00it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.73it/s]
 37%|███▋      | 18432/50176 [02:56<04:54, 107.71it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.57it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.62it/s]
 38%|███▊      | 19200/50176 [03:04<05:09, 99.95it/s] 
 39%|███▉      | 19456/50176 [03:07<05:12, 98.19it/s]
 39%|███▉      | 19712/50176 [03:10<05:12, 97.36it/s]
 40%|███▉      | 19968/50176 [03:12<05:12, 96.78it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.35it/s]
 41%|████      | 20480/50176 [03:17<04:47, 103.12it/s]
 41%|████▏     | 20736/50176 [03:19<04:41, 104.40it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.90it/s]
 42%|████▏     | 21248/50176 [03:24<04:41, 102.59it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 102.97it/s]
 43%|████▎     | 21760/50176 [03:29<04:30, 105.19it/s]
 44%|████▍     | 22016/50176 [03:31<04:20, 108.30it/s]
 44%|████▍     | 22272/50176 [03:34<04:17, 108.27it/s]
 45%|████▍     | 22528/50176 [03:36<04:19, 106.36it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.43it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.28it/s] 
 46%|████▋     | 23296/50176 [03:44<04:33, 98.30it/s]
 47%|████▋     | 23552/50176 [03:49<05:44, 77.21it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.04it/s]
 48%|████▊     | 24064/50176 [03:54<04:56, 88.14it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.89it/s]
 49%|████▉     | 24576/50176 [04:00<04:32, 93.92it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.96it/s]
 50%|█████     | 25088/50176 [04:04<04:06, 101.89it/s]
 51%|█████     | 25344/50176 [04:06<03:56, 104.97it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.23it/s]
 52%|█████▏    | 25856/50176 [04:11<03:47, 107.10it/s]
 52%|█████▏    | 26112/50176 [04:14<03:44, 107.40it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.16it/s]
 53%|█████▎    | 26624/50176 [04:18<03:38, 107.90it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.14it/s]
 54%|█████▍    | 27136/50176 [04:23<03:41, 104.15it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.66it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.59it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.53it/s]
 56%|█████▌    | 28160/50176 [04:35<04:00, 91.51it/s]
 57%|█████▋    | 28416/50176 [04:38<03:51, 93.86it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 94.01it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.41it/s]
 58%|█████▊    | 29184/50176 [04:45<03:29, 100.00it/s]
 59%|█████▊    | 29440/50176 [04:48<03:37, 95.56it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:33, 95.78it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.60it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.78it/s]
 61%|██████    | 30464/50176 [04:58<03:10, 103.34it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.55it/s]
 62%|██████▏   | 30976/50176 [05:03<03:02, 105.01it/s]
 62%|██████▏   | 31232/50176 [05:05<02:59, 105.51it/s]
 63%|██████▎   | 31488/50176 [05:08<02:52, 108.54it/s]
 63%|██████▎   | 31744/50176 [05:10<02:50, 107.85it/s]
 64%|██████▍   | 32000/50176 [05:12<02:48, 107.86it/s]
 64%|██████▍   | 32256/50176 [05:15<02:42, 109.99it/s]
 65%|██████▍   | 32512/50176 [05:17<02:47, 105.30it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.14it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.65it/s]
 66%|██████▋   | 33280/50176 [05:24<02:36, 107.81it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.63it/s]
 67%|██████▋   | 33792/50176 [05:29<02:32, 107.59it/s]
 68%|██████▊   | 34048/50176 [05:31<02:24, 111.43it/s]
 68%|██████▊   | 34304/50176 [05:33<02:19, 114.06it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.16it/s]
 69%|██████▉   | 34816/50176 [05:38<02:23, 107.11it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.20it/s]
 70%|███████   | 35328/50176 [05:43<02:16, 108.46it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 107.88it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.77it/s]
 72%|███████▏  | 36096/50176 [05:50<02:11, 107.35it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.50it/s]
 73%|███████▎  | 36608/50176 [05:55<02:09, 104.60it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.82it/s]
 74%|███████▍  | 37120/50176 [06:00<02:05, 103.92it/s]
 74%|███████▍  | 37376/50176 [06:03<01:56, 109.60it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.75it/s]
 76%|███████▌  | 37888/50176 [06:07<01:53, 107.83it/s]
 76%|███████▌  | 38144/50176 [06:10<01:50, 108.46it/s]
 77%|███████▋  | 38400/50176 [06:12<01:49, 107.37it/s]
 77%|███████▋  | 38656/50176 [06:15<01:48, 106.56it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.47it/s]
 78%|███████▊  | 39168/50176 [06:19<01:44, 105.49it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.20it/s]
 79%|███████▉  | 39680/50176 [06:24<01:38, 106.74it/s]
 80%|███████▉  | 39936/50176 [06:26<01:34, 108.17it/s]
 80%|████████  | 40192/50176 [06:29<01:34, 106.13it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.81it/s]
 81%|████████  | 40704/50176 [06:34<01:33, 101.17it/s]
 82%|████████▏ | 40960/50176 [06:36<01:27, 105.48it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.51it/s]
 83%|████████▎ | 41472/50176 [06:41<01:22, 105.53it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.33it/s]
 84%|████████▎ | 41984/50176 [06:46<01:18, 104.65it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.90it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.41it/s]
 85%|████████▌ | 42752/50176 [06:53<01:08, 108.34it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.82it/s]
 86%|████████▌ | 43264/50176 [06:58<01:05, 105.60it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.31it/s]
 87%|████████▋ | 43776/50176 [07:03<01:02, 103.01it/s]
 88%|████████▊ | 44032/50176 [07:06<00:59, 104.13it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.88it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 100.87it/s]
 89%|████████▉ | 44800/50176 [07:13<00:52, 102.34it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.10it/s]
 90%|█████████ | 45312/50176 [07:18<00:47, 102.52it/s]
 91%|█████████ | 45568/50176 [07:20<00:43, 106.01it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.86it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.16it/s]
 92%|█████████▏| 46336/50176 [07:28<00:37, 101.09it/s]
 93%|█████████▎| 46592/50176 [07:30<00:34, 105.18it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.24it/s]
 94%|█████████▍| 47104/50176 [07:36<00:30, 101.97it/s]
 94%|█████████▍| 47360/50176 [07:38<00:28, 100.20it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.48it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.77it/s]
 96%|█████████▌| 48128/50176 [07:45<00:19, 104.47it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.08it/s]
 97%|█████████▋| 48640/50176 [07:50<00:14, 104.60it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.40it/s]
 98%|█████████▊| 49152/50176 [07:55<00:09, 103.34it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.21it/s]
 99%|█████████▉| 49664/50176 [08:00<00:05, 102.34it/s]
 99%|█████████▉| 49920/50176 [08:03<00:02, 95.40it/s] 
100%|██████████| 50176/50176 [08:04<00:00, 124.66it/s]
100%|██████████| 50176/50176 [08:04<00:00, 103.54it/s]
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5533	imagenet-zeroshot-val-top5: 0.8258
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a5acd29848adcc20adc213519b3d372c70febff1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ed5abf7e7d0f4ebccc5445b249fe1ad5b224cf3a286014b785a1e4c139bbbd8
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..afe6ca0e6efb6e453e824ff862ce759ec3ff9e62
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt).
+2022-12-21,09:54:46 | INFO | Model:
+2022-12-21,09:54:46 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:46 | INFO | Params:
+2022-12-21,09:54:46 | INFO |   batch_size: 256
+2022-12-21,09:54:46 | INFO |   beta1: 0.9
+2022-12-21,09:54:46 | INFO |   beta2: 0.98
+2022-12-21,09:54:46 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/checkpoints
+2022-12-21,09:54:46 | INFO |   copy_codebase: False
+2022-12-21,09:54:46 | INFO |   csv_caption_key: title
+2022-12-21,09:54:46 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:46 | INFO |   csv_separator: 	
+2022-12-21,09:54:46 | INFO |   dataset_resampled: False
+2022-12-21,09:54:46 | INFO |   dataset_type: auto
+2022-12-21,09:54:46 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:46 | INFO |   debug: False
+2022-12-21,09:54:46 | INFO |   device: cuda:0
+2022-12-21,09:54:46 | INFO |   dist_backend: nccl
+2022-12-21,09:54:46 | INFO |   dist_url: env://
+2022-12-21,09:54:46 | INFO |   distributed: False
+2022-12-21,09:54:46 | INFO |   epochs: 32
+2022-12-21,09:54:46 | INFO |   eps: 1e-06
+2022-12-21,09:54:46 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:46 | INFO |   gather_with_grad: False
+2022-12-21,09:54:46 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:46 | INFO |   horovod: False
+2022-12-21,09:54:46 | INFO |   image_mean: None
+2022-12-21,09:54:46 | INFO |   image_std: None
+2022-12-21,09:54:46 | INFO |   imagenet_v2: None
+2022-12-21,09:54:46 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:46 | INFO |   local_loss: False
+2022-12-21,09:54:46 | INFO |   local_rank: 0
+2022-12-21,09:54:46 | INFO |   lock_image: False
+2022-12-21,09:54:46 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:46 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:46 | INFO |   log_level: 20
+2022-12-21,09:54:46 | INFO |   log_local: False
+2022-12-21,09:54:46 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/out.log
+2022-12-21,09:54:46 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:46 | INFO |   lr: 0.0005
+2022-12-21,09:54:46 | INFO |   model: ViT-B-16
+2022-12-21,09:54:46 | INFO |   name: epoch_17.pt
+2022-12-21,09:54:46 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:46 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:46 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:46 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt
+2022-12-21,09:54:46 | INFO |   pretrained_image: False
+2022-12-21,09:54:46 | INFO |   rank: 0
+2022-12-21,09:54:46 | INFO |   report_to: 
+2022-12-21,09:54:46 | INFO |   resume: None
+2022-12-21,09:54:46 | INFO |   save_frequency: 1
+2022-12-21,09:54:46 | INFO |   save_most_recent: False
+2022-12-21,09:54:46 | INFO |   seed: 0
+2022-12-21,09:54:46 | INFO |   skip_scheduler: False
+2022-12-21,09:54:46 | INFO |   tensorboard: False
+2022-12-21,09:54:46 | INFO |   tensorboard_path: 
+2022-12-21,09:54:46 | INFO |   torchscript: False
+2022-12-21,09:54:46 | INFO |   trace: False
+2022-12-21,09:54:46 | INFO |   train_data: None
+2022-12-21,09:54:46 | INFO |   train_num_samples: None
+2022-12-21,09:54:46 | INFO |   use_bn_sync: False
+2022-12-21,09:54:46 | INFO |   val_data: None
+2022-12-21,09:54:46 | INFO |   val_frequency: 1
+2022-12-21,09:54:46 | INFO |   val_num_samples: None
+2022-12-21,09:54:46 | INFO |   wandb: False
+2022-12-21,09:54:46 | INFO |   wandb_notes: 
+2022-12-21,09:54:46 | INFO |   warmup: 10000
+2022-12-21,09:54:46 | INFO |   wd: 0.2
+2022-12-21,09:54:46 | INFO |   workers: 1
+2022-12-21,09:54:46 | INFO |   world_size: 1
+2022-12-21,09:54:46 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:46 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:46 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:17,  1.48it/s]
  1%|          | 6/1000 [00:00<01:40,  9.91it/s]
  1%|          | 11/1000 [00:00<00:55, 17.83it/s]
  2%|▏         | 17/1000 [00:00<00:36, 26.77it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.34it/s]
  3%|▎         | 29/1000 [00:01<00:24, 40.44it/s]
  4%|▎         | 35/1000 [00:01<00:21, 45.07it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.64it/s]
  5%|▍         | 47/1000 [00:01<00:18, 51.05it/s]
  5%|▌         | 53/1000 [00:01<00:17, 52.98it/s]
  6%|▌         | 59/1000 [00:01<00:17, 54.22it/s]
  6%|▋         | 65/1000 [00:01<00:16, 55.23it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.91it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.42it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.93it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.32it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.52it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.69it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.97it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.06it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.01it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.96it/s]
 13%|█▎        | 131/1000 [00:02<00:14, 58.06it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.88it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.92it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.97it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.06it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.95it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.80it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.83it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.64it/s]
 19%|█▉        | 191/1000 [00:03<00:14, 57.62it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.47it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.36it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.34it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.37it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.57it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.62it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.48it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 57.47it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.53it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.73it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.78it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.55it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.79it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.67it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.94it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.96it/s]
 30%|███       | 305/1000 [00:05<00:11, 57.99it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.08it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.90it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.98it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.15it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.94it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.08it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.02it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.16it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.80it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.77it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.85it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.03it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.06it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 419/1000 [00:07<00:09, 58.13it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.04it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.21it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.23it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.04it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.10it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.15it/s]
 48%|████▊     | 479/1000 [00:08<00:08, 58.06it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.05it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.04it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.96it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.09it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.05it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.14it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.04it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.15it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.98it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.02it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.92it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.87it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.04it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 593/1000 [00:10<00:07, 57.92it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.05it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.95it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.12it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.16it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.88it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.09it/s]
 65%|██████▌   | 653/1000 [00:11<00:05, 58.02it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.11it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.08it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.07it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.11it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.06it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.81it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.80it/s]
 70%|███████   | 701/1000 [00:12<00:05, 57.96it/s]
 71%|███████   | 707/1000 [00:12<00:05, 57.99it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.03it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.04it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.03it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.03it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.06it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.31it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.03it/s]
 77%|███████▋  | 767/1000 [00:13<00:04, 58.06it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.04it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.10it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.14it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.95it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.94it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.97it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.98it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 57.98it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.03it/s]
 83%|████████▎ | 827/1000 [00:14<00:02, 58.05it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.25it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.34it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.25it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.21it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.25it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.21it/s]
 89%|████████▊ | 887/1000 [00:15<00:01, 58.02it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.12it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.07it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.20it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.28it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.27it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.26it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.27it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.20it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.32it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.06it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.02it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 57.77it/s]
100%|█████████▉| 995/1000 [00:17<00:00, 57.41it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.72it/s]
+2022-12-21,09:55:04 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:54, 140.83it/s]
  1%|          | 512/50176 [00:03<05:38, 146.89it/s]
  2%|▏         | 768/50176 [00:05<05:35, 147.25it/s]
  2%|▏         | 1024/50176 [00:06<05:35, 146.69it/s]
  3%|▎         | 1280/50176 [00:08<05:38, 144.60it/s]
  3%|▎         | 1536/50176 [00:10<05:54, 137.35it/s]
  4%|▎         | 1792/50176 [00:12<05:53, 136.69it/s]
  4%|▍         | 2048/50176 [00:14<05:44, 139.63it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 138.94it/s]
  5%|▌         | 2560/50176 [00:18<05:35, 141.81it/s]
  6%|▌         | 2816/50176 [00:19<05:37, 140.14it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 142.75it/s]
  7%|▋         | 3328/50176 [00:23<05:26, 143.44it/s]
  7%|▋         | 3584/50176 [00:25<05:19, 145.73it/s]
  8%|▊         | 3840/50176 [00:27<05:31, 139.89it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.33it/s]
  9%|▊         | 4352/50176 [00:30<05:22, 142.16it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.32it/s]
 10%|▉         | 4864/50176 [00:34<05:11, 145.25it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.29it/s]
 11%|█         | 5376/50176 [00:37<05:04, 147.19it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.08it/s]
 12%|█▏        | 5888/50176 [00:41<04:58, 148.20it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 135.96it/s]
 13%|█▎        | 6400/50176 [00:44<05:11, 140.38it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.45it/s]
 14%|█▍        | 6912/50176 [00:48<05:03, 142.57it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 144.09it/s]
 15%|█▍        | 7424/50176 [00:51<04:51, 146.62it/s]
 15%|█▌        | 7680/50176 [00:53<04:51, 145.80it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 148.94it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.55it/s]
 17%|█▋        | 8448/50176 [00:58<04:38, 149.83it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.39it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.57it/s]
 18%|█▊        | 9216/50176 [01:03<04:35, 148.60it/s]
 19%|█▉        | 9472/50176 [01:05<04:32, 149.36it/s]
 19%|█▉        | 9728/50176 [01:07<04:36, 146.29it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.14it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.99it/s]
 21%|██        | 10496/50176 [01:12<04:21, 151.81it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.24it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.50it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.22it/s]
 23%|██▎       | 11520/50176 [01:19<04:18, 149.78it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 148.90it/s]
 24%|██▍       | 12032/50176 [01:22<04:14, 149.66it/s]
 24%|██▍       | 12288/50176 [01:24<04:13, 149.66it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.17it/s]
 26%|██▌       | 12800/50176 [01:27<04:07, 150.78it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.37it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.69it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 145.22it/s]
 28%|██▊       | 13824/50176 [01:35<04:16, 141.81it/s]
 28%|██▊       | 14080/50176 [01:36<04:12, 143.01it/s]
 29%|██▊       | 14336/50176 [01:38<04:15, 140.40it/s]
 29%|██▉       | 14592/50176 [01:40<04:05, 145.14it/s]
 30%|██▉       | 14848/50176 [01:42<04:17, 137.00it/s]
 30%|███       | 15104/50176 [01:44<04:18, 135.79it/s]
 31%|███       | 15360/50176 [01:46<04:10, 139.10it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.56it/s]
 32%|███▏      | 15872/50176 [01:49<04:09, 137.62it/s]
 32%|███▏      | 16128/50176 [01:51<04:02, 140.29it/s]
 33%|███▎      | 16384/50176 [01:53<04:00, 140.54it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 135.22it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.71it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.16it/s]
 35%|███▍      | 17408/50176 [02:00<03:46, 144.77it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.51it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.35it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.54it/s]
 37%|███▋      | 18432/50176 [02:07<03:33, 148.88it/s]
 37%|███▋      | 18688/50176 [02:09<03:33, 147.49it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.20it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.73it/s]
 39%|███▉      | 19456/50176 [02:15<03:54, 131.21it/s]
 39%|███▉      | 19712/50176 [02:17<03:52, 131.30it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.53it/s]
 40%|████      | 20224/50176 [02:20<03:32, 140.88it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.96it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 144.10it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.63it/s]
 42%|████▏     | 21248/50176 [02:27<03:24, 141.49it/s]
 43%|████▎     | 21504/50176 [02:29<03:16, 145.96it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.07it/s]
 44%|████▍     | 22016/50176 [02:32<03:07, 150.37it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.76it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.34it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 144.98it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.40it/s]
 46%|████▋     | 23296/50176 [02:41<03:11, 140.05it/s]
 47%|████▋     | 23552/50176 [02:43<03:15, 136.33it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.94it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.33it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 142.98it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.50it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.74it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 149.03it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.48it/s]
 51%|█████     | 25600/50176 [02:57<02:47, 147.06it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.12it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.27it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 143.56it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 146.10it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 146.05it/s]
 54%|█████▍    | 27136/50176 [03:07<02:36, 147.48it/s]
 55%|█████▍    | 27392/50176 [03:09<02:34, 147.82it/s]
 55%|█████▌    | 27648/50176 [03:11<02:27, 153.08it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.63it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.77it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.44it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.06it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.10it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.38it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.10it/s]
 59%|█████▉    | 29696/50176 [03:26<02:31, 134.87it/s]
 60%|█████▉    | 29952/50176 [03:28<02:31, 133.79it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 132.80it/s]
 61%|██████    | 30464/50176 [03:31<02:20, 140.35it/s]
 61%|██████    | 30720/50176 [03:33<02:18, 140.50it/s]
 62%|██████▏   | 30976/50176 [03:35<02:13, 143.40it/s]
 62%|██████▏   | 31232/50176 [03:36<02:09, 146.61it/s]
 63%|██████▎   | 31488/50176 [03:38<02:06, 148.27it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 148.44it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 146.86it/s]
 64%|██████▍   | 32256/50176 [03:43<02:01, 147.32it/s]
 65%|██████▍   | 32512/50176 [03:45<02:03, 142.46it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 145.75it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.31it/s]
 66%|██████▋   | 33280/50176 [03:50<01:55, 145.70it/s]
 67%|██████▋   | 33536/50176 [03:52<01:56, 142.64it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.39it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.55it/s]
 68%|██████▊   | 34304/50176 [03:57<01:44, 151.60it/s]
 69%|██████▉   | 34560/50176 [03:59<01:47, 145.26it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.35it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 143.96it/s]
 70%|███████   | 35328/50176 [04:04<01:40, 148.04it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 146.22it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 141.53it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 143.71it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.67it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 144.37it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 142.46it/s]
 74%|███████▍  | 37120/50176 [04:17<01:31, 143.37it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.56it/s]
 75%|███████▌  | 37632/50176 [04:20<01:27, 144.11it/s]
 76%|███████▌  | 37888/50176 [04:22<01:25, 143.66it/s]
 76%|███████▌  | 38144/50176 [04:24<01:22, 145.85it/s]
 77%|███████▋  | 38400/50176 [04:26<01:21, 145.37it/s]
 77%|███████▋  | 38656/50176 [04:28<01:19, 144.06it/s]
 78%|███████▊  | 38912/50176 [04:29<01:18, 142.60it/s]
 78%|███████▊  | 39168/50176 [04:31<01:19, 138.50it/s]
 79%|███████▊  | 39424/50176 [04:33<01:16, 141.21it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.33it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.25it/s]
 80%|████████  | 40192/50176 [04:38<01:07, 147.20it/s]
 81%|████████  | 40448/50176 [04:40<01:09, 140.74it/s]
 81%|████████  | 40704/50176 [04:42<01:09, 135.34it/s]
 82%|████████▏ | 40960/50176 [04:44<01:05, 140.40it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.24it/s]
 83%|████████▎ | 41472/50176 [04:47<00:59, 146.68it/s]
 83%|████████▎ | 41728/50176 [04:49<00:56, 149.11it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.17it/s]
 84%|████████▍ | 42240/50176 [04:52<00:54, 146.01it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 148.13it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.84it/s]
 86%|████████▌ | 43008/50176 [04:58<00:47, 149.62it/s]
 86%|████████▌ | 43264/50176 [04:59<00:46, 149.12it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.04it/s]
 87%|████████▋ | 43776/50176 [05:05<01:03, 101.41it/s]
 88%|████████▊ | 44032/50176 [05:07<00:53, 113.79it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.46it/s]
 89%|████████▉ | 44544/50176 [05:10<00:43, 129.85it/s]
 89%|████████▉ | 44800/50176 [05:12<00:40, 134.23it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.14it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.27it/s]
 91%|█████████ | 45568/50176 [05:18<00:35, 128.04it/s]
 91%|█████████▏| 45824/50176 [05:20<00:32, 135.52it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.19it/s]
 92%|█████████▏| 46336/50176 [05:23<00:26, 145.11it/s]
 93%|█████████▎| 46592/50176 [05:25<00:24, 148.03it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.66it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 144.70it/s]
 94%|█████████▍| 47360/50176 [05:30<00:19, 146.61it/s]
 95%|█████████▍| 47616/50176 [05:32<00:17, 145.44it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.42it/s]
 96%|█████████▌| 48128/50176 [05:35<00:13, 150.32it/s]
 96%|█████████▋| 48384/50176 [05:37<00:11, 152.05it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.19it/s]
 97%|█████████▋| 48896/50176 [05:40<00:08, 154.93it/s]
 98%|█████████▊| 49152/50176 [05:42<00:06, 150.32it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.14it/s]
 99%|█████████▉| 49664/50176 [05:45<00:03, 150.15it/s]
 99%|█████████▉| 49920/50176 [05:47<00:01, 142.49it/s]
100%|██████████| 50176/50176 [05:48<00:00, 186.20it/s]
100%|██████████| 50176/50176 [05:48<00:00, 144.02it/s]
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5545	imagenet-zeroshot-val-top5: 0.8302
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt
new file mode 100644
index 0000000000000000000000000000000000000000..526d9c747c1911d170d5c6c13a1a36d696819372
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f8195d9ce0dd3f92642ec6986268203b27a1a1b55d8fbaea363a067974012225
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3d6ff112a7f18110c693a03e94f3dfa2b835c618
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt).
+2022-12-21,09:54:45 | INFO | Model:
+2022-12-21,09:54:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:45 | INFO | Params:
+2022-12-21,09:54:45 | INFO |   batch_size: 256
+2022-12-21,09:54:45 | INFO |   beta1: 0.9
+2022-12-21,09:54:45 | INFO |   beta2: 0.98
+2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/checkpoints
+2022-12-21,09:54:45 | INFO |   copy_codebase: False
+2022-12-21,09:54:45 | INFO |   csv_caption_key: title
+2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:45 | INFO |   csv_separator: 	
+2022-12-21,09:54:45 | INFO |   dataset_resampled: False
+2022-12-21,09:54:45 | INFO |   dataset_type: auto
+2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:45 | INFO |   debug: False
+2022-12-21,09:54:45 | INFO |   device: cuda:0
+2022-12-21,09:54:45 | INFO |   dist_backend: nccl
+2022-12-21,09:54:45 | INFO |   dist_url: env://
+2022-12-21,09:54:45 | INFO |   distributed: False
+2022-12-21,09:54:45 | INFO |   epochs: 32
+2022-12-21,09:54:45 | INFO |   eps: 1e-06
+2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:45 | INFO |   gather_with_grad: False
+2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:45 | INFO |   horovod: False
+2022-12-21,09:54:45 | INFO |   image_mean: None
+2022-12-21,09:54:45 | INFO |   image_std: None
+2022-12-21,09:54:45 | INFO |   imagenet_v2: None
+2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:45 | INFO |   local_loss: False
+2022-12-21,09:54:45 | INFO |   local_rank: 0
+2022-12-21,09:54:45 | INFO |   lock_image: False
+2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:45 | INFO |   log_level: 20
+2022-12-21,09:54:45 | INFO |   log_local: False
+2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/out.log
+2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:45 | INFO |   lr: 0.0005
+2022-12-21,09:54:45 | INFO |   model: ViT-B-16
+2022-12-21,09:54:45 | INFO |   name: epoch_18.pt
+2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt
+2022-12-21,09:54:45 | INFO |   pretrained_image: False
+2022-12-21,09:54:45 | INFO |   rank: 0
+2022-12-21,09:54:45 | INFO |   report_to: 
+2022-12-21,09:54:45 | INFO |   resume: None
+2022-12-21,09:54:45 | INFO |   save_frequency: 1
+2022-12-21,09:54:45 | INFO |   save_most_recent: False
+2022-12-21,09:54:45 | INFO |   seed: 0
+2022-12-21,09:54:45 | INFO |   skip_scheduler: False
+2022-12-21,09:54:45 | INFO |   tensorboard: False
+2022-12-21,09:54:45 | INFO |   tensorboard_path: 
+2022-12-21,09:54:45 | INFO |   torchscript: False
+2022-12-21,09:54:45 | INFO |   trace: False
+2022-12-21,09:54:45 | INFO |   train_data: None
+2022-12-21,09:54:45 | INFO |   train_num_samples: None
+2022-12-21,09:54:45 | INFO |   use_bn_sync: False
+2022-12-21,09:54:45 | INFO |   val_data: None
+2022-12-21,09:54:45 | INFO |   val_frequency: 1
+2022-12-21,09:54:45 | INFO |   val_num_samples: None
+2022-12-21,09:54:45 | INFO |   wandb: False
+2022-12-21,09:54:45 | INFO |   wandb_notes: 
+2022-12-21,09:54:45 | INFO |   warmup: 10000
+2022-12-21,09:54:45 | INFO |   wd: 0.2
+2022-12-21,09:54:45 | INFO |   workers: 1
+2022-12-21,09:54:45 | INFO |   world_size: 1
+2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:45 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:45 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:34,  1.32it/s]
  1%|          | 6/1000 [00:00<01:50,  9.00it/s]
  1%|          | 11/1000 [00:00<01:00, 16.46it/s]
  2%|▏         | 16/1000 [00:01<00:42, 23.37it/s]
  2%|▏         | 22/1000 [00:01<00:31, 31.08it/s]
  3%|▎         | 28/1000 [00:01<00:25, 37.45it/s]
  3%|▎         | 34/1000 [00:01<00:22, 42.70it/s]
  4%|▍         | 40/1000 [00:01<00:20, 46.76it/s]
  5%|▍         | 46/1000 [00:01<00:19, 49.66it/s]
  5%|▌         | 52/1000 [00:01<00:18, 51.88it/s]
  6%|▌         | 58/1000 [00:01<00:17, 53.40it/s]
  6%|▋         | 64/1000 [00:01<00:17, 54.58it/s]
  7%|▋         | 70/1000 [00:02<00:16, 55.38it/s]
  8%|▊         | 76/1000 [00:02<00:16, 55.95it/s]
  8%|▊         | 82/1000 [00:02<00:16, 56.58it/s]
  9%|▉         | 88/1000 [00:02<00:16, 56.96it/s]
  9%|▉         | 94/1000 [00:02<00:16, 53.61it/s]
 10%|█         | 100/1000 [00:02<00:16, 54.81it/s]
 11%|█         | 106/1000 [00:02<00:16, 55.82it/s]
 11%|█         | 112/1000 [00:02<00:15, 56.49it/s]
 12%|█▏        | 118/1000 [00:02<00:15, 56.98it/s]
 12%|█▏        | 124/1000 [00:02<00:15, 57.00it/s]
 13%|█▎        | 130/1000 [00:03<00:15, 57.33it/s]
 14%|█▎        | 136/1000 [00:03<00:15, 57.48it/s]
 14%|█▍        | 142/1000 [00:03<00:14, 57.54it/s]
 15%|█▍        | 148/1000 [00:03<00:14, 57.71it/s]
 15%|█▌        | 154/1000 [00:03<00:14, 57.83it/s]
 16%|█▌        | 160/1000 [00:03<00:14, 57.75it/s]
 17%|█▋        | 166/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 172/1000 [00:03<00:14, 57.66it/s]
 18%|█▊        | 178/1000 [00:03<00:14, 57.68it/s]
 18%|█▊        | 184/1000 [00:04<00:14, 57.56it/s]
 19%|█▉        | 190/1000 [00:04<00:14, 57.57it/s]
 20%|█▉        | 196/1000 [00:04<00:13, 57.53it/s]
 20%|██        | 202/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 208/1000 [00:04<00:13, 57.22it/s]
 21%|██▏       | 214/1000 [00:04<00:13, 57.24it/s]
 22%|██▏       | 220/1000 [00:04<00:13, 57.28it/s]
 23%|██▎       | 226/1000 [00:04<00:13, 57.43it/s]
 23%|██▎       | 232/1000 [00:04<00:13, 57.53it/s]
 24%|██▍       | 238/1000 [00:04<00:13, 57.49it/s]
 24%|██▍       | 244/1000 [00:05<00:13, 57.40it/s]
 25%|██▌       | 250/1000 [00:05<00:13, 57.50it/s]
 26%|██▌       | 256/1000 [00:05<00:12, 57.68it/s]
 26%|██▌       | 262/1000 [00:05<00:12, 57.73it/s]
 27%|██▋       | 268/1000 [00:05<00:12, 57.65it/s]
 27%|██▋       | 274/1000 [00:05<00:12, 57.49it/s]
 28%|██▊       | 280/1000 [00:05<00:12, 57.69it/s]
 29%|██▊       | 286/1000 [00:05<00:12, 57.65it/s]
 29%|██▉       | 292/1000 [00:05<00:12, 57.89it/s]
 30%|██▉       | 298/1000 [00:05<00:12, 57.92it/s]
 30%|███       | 304/1000 [00:06<00:12, 57.97it/s]
 31%|███       | 310/1000 [00:06<00:11, 58.05it/s]
 32%|███▏      | 316/1000 [00:06<00:11, 58.06it/s]
 32%|███▏      | 322/1000 [00:06<00:11, 58.08it/s]
 33%|███▎      | 328/1000 [00:06<00:11, 57.84it/s]
 33%|███▎      | 334/1000 [00:06<00:11, 57.87it/s]
 34%|███▍      | 340/1000 [00:06<00:11, 57.89it/s]
 35%|███▍      | 346/1000 [00:06<00:11, 58.04it/s]
 35%|███▌      | 352/1000 [00:06<00:11, 57.90it/s]
 36%|███▌      | 358/1000 [00:07<00:11, 57.99it/s]
 36%|███▋      | 364/1000 [00:07<00:10, 57.98it/s]
 37%|███▋      | 370/1000 [00:07<00:10, 57.98it/s]
 38%|███▊      | 376/1000 [00:07<00:10, 57.85it/s]
 38%|███▊      | 382/1000 [00:07<00:10, 57.65it/s]
 39%|███▉      | 388/1000 [00:07<00:10, 57.58it/s]
 39%|███▉      | 394/1000 [00:07<00:10, 57.66it/s]
 40%|████      | 400/1000 [00:07<00:10, 57.88it/s]
 41%|████      | 406/1000 [00:07<00:10, 57.93it/s]
 41%|████      | 412/1000 [00:07<00:10, 58.03it/s]
 42%|████▏     | 418/1000 [00:08<00:10, 58.05it/s]
 42%|████▏     | 424/1000 [00:08<00:09, 57.95it/s]
 43%|████▎     | 430/1000 [00:08<00:09, 58.11it/s]
 44%|████▎     | 436/1000 [00:08<00:09, 58.13it/s]
 44%|████▍     | 442/1000 [00:08<00:09, 57.92it/s]
 45%|████▍     | 448/1000 [00:08<00:09, 57.93it/s]
 45%|████▌     | 454/1000 [00:08<00:09, 57.98it/s]
 46%|████▌     | 460/1000 [00:08<00:09, 57.96it/s]
 47%|████▋     | 466/1000 [00:08<00:09, 58.04it/s]
 47%|████▋     | 472/1000 [00:08<00:09, 58.02it/s]
 48%|████▊     | 478/1000 [00:09<00:08, 58.09it/s]
 48%|████▊     | 484/1000 [00:09<00:08, 57.93it/s]
 49%|████▉     | 490/1000 [00:09<00:08, 57.91it/s]
 50%|████▉     | 496/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 502/1000 [00:09<00:08, 57.94it/s]
 51%|█████     | 508/1000 [00:09<00:08, 57.87it/s]
 51%|█████▏    | 514/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 520/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 526/1000 [00:09<00:08, 58.01it/s]
 53%|█████▎    | 532/1000 [00:10<00:08, 57.97it/s]
 54%|█████▍    | 538/1000 [00:10<00:07, 58.03it/s]
 54%|█████▍    | 544/1000 [00:10<00:07, 58.10it/s]
 55%|█████▌    | 550/1000 [00:10<00:07, 57.98it/s]
 56%|█████▌    | 556/1000 [00:10<00:07, 57.95it/s]
 56%|█████▌    | 562/1000 [00:10<00:07, 58.00it/s]
 57%|█████▋    | 568/1000 [00:10<00:07, 57.92it/s]
 57%|█████▋    | 574/1000 [00:10<00:07, 57.85it/s]
 58%|█████▊    | 580/1000 [00:10<00:07, 58.01it/s]
 59%|█████▊    | 586/1000 [00:10<00:07, 57.99it/s]
 59%|█████▉    | 592/1000 [00:11<00:07, 57.92it/s]
 60%|█████▉    | 598/1000 [00:11<00:06, 57.98it/s]
 60%|██████    | 604/1000 [00:11<00:06, 57.89it/s]
 61%|██████    | 610/1000 [00:11<00:06, 58.02it/s]
 62%|██████▏   | 616/1000 [00:11<00:06, 58.01it/s]
 62%|██████▏   | 622/1000 [00:11<00:06, 58.07it/s]
 63%|██████▎   | 628/1000 [00:11<00:06, 58.14it/s]
 63%|██████▎   | 634/1000 [00:11<00:06, 57.99it/s]
 64%|██████▍   | 640/1000 [00:11<00:06, 57.93it/s]
 65%|██████▍   | 646/1000 [00:11<00:06, 58.06it/s]
 65%|██████▌   | 652/1000 [00:12<00:05, 58.06it/s]
 66%|██████▌   | 658/1000 [00:12<00:05, 58.07it/s]
 66%|██████▋   | 664/1000 [00:12<00:05, 58.08it/s]
 67%|██████▋   | 670/1000 [00:12<00:05, 58.07it/s]
 68%|██████▊   | 676/1000 [00:12<00:05, 58.11it/s]
 68%|██████▊   | 682/1000 [00:12<00:05, 58.08it/s]
 69%|██████▉   | 688/1000 [00:12<00:05, 58.15it/s]
 69%|██████▉   | 694/1000 [00:12<00:05, 58.01it/s]
 70%|███████   | 700/1000 [00:12<00:05, 58.10it/s]
 71%|███████   | 706/1000 [00:13<00:05, 58.05it/s]
 71%|███████   | 712/1000 [00:13<00:04, 58.10it/s]
 72%|███████▏  | 718/1000 [00:13<00:04, 58.06it/s]
 72%|███████▏  | 724/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 730/1000 [00:13<00:04, 58.04it/s]
 74%|███████▎  | 736/1000 [00:13<00:04, 58.09it/s]
 74%|███████▍  | 742/1000 [00:13<00:04, 58.00it/s]
 75%|███████▍  | 748/1000 [00:13<00:04, 58.07it/s]
 75%|███████▌  | 754/1000 [00:13<00:04, 58.09it/s]
 76%|███████▌  | 760/1000 [00:13<00:04, 57.94it/s]
 77%|███████▋  | 766/1000 [00:14<00:04, 58.01it/s]
 77%|███████▋  | 772/1000 [00:14<00:03, 58.03it/s]
 78%|███████▊  | 778/1000 [00:14<00:03, 58.08it/s]
 78%|███████▊  | 784/1000 [00:14<00:03, 58.08it/s]
 79%|███████▉  | 790/1000 [00:14<00:03, 57.93it/s]
 80%|███████▉  | 796/1000 [00:14<00:03, 57.97it/s]
 80%|████████  | 802/1000 [00:14<00:03, 57.91it/s]
 81%|████████  | 808/1000 [00:14<00:03, 57.90it/s]
 81%|████████▏ | 814/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 820/1000 [00:14<00:03, 58.01it/s]
 83%|████████▎ | 826/1000 [00:15<00:03, 57.95it/s]
 83%|████████▎ | 832/1000 [00:15<00:02, 58.11it/s]
 84%|████████▍ | 838/1000 [00:15<00:02, 58.19it/s]
 84%|████████▍ | 844/1000 [00:15<00:02, 58.07it/s]
 85%|████████▌ | 850/1000 [00:15<00:02, 58.11it/s]
 86%|████████▌ | 856/1000 [00:15<00:02, 58.07it/s]
 86%|████████▌ | 862/1000 [00:15<00:02, 58.04it/s]
 87%|████████▋ | 868/1000 [00:15<00:02, 57.99it/s]
 87%|████████▋ | 874/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 880/1000 [00:16<00:02, 58.05it/s]
 89%|████████▊ | 886/1000 [00:16<00:01, 57.92it/s]
 89%|████████▉ | 892/1000 [00:16<00:01, 57.99it/s]
 90%|████████▉ | 898/1000 [00:16<00:01, 58.04it/s]
 90%|█████████ | 904/1000 [00:16<00:01, 58.04it/s]
 91%|█████████ | 910/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 916/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 922/1000 [00:16<00:01, 57.95it/s]
 93%|█████████▎| 928/1000 [00:16<00:01, 58.11it/s]
 93%|█████████▎| 934/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▍| 940/1000 [00:17<00:01, 58.14it/s]
 95%|█████████▍| 946/1000 [00:17<00:00, 58.06it/s]
 95%|█████████▌| 952/1000 [00:17<00:00, 58.09it/s]
 96%|█████████▌| 958/1000 [00:17<00:00, 58.05it/s]
 96%|█████████▋| 964/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 970/1000 [00:17<00:00, 58.20it/s]
 98%|█████████▊| 976/1000 [00:17<00:00, 58.26it/s]
 98%|█████████▊| 982/1000 [00:17<00:00, 58.26it/s]
 99%|█████████▉| 988/1000 [00:17<00:00, 58.16it/s]
 99%|█████████▉| 994/1000 [00:17<00:00, 58.12it/s]
100%|██████████| 1000/1000 [00:18<00:00, 57.94it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.28it/s]
+2022-12-21,09:55:03 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:16, 114.32it/s]
  1%|          | 512/50176 [00:04<06:27, 128.28it/s]
  2%|▏         | 768/50176 [00:05<06:01, 136.74it/s]
  2%|▏         | 1024/50176 [00:07<05:51, 139.91it/s]
  3%|▎         | 1280/50176 [00:09<05:48, 140.22it/s]
  3%|▎         | 1536/50176 [00:11<06:01, 134.62it/s]
  4%|▎         | 1792/50176 [00:13<05:56, 135.67it/s]
  4%|▍         | 2048/50176 [00:15<05:48, 138.26it/s]
  5%|▍         | 2304/50176 [00:16<05:46, 138.26it/s]
  5%|▌         | 2560/50176 [00:18<05:38, 140.69it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 139.74it/s]
  6%|▌         | 3072/50176 [00:22<05:31, 142.10it/s]
  7%|▋         | 3328/50176 [00:24<05:27, 143.14it/s]
  7%|▋         | 3584/50176 [00:25<05:20, 145.30it/s]
  8%|▊         | 3840/50176 [00:27<05:31, 139.86it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.15it/s]
  9%|▊         | 4352/50176 [00:31<05:22, 142.08it/s]
  9%|▉         | 4608/50176 [00:33<05:18, 142.86it/s]
 10%|▉         | 4864/50176 [00:34<05:12, 144.81it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.33it/s]
 11%|█         | 5376/50176 [00:38<05:04, 147.11it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.43it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 148.08it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 136.15it/s]
 13%|█▎        | 6400/50176 [00:45<05:12, 140.27it/s]
 13%|█▎        | 6656/50176 [00:47<05:05, 142.65it/s]
 14%|█▍        | 6912/50176 [00:49<05:03, 142.45it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 143.90it/s]
 15%|█▍        | 7424/50176 [00:52<04:51, 146.49it/s]
 15%|█▌        | 7680/50176 [00:54<04:51, 145.80it/s]
 16%|█▌        | 7936/50176 [00:55<04:42, 149.28it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.38it/s]
 17%|█▋        | 8448/50176 [00:59<04:38, 149.75it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.58it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.38it/s]
 18%|█▊        | 9216/50176 [01:04<04:35, 148.53it/s]
 19%|█▉        | 9472/50176 [01:06<04:32, 149.29it/s]
 19%|█▉        | 9728/50176 [01:08<04:36, 146.34it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.24it/s]
 20%|██        | 10240/50176 [01:11<04:26, 150.11it/s]
 21%|██        | 10496/50176 [01:12<04:21, 151.70it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.27it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.34it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.14it/s]
 23%|██▎       | 11520/50176 [01:19<04:18, 149.62it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 149.02it/s]
 24%|██▍       | 12032/50176 [01:23<04:15, 149.07it/s]
 24%|██▍       | 12288/50176 [01:25<04:12, 149.80it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.31it/s]
 26%|██▌       | 12800/50176 [01:28<04:07, 150.80it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.55it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.85it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 145.07it/s]
 28%|██▊       | 13824/50176 [01:35<04:15, 142.21it/s]
 28%|██▊       | 14080/50176 [01:37<04:12, 142.90it/s]
 29%|██▊       | 14336/50176 [01:39<04:16, 139.75it/s]
 29%|██▉       | 14592/50176 [01:40<04:04, 145.32it/s]
 30%|██▉       | 14848/50176 [01:43<04:17, 137.12it/s]
 30%|███       | 15104/50176 [01:44<04:18, 135.62it/s]
 31%|███       | 15360/50176 [01:46<04:09, 139.33it/s]
 31%|███       | 15616/50176 [01:48<04:04, 141.38it/s]
 32%|███▏      | 15872/50176 [01:50<04:08, 137.82it/s]
 32%|███▏      | 16128/50176 [01:52<04:02, 140.44it/s]
 33%|███▎      | 16384/50176 [01:53<04:00, 140.58it/s]
 33%|███▎      | 16640/50176 [01:56<04:07, 135.47it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.74it/s]
 34%|███▍      | 17152/50176 [01:59<03:49, 144.10it/s]
 35%|███▍      | 17408/50176 [02:01<03:45, 145.26it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.32it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.18it/s]
 36%|███▌      | 18176/50176 [02:06<03:35, 148.50it/s]
 37%|███▋      | 18432/50176 [02:07<03:33, 148.77it/s]
 37%|███▋      | 18688/50176 [02:09<03:34, 147.09it/s]
 38%|███▊      | 18944/50176 [02:11<03:40, 141.70it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.97it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.40it/s]
 39%|███▉      | 19712/50176 [02:17<03:51, 131.73it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.43it/s]
 40%|████      | 20224/50176 [02:21<03:32, 140.86it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.91it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 143.90it/s]
 42%|████▏     | 20992/50176 [02:26<03:19, 146.64it/s]
 42%|████▏     | 21248/50176 [02:28<03:23, 141.94it/s]
 43%|████▎     | 21504/50176 [02:29<03:17, 145.12it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.15it/s]
 44%|████▍     | 22016/50176 [02:33<03:07, 150.40it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.81it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.48it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.37it/s]
 46%|████▌     | 23040/50176 [02:40<03:09, 143.19it/s]
 46%|████▋     | 23296/50176 [02:42<03:12, 139.47it/s]
 47%|████▋     | 23552/50176 [02:44<03:14, 136.69it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.90it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.43it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.07it/s]
 49%|████▉     | 24576/50176 [02:51<02:55, 145.50it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.78it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 149.04it/s]
 51%|█████     | 25344/50176 [02:56<02:46, 149.54it/s]
 51%|█████     | 25600/50176 [02:58<02:47, 147.06it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.09it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.07it/s]
 53%|█████▎    | 26368/50176 [03:03<02:45, 143.65it/s]
 53%|█████▎    | 26624/50176 [03:05<02:40, 146.33it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 145.94it/s]
 54%|█████▍    | 27136/50176 [03:08<02:36, 147.48it/s]
 55%|█████▍    | 27392/50176 [03:10<02:34, 147.71it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.73it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.37it/s]
 56%|█████▌    | 28160/50176 [03:15<02:33, 143.67it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.37it/s]
 57%|█████▋    | 28672/50176 [03:19<02:32, 141.04it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.12it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.34it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.29it/s]
 59%|█████▉    | 29696/50176 [03:26<02:32, 134.32it/s]
 60%|█████▉    | 29952/50176 [03:28<02:30, 134.03it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 133.02it/s]
 61%|██████    | 30464/50176 [03:32<02:20, 140.28it/s]
 61%|██████    | 30720/50176 [03:34<02:18, 140.35it/s]
 62%|██████▏   | 30976/50176 [03:35<02:14, 142.84it/s]
 62%|██████▏   | 31232/50176 [03:37<02:08, 146.88it/s]
 63%|██████▎   | 31488/50176 [03:39<02:05, 148.37it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 148.54it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 146.93it/s]
 64%|██████▍   | 32256/50176 [03:44<02:01, 147.39it/s]
 65%|██████▍   | 32512/50176 [03:46<02:04, 142.17it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 145.74it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.50it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 145.72it/s]
 67%|██████▋   | 33536/50176 [03:53<01:56, 142.70it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.46it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.51it/s]
 68%|██████▊   | 34304/50176 [03:58<01:44, 151.72it/s]
 69%|██████▉   | 34560/50176 [04:00<01:47, 144.61it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.53it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.14it/s]
 70%|███████   | 35328/50176 [04:05<01:40, 148.06it/s]
 71%|███████   | 35584/50176 [04:07<01:39, 146.36it/s]
 71%|███████▏  | 35840/50176 [04:09<01:41, 141.64it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 143.79it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.60it/s]
 73%|███████▎  | 36608/50176 [04:14<01:34, 144.33it/s]
 73%|███████▎  | 36864/50176 [04:16<01:33, 142.06it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.71it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.31it/s]
 75%|███████▌  | 37632/50176 [04:21<01:27, 144.17it/s]
 76%|███████▌  | 37888/50176 [04:23<01:25, 143.68it/s]
 76%|███████▌  | 38144/50176 [04:25<01:22, 145.80it/s]
 77%|███████▋  | 38400/50176 [04:26<01:21, 145.32it/s]
 77%|███████▋  | 38656/50176 [04:28<01:19, 144.11it/s]
 78%|███████▊  | 38912/50176 [04:30<01:18, 142.66it/s]
 78%|███████▊  | 39168/50176 [04:32<01:19, 138.34it/s]
 79%|███████▊  | 39424/50176 [04:34<01:16, 141.27it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.34it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.29it/s]
 80%|████████  | 40192/50176 [04:39<01:07, 147.24it/s]
 81%|████████  | 40448/50176 [04:41<01:09, 140.84it/s]
 81%|████████  | 40704/50176 [04:43<01:10, 135.25it/s]
 82%|████████▏ | 40960/50176 [04:45<01:05, 140.42it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.21it/s]
 83%|████████▎ | 41472/50176 [04:48<00:59, 146.06it/s]
 83%|████████▎ | 41728/50176 [04:50<00:56, 149.28it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.40it/s]
 84%|████████▍ | 42240/50176 [04:53<00:54, 145.49it/s]
 85%|████████▍ | 42496/50176 [04:55<00:51, 148.42it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.94it/s]
 86%|████████▌ | 43008/50176 [04:58<00:48, 149.07it/s]
 86%|████████▌ | 43264/50176 [05:00<00:46, 149.43it/s]
 87%|████████▋ | 43520/50176 [05:02<00:44, 148.28it/s]
 87%|████████▋ | 43776/50176 [05:06<01:03, 101.49it/s]
 88%|████████▊ | 44032/50176 [05:08<00:54, 113.60it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.29it/s]
 89%|████████▉ | 44544/50176 [05:11<00:43, 129.93it/s]
 89%|████████▉ | 44800/50176 [05:13<00:40, 133.77it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.08it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.48it/s]
 91%|█████████ | 45568/50176 [05:19<00:35, 128.29it/s]
 91%|█████████▏| 45824/50176 [05:21<00:32, 135.68it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.83it/s]
 92%|█████████▏| 46336/50176 [05:24<00:26, 145.11it/s]
 93%|█████████▎| 46592/50176 [05:26<00:24, 147.96it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.55it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 145.06it/s]
 94%|█████████▍| 47360/50176 [05:31<00:19, 146.44it/s]
 95%|█████████▍| 47616/50176 [05:33<00:17, 145.30it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.32it/s]
 96%|█████████▌| 48128/50176 [05:36<00:13, 150.22it/s]
 96%|█████████▋| 48384/50176 [05:38<00:11, 152.09it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.58it/s]
 97%|█████████▋| 48896/50176 [05:41<00:08, 154.83it/s]
 98%|█████████▊| 49152/50176 [05:43<00:06, 150.80it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.19it/s]
 99%|█████████▉| 49664/50176 [05:46<00:03, 149.80it/s]
 99%|█████████▉| 49920/50176 [05:48<00:01, 142.19it/s]
100%|██████████| 50176/50176 [05:48<00:00, 186.96it/s]
100%|██████████| 50176/50176 [05:48<00:00, 143.79it/s]
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5534	imagenet-zeroshot-val-top5: 0.8278
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4c2fdaaf76438d3be9fb5b1218dfe5d0d05f3746
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a3ae3d4bf88816e5397de2ca12fd64ed48d51e9a8f0cd4638a1ee7d2510d9748
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2cba1b8cdcdfdfa988350b484034e9081865d748
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt).
+2022-12-21,09:54:45 | INFO | Model:
+2022-12-21,09:54:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:45 | INFO | Params:
+2022-12-21,09:54:45 | INFO |   batch_size: 256
+2022-12-21,09:54:45 | INFO |   beta1: 0.9
+2022-12-21,09:54:45 | INFO |   beta2: 0.98
+2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/checkpoints
+2022-12-21,09:54:45 | INFO |   copy_codebase: False
+2022-12-21,09:54:45 | INFO |   csv_caption_key: title
+2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:45 | INFO |   csv_separator: 	
+2022-12-21,09:54:45 | INFO |   dataset_resampled: False
+2022-12-21,09:54:45 | INFO |   dataset_type: auto
+2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:45 | INFO |   debug: False
+2022-12-21,09:54:45 | INFO |   device: cuda:0
+2022-12-21,09:54:45 | INFO |   dist_backend: nccl
+2022-12-21,09:54:45 | INFO |   dist_url: env://
+2022-12-21,09:54:45 | INFO |   distributed: False
+2022-12-21,09:54:45 | INFO |   epochs: 32
+2022-12-21,09:54:45 | INFO |   eps: 1e-06
+2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:45 | INFO |   gather_with_grad: False
+2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:45 | INFO |   horovod: False
+2022-12-21,09:54:45 | INFO |   image_mean: None
+2022-12-21,09:54:45 | INFO |   image_std: None
+2022-12-21,09:54:45 | INFO |   imagenet_v2: None
+2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:45 | INFO |   local_loss: False
+2022-12-21,09:54:45 | INFO |   local_rank: 0
+2022-12-21,09:54:45 | INFO |   lock_image: False
+2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:45 | INFO |   log_level: 20
+2022-12-21,09:54:45 | INFO |   log_local: False
+2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/out.log
+2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:45 | INFO |   lr: 0.0005
+2022-12-21,09:54:45 | INFO |   model: ViT-B-16
+2022-12-21,09:54:45 | INFO |   name: epoch_19.pt
+2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt
+2022-12-21,09:54:45 | INFO |   pretrained_image: False
+2022-12-21,09:54:45 | INFO |   rank: 0
+2022-12-21,09:54:45 | INFO |   report_to: 
+2022-12-21,09:54:45 | INFO |   resume: None
+2022-12-21,09:54:45 | INFO |   save_frequency: 1
+2022-12-21,09:54:45 | INFO |   save_most_recent: False
+2022-12-21,09:54:45 | INFO |   seed: 0
+2022-12-21,09:54:45 | INFO |   skip_scheduler: False
+2022-12-21,09:54:45 | INFO |   tensorboard: False
+2022-12-21,09:54:45 | INFO |   tensorboard_path: 
+2022-12-21,09:54:45 | INFO |   torchscript: False
+2022-12-21,09:54:45 | INFO |   trace: False
+2022-12-21,09:54:45 | INFO |   train_data: None
+2022-12-21,09:54:45 | INFO |   train_num_samples: None
+2022-12-21,09:54:45 | INFO |   use_bn_sync: False
+2022-12-21,09:54:45 | INFO |   val_data: None
+2022-12-21,09:54:45 | INFO |   val_frequency: 1
+2022-12-21,09:54:45 | INFO |   val_num_samples: None
+2022-12-21,09:54:45 | INFO |   wandb: False
+2022-12-21,09:54:45 | INFO |   wandb_notes: 
+2022-12-21,09:54:45 | INFO |   warmup: 10000
+2022-12-21,09:54:45 | INFO |   wd: 0.2
+2022-12-21,09:54:45 | INFO |   workers: 1
+2022-12-21,09:54:45 | INFO |   world_size: 1
+2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:46 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:46 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:41,  1.56it/s]
  1%|          | 6/1000 [00:00<01:35, 10.36it/s]
  1%|          | 12/1000 [00:00<00:49, 19.86it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.10it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.26it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.07it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.38it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.71it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.20it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.03it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.38it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.11it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.82it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.80it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.13it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.54it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.75it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.01it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.15it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.12it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.04it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.03it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.97it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.02it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.10it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.13it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.00it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.87it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.90it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.81it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.75it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.74it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.69it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.37it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.50it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.44it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.62it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.54it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.62it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.73it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.82it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.86it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.67it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.89it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.99it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.20it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.18it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.17it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.30it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.18it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.01it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.10it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.20it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.29it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.07it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 58.07it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.96it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.03it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.72it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.81it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.92it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.09it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.18it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.19it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.21it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.23it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.12it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.19it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.25it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.90it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.13it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.09it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.08it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.11it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.03it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.09it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.98it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.06it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.20it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.14it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.26it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.21it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.09it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.06it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.03it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.16it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.13it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.16it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.07it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.19it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.27it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.10it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.05it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.19it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.13it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.26it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.20it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.30it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.19it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.29it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.27it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.25it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.27it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.22it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.24it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.20it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.29it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.37it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.19it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.31it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.26it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.29it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.30it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.10it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.11it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.18it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.15it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.24it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.15it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.20it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.14it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.19it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.16it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.18it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.17it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.03it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.14it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.10it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.11it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.25it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.26it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.31it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.44it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.05it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.10it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.95it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.90it/s]
+2022-12-21,09:55:04 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:57, 139.64it/s]
  1%|          | 512/50176 [00:03<06:00, 137.68it/s]
  2%|▏         | 768/50176 [00:05<05:46, 142.67it/s]
  2%|▏         | 1024/50176 [00:07<05:41, 143.90it/s]
  3%|▎         | 1280/50176 [00:09<05:43, 142.33it/s]
  3%|▎         | 1536/50176 [00:11<05:57, 136.16it/s]
  4%|▎         | 1792/50176 [00:12<05:54, 136.56it/s]
  4%|▍         | 2048/50176 [00:14<05:45, 139.15it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 139.02it/s]
  5%|▌         | 2560/50176 [00:18<05:37, 141.20it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 139.99it/s]
  6%|▌         | 3072/50176 [00:21<05:30, 142.53it/s]
  7%|▋         | 3328/50176 [00:23<05:26, 143.34it/s]
  7%|▋         | 3584/50176 [00:25<05:20, 145.49it/s]
  8%|▊         | 3840/50176 [00:27<05:30, 140.04it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.26it/s]
  9%|▊         | 4352/50176 [00:30<05:22, 141.96it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.55it/s]
 10%|▉         | 4864/50176 [00:34<05:12, 144.93it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.45it/s]
 11%|█         | 5376/50176 [00:37<05:04, 146.98it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.19it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 147.92it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 136.02it/s]
 13%|█▎        | 6400/50176 [00:45<05:11, 140.53it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.44it/s]
 14%|█▍        | 6912/50176 [00:48<05:03, 142.50it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 144.09it/s]
 15%|█▍        | 7424/50176 [00:52<04:51, 146.63it/s]
 15%|█▌        | 7680/50176 [00:53<04:51, 145.87it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 148.95it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.55it/s]
 17%|█▋        | 8448/50176 [00:58<04:38, 149.69it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.51it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.59it/s]
 18%|█▊        | 9216/50176 [01:04<04:35, 148.72it/s]
 19%|█▉        | 9472/50176 [01:05<04:32, 149.35it/s]
 19%|█▉        | 9728/50176 [01:07<04:37, 145.53it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.29it/s]
 20%|██        | 10240/50176 [01:10<04:26, 150.09it/s]
 21%|██        | 10496/50176 [01:12<04:21, 152.00it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.27it/s]
 22%|██▏       | 11008/50176 [01:16<04:26, 146.86it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.49it/s]
 23%|██▎       | 11520/50176 [01:19<04:19, 149.18it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 149.18it/s]
 24%|██▍       | 12032/50176 [01:23<04:14, 149.85it/s]
 24%|██▍       | 12288/50176 [01:24<04:12, 149.90it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.38it/s]
 26%|██▌       | 12800/50176 [01:28<04:08, 150.68it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.40it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.75it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 145.19it/s]
 28%|██▊       | 13824/50176 [01:35<04:15, 142.25it/s]
 28%|██▊       | 14080/50176 [01:37<04:12, 142.79it/s]
 29%|██▊       | 14336/50176 [01:38<04:16, 139.97it/s]
 29%|██▉       | 14592/50176 [01:40<04:05, 145.18it/s]
 30%|██▉       | 14848/50176 [01:42<04:17, 137.15it/s]
 30%|███       | 15104/50176 [01:44<04:18, 135.51it/s]
 31%|███       | 15360/50176 [01:46<04:10, 139.26it/s]
 31%|███       | 15616/50176 [01:48<04:04, 141.20it/s]
 32%|███▏      | 15872/50176 [01:50<04:08, 137.88it/s]
 32%|███▏      | 16128/50176 [01:51<04:02, 140.42it/s]
 33%|███▎      | 16384/50176 [01:53<04:01, 140.17it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.73it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.78it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.15it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.46it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.26it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.15it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.48it/s]
 37%|███▋      | 18432/50176 [02:07<03:33, 148.77it/s]
 37%|███▋      | 18688/50176 [02:09<03:33, 147.19it/s]
 38%|███▊      | 18944/50176 [02:11<03:40, 141.65it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.92it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.80it/s]
 39%|███▉      | 19712/50176 [02:17<03:51, 131.72it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.30it/s]
 40%|████      | 20224/50176 [02:20<03:32, 140.77it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.83it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 144.10it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.54it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 141.88it/s]
 43%|████▎     | 21504/50176 [02:29<03:17, 145.25it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.23it/s]
 44%|████▍     | 22016/50176 [02:32<03:07, 150.46it/s]
 44%|████▍     | 22272/50176 [02:34<03:04, 150.86it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.31it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.27it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.18it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.38it/s]
 47%|████▋     | 23552/50176 [02:43<03:14, 136.56it/s]
 47%|████▋     | 23808/50176 [02:45<03:06, 141.06it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.38it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.02it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.48it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.78it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 149.04it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.55it/s]
 51%|█████     | 25600/50176 [02:57<02:47, 147.03it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.30it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.20it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 143.73it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 146.18it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 146.02it/s]
 54%|█████▍    | 27136/50176 [03:08<02:36, 147.37it/s]
 55%|█████▍    | 27392/50176 [03:09<02:34, 147.84it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.81it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.26it/s]
 56%|█████▌    | 28160/50176 [03:15<02:33, 143.73it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.27it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.20it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 142.98it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.43it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.18it/s]
 59%|█████▉    | 29696/50176 [03:26<02:32, 134.47it/s]
 60%|█████▉    | 29952/50176 [03:28<02:31, 133.88it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 133.10it/s]
 61%|██████    | 30464/50176 [03:31<02:20, 140.13it/s]
 61%|██████    | 30720/50176 [03:33<02:19, 139.96it/s]
 62%|██████▏   | 30976/50176 [03:35<02:13, 143.54it/s]
 62%|██████▏   | 31232/50176 [03:37<02:09, 146.71it/s]
 63%|██████▎   | 31488/50176 [03:38<02:06, 148.24it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 147.81it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 147.09it/s]
 64%|██████▍   | 32256/50176 [03:43<02:01, 147.50it/s]
 65%|██████▍   | 32512/50176 [03:45<02:04, 142.02it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 146.03it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.42it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 145.99it/s]
 67%|██████▋   | 33536/50176 [03:52<01:56, 142.67it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.44it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.71it/s]
 68%|██████▊   | 34304/50176 [03:57<01:44, 151.64it/s]
 69%|██████▉   | 34560/50176 [03:59<01:48, 144.57it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.51it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.12it/s]
 70%|███████   | 35328/50176 [04:04<01:40, 148.21it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 146.24it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 141.61it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 143.77it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.76it/s]
 73%|███████▎  | 36608/50176 [04:13<01:34, 144.27it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 141.81it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.77it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.40it/s]
 75%|███████▌  | 37632/50176 [04:21<01:27, 144.17it/s]
 76%|███████▌  | 37888/50176 [04:22<01:25, 143.84it/s]
 76%|███████▌  | 38144/50176 [04:24<01:22, 145.94it/s]
 77%|███████▋  | 38400/50176 [04:26<01:20, 145.41it/s]
 77%|███████▋  | 38656/50176 [04:28<01:20, 144.00it/s]
 78%|███████▊  | 38912/50176 [04:30<01:18, 142.84it/s]
 78%|███████▊  | 39168/50176 [04:32<01:19, 138.21it/s]
 79%|███████▊  | 39424/50176 [04:33<01:16, 141.19it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.41it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.61it/s]
 80%|████████  | 40192/50176 [04:38<01:07, 147.06it/s]
 81%|████████  | 40448/50176 [04:40<01:09, 140.82it/s]
 81%|████████  | 40704/50176 [04:42<01:10, 135.10it/s]
 82%|████████▏ | 40960/50176 [04:44<01:05, 140.39it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.16it/s]
 83%|████████▎ | 41472/50176 [04:47<00:59, 146.67it/s]
 83%|████████▎ | 41728/50176 [04:49<00:56, 149.22it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.15it/s]
 84%|████████▍ | 42240/50176 [04:53<00:54, 145.96it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 148.11it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.90it/s]
 86%|████████▌ | 43008/50176 [04:58<00:47, 149.52it/s]
 86%|████████▌ | 43264/50176 [04:59<00:46, 149.09it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.06it/s]
 87%|████████▋ | 43776/50176 [05:06<01:03, 101.40it/s]
 88%|████████▊ | 44032/50176 [05:07<00:54, 113.47it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.35it/s]
 89%|████████▉ | 44544/50176 [05:11<00:43, 129.70it/s]
 89%|████████▉ | 44800/50176 [05:12<00:40, 134.08it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.15it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.55it/s]
 91%|█████████ | 45568/50176 [05:19<00:35, 128.35it/s]
 91%|█████████▏| 45824/50176 [05:20<00:32, 135.68it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.64it/s]
 92%|█████████▏| 46336/50176 [05:24<00:26, 144.31it/s]
 93%|█████████▎| 46592/50176 [05:25<00:24, 148.07it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.66it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 145.12it/s]
 94%|█████████▍| 47360/50176 [05:30<00:19, 146.50it/s]
 95%|█████████▍| 47616/50176 [05:32<00:17, 145.48it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.50it/s]
 96%|█████████▌| 48128/50176 [05:36<00:13, 150.31it/s]
 96%|█████████▋| 48384/50176 [05:37<00:11, 152.01it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.66it/s]
 97%|█████████▋| 48896/50176 [05:40<00:08, 154.75it/s]
 98%|█████████▊| 49152/50176 [05:42<00:06, 150.65it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.43it/s]
 99%|█████████▉| 49664/50176 [05:46<00:03, 149.85it/s]
 99%|█████████▉| 49920/50176 [05:48<00:01, 142.22it/s]
100%|██████████| 50176/50176 [05:48<00:00, 186.72it/s]
100%|██████████| 50176/50176 [05:48<00:00, 143.94it/s]
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5604	imagenet-zeroshot-val-top5: 0.8313
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c0aa3acabac30d194b7ae5f0d681448167043b6e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e4a1b6f45c6aabe21160e267380a3fd7d35248538110766c155eb035003a681f
+size 1795822369
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d24b344d9f0cbe22dc1c1cce9338c10d1d98f369
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt).
+2022-12-21,09:54:45 | INFO | Model:
+2022-12-21,09:54:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:45 | INFO | Params:
+2022-12-21,09:54:45 | INFO |   batch_size: 256
+2022-12-21,09:54:45 | INFO |   beta1: 0.9
+2022-12-21,09:54:45 | INFO |   beta2: 0.98
+2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/checkpoints
+2022-12-21,09:54:45 | INFO |   copy_codebase: False
+2022-12-21,09:54:45 | INFO |   csv_caption_key: title
+2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:45 | INFO |   csv_separator: 	
+2022-12-21,09:54:45 | INFO |   dataset_resampled: False
+2022-12-21,09:54:45 | INFO |   dataset_type: auto
+2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:45 | INFO |   debug: False
+2022-12-21,09:54:45 | INFO |   device: cuda:0
+2022-12-21,09:54:45 | INFO |   dist_backend: nccl
+2022-12-21,09:54:45 | INFO |   dist_url: env://
+2022-12-21,09:54:45 | INFO |   distributed: False
+2022-12-21,09:54:45 | INFO |   epochs: 32
+2022-12-21,09:54:45 | INFO |   eps: 1e-06
+2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:45 | INFO |   gather_with_grad: False
+2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:45 | INFO |   horovod: False
+2022-12-21,09:54:45 | INFO |   image_mean: None
+2022-12-21,09:54:45 | INFO |   image_std: None
+2022-12-21,09:54:45 | INFO |   imagenet_v2: None
+2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:45 | INFO |   local_loss: False
+2022-12-21,09:54:45 | INFO |   local_rank: 0
+2022-12-21,09:54:45 | INFO |   lock_image: False
+2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:45 | INFO |   log_level: 20
+2022-12-21,09:54:45 | INFO |   log_local: False
+2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/out.log
+2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:45 | INFO |   lr: 0.0005
+2022-12-21,09:54:45 | INFO |   model: ViT-B-16
+2022-12-21,09:54:45 | INFO |   name: epoch_2.pt
+2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt
+2022-12-21,09:54:45 | INFO |   pretrained_image: False
+2022-12-21,09:54:45 | INFO |   rank: 0
+2022-12-21,09:54:45 | INFO |   report_to: 
+2022-12-21,09:54:45 | INFO |   resume: None
+2022-12-21,09:54:45 | INFO |   save_frequency: 1
+2022-12-21,09:54:45 | INFO |   save_most_recent: False
+2022-12-21,09:54:45 | INFO |   seed: 0
+2022-12-21,09:54:45 | INFO |   skip_scheduler: False
+2022-12-21,09:54:45 | INFO |   tensorboard: False
+2022-12-21,09:54:45 | INFO |   tensorboard_path: 
+2022-12-21,09:54:45 | INFO |   torchscript: False
+2022-12-21,09:54:45 | INFO |   trace: False
+2022-12-21,09:54:45 | INFO |   train_data: None
+2022-12-21,09:54:45 | INFO |   train_num_samples: None
+2022-12-21,09:54:45 | INFO |   use_bn_sync: False
+2022-12-21,09:54:45 | INFO |   val_data: None
+2022-12-21,09:54:45 | INFO |   val_frequency: 1
+2022-12-21,09:54:45 | INFO |   val_num_samples: None
+2022-12-21,09:54:45 | INFO |   wandb: False
+2022-12-21,09:54:45 | INFO |   wandb_notes: 
+2022-12-21,09:54:45 | INFO |   warmup: 10000
+2022-12-21,09:54:45 | INFO |   wd: 0.2
+2022-12-21,09:54:45 | INFO |   workers: 1
+2022-12-21,09:54:45 | INFO |   world_size: 1
+2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:45 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:45 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:00,  1.39it/s]
  1%|          | 6/1000 [00:00<01:46,  9.35it/s]
  1%|          | 12/1000 [00:00<00:53, 18.46it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.52it/s]
  2%|▏         | 24/1000 [00:01<00:28, 33.79it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.78it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.44it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.02it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.56it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.54it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.92it/s]
  7%|▋         | 66/1000 [00:01<00:16, 54.99it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.74it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.13it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.71it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.07it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.48it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.68it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.94it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.08it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.07it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.00it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.05it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.99it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.03it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.11it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.11it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.97it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.81it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.79it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.70it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.65it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.59it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.42it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.31it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.41it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.36it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.59it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.60it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.54it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.59it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.68it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.78it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.80it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.53it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.59it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.77it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.88it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.10it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.10it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.07it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.19it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.09it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.93it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.02it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.13it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.22it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.02it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.99it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.08it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.17it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.75it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.79it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.90it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.03it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.10it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.12it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.03it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.14it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.12it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.08it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.21it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.12it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.10it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.10it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.00it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.05it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.96it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.03it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.07it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.16it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.11it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.17it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.18it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.09it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.11it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.14it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.99it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.09it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.02it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.17it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.16it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.27it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.10it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.01it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.17it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.13it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.22it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.14it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.17it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.18it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.11it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.02it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.13it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.13it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.10it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.10it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.19it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.28it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.11it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.21it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.15it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.17it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.21it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.99it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.98it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.03it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.00it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.98it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.01it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.12it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.30it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.22it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.24it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.20it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.17it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.05it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.08it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.09it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.23it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.23it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.23it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.26it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.25it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.34it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.37it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.36it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.22it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.71it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.62it/s]
+2022-12-21,09:55:03 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:29, 128.16it/s]
  1%|          | 512/50176 [00:03<06:09, 134.26it/s]
  2%|▏         | 768/50176 [00:05<05:53, 139.93it/s]
  2%|▏         | 1024/50176 [00:07<05:44, 142.66it/s]
  3%|▎         | 1280/50176 [00:09<05:45, 141.67it/s]
  3%|▎         | 1536/50176 [00:11<05:58, 135.79it/s]
  4%|▎         | 1792/50176 [00:13<05:56, 135.58it/s]
  4%|▍         | 2048/50176 [00:14<05:46, 138.98it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 138.79it/s]
  5%|▌         | 2560/50176 [00:18<05:37, 140.95it/s]
  6%|▌         | 2816/50176 [00:20<05:39, 139.66it/s]
  6%|▌         | 3072/50176 [00:22<05:30, 142.63it/s]
  7%|▋         | 3328/50176 [00:23<05:26, 143.39it/s]
  7%|▋         | 3584/50176 [00:25<05:20, 145.45it/s]
  8%|▊         | 3840/50176 [00:27<05:31, 139.98it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.42it/s]
  9%|▊         | 4352/50176 [00:31<05:22, 142.03it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.51it/s]
 10%|▉         | 4864/50176 [00:34<05:13, 144.40it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.59it/s]
 11%|█         | 5376/50176 [00:37<05:05, 146.62it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.38it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 147.69it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 136.31it/s]
 13%|█▎        | 6400/50176 [00:45<05:11, 140.66it/s]
 13%|█▎        | 6656/50176 [00:47<05:06, 141.93it/s]
 14%|█▍        | 6912/50176 [00:48<05:02, 142.81it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 144.20it/s]
 15%|█▍        | 7424/50176 [00:52<04:51, 146.71it/s]
 15%|█▌        | 7680/50176 [00:53<04:51, 145.94it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 149.00it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.52it/s]
 17%|█▋        | 8448/50176 [00:59<04:38, 149.78it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.55it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.45it/s]
 18%|█▊        | 9216/50176 [01:04<04:36, 147.91it/s]
 19%|█▉        | 9472/50176 [01:05<04:32, 149.62it/s]
 19%|█▉        | 9728/50176 [01:07<04:36, 146.44it/s]
 20%|█▉        | 9984/50176 [01:09<04:30, 148.33it/s]
 20%|██        | 10240/50176 [01:11<04:26, 149.95it/s]
 21%|██        | 10496/50176 [01:12<04:21, 151.84it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.37it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.56it/s]
 22%|██▏       | 11264/50176 [01:18<04:23, 147.61it/s]
 23%|██▎       | 11520/50176 [01:19<04:17, 149.89it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 149.02it/s]
 24%|██▍       | 12032/50176 [01:23<04:14, 149.78it/s]
 24%|██▍       | 12288/50176 [01:24<04:12, 149.80it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.25it/s]
 26%|██▌       | 12800/50176 [01:28<04:07, 150.85it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.36it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.77it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 144.86it/s]
 28%|██▊       | 13824/50176 [01:35<04:14, 142.57it/s]
 28%|██▊       | 14080/50176 [01:37<04:12, 142.91it/s]
 29%|██▊       | 14336/50176 [01:39<04:15, 140.35it/s]
 29%|██▉       | 14592/50176 [01:40<04:05, 145.18it/s]
 30%|██▉       | 14848/50176 [01:42<04:18, 136.83it/s]
 30%|███       | 15104/50176 [01:44<04:17, 136.01it/s]
 31%|███       | 15360/50176 [01:46<04:10, 139.03it/s]
 31%|███       | 15616/50176 [01:48<04:04, 141.59it/s]
 32%|███▏      | 15872/50176 [01:50<04:09, 137.72it/s]
 32%|███▏      | 16128/50176 [01:51<04:02, 140.22it/s]
 33%|███▎      | 16384/50176 [01:53<04:00, 140.49it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.27it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.63it/s]
 34%|███▍      | 17152/50176 [01:59<03:49, 144.08it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.24it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.28it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.20it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.51it/s]
 37%|███▋      | 18432/50176 [02:07<03:34, 148.10it/s]
 37%|███▋      | 18688/50176 [02:09<03:32, 147.92it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.28it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.81it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.35it/s]
 39%|███▉      | 19712/50176 [02:17<03:52, 131.17it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.51it/s]
 40%|████      | 20224/50176 [02:20<03:32, 140.81it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.95it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 143.95it/s]
 42%|████▏     | 20992/50176 [02:26<03:18, 146.70it/s]
 42%|████▏     | 21248/50176 [02:28<03:24, 141.55it/s]
 43%|████▎     | 21504/50176 [02:29<03:16, 145.98it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.09it/s]
 44%|████▍     | 22016/50176 [02:32<03:07, 150.36it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.70it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.38it/s]
 45%|████▌     | 22784/50176 [02:38<03:09, 144.77it/s]
 46%|████▌     | 23040/50176 [02:40<03:09, 143.46it/s]
 46%|████▋     | 23296/50176 [02:42<03:11, 140.13it/s]
 47%|████▋     | 23552/50176 [02:44<03:15, 136.50it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.90it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.35it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 142.99it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.48it/s]
 49%|████▉     | 24832/50176 [02:52<02:51, 148.07it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 149.20it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.68it/s]
 51%|█████     | 25600/50176 [02:57<02:47, 146.59it/s]
 52%|█████▏    | 25856/50176 [02:59<02:41, 151.01it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.54it/s]
 53%|█████▎    | 26368/50176 [03:03<02:46, 143.33it/s]
 53%|█████▎    | 26624/50176 [03:04<02:40, 146.52it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 146.15it/s]
 54%|█████▍    | 27136/50176 [03:08<02:36, 147.59it/s]
 55%|█████▍    | 27392/50176 [03:09<02:34, 147.93it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.85it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.40it/s]
 56%|█████▌    | 28160/50176 [03:15<02:33, 143.79it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.36it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.06it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.03it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.33it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.16it/s]
 59%|█████▉    | 29696/50176 [03:26<02:32, 134.37it/s]
 60%|█████▉    | 29952/50176 [03:28<02:30, 134.48it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 132.56it/s]
 61%|██████    | 30464/50176 [03:31<02:20, 140.31it/s]
 61%|██████    | 30720/50176 [03:33<02:18, 140.55it/s]
 62%|██████▏   | 30976/50176 [03:35<02:13, 143.40it/s]
 62%|██████▏   | 31232/50176 [03:37<02:09, 146.60it/s]
 63%|██████▎   | 31488/50176 [03:38<02:06, 148.13it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 148.46it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 146.86it/s]
 64%|██████▍   | 32256/50176 [03:44<02:01, 147.32it/s]
 65%|██████▍   | 32512/50176 [03:45<02:03, 142.50it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 145.62it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.35it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 145.81it/s]
 67%|██████▋   | 33536/50176 [03:53<01:56, 142.32it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.47it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.63it/s]
 68%|██████▊   | 34304/50176 [03:57<01:44, 151.68it/s]
 69%|██████▉   | 34560/50176 [03:59<01:47, 145.12it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.15it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.14it/s]
 70%|███████   | 35328/50176 [04:05<01:40, 148.03it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 146.23it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 140.95it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 144.02it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.44it/s]
 73%|███████▎  | 36608/50176 [04:14<01:33, 144.54it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 142.61it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.70it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.58it/s]
 75%|███████▌  | 37632/50176 [04:21<01:27, 144.07it/s]
 76%|███████▌  | 37888/50176 [04:23<01:25, 143.61it/s]
 76%|███████▌  | 38144/50176 [04:24<01:22, 145.20it/s]
 77%|███████▋  | 38400/50176 [04:26<01:20, 145.54it/s]
 77%|███████▋  | 38656/50176 [04:28<01:19, 144.22it/s]
 78%|███████▊  | 38912/50176 [04:30<01:18, 142.77it/s]
 78%|███████▊  | 39168/50176 [04:32<01:19, 139.02it/s]
 79%|███████▊  | 39424/50176 [04:33<01:16, 140.55it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.39it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.34it/s]
 80%|████████  | 40192/50176 [04:39<01:07, 147.22it/s]
 81%|████████  | 40448/50176 [04:41<01:08, 140.99it/s]
 81%|████████  | 40704/50176 [04:43<01:10, 135.13it/s]
 82%|████████▏ | 40960/50176 [04:44<01:05, 140.47it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.20it/s]
 83%|████████▎ | 41472/50176 [04:48<00:59, 146.67it/s]
 83%|████████▎ | 41728/50176 [04:49<00:56, 148.58it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.34it/s]
 84%|████████▍ | 42240/50176 [04:53<00:54, 146.14it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 148.18it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.91it/s]
 86%|████████▌ | 43008/50176 [04:58<00:47, 149.50it/s]
 86%|████████▌ | 43264/50176 [05:00<00:46, 149.17it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.11it/s]
 87%|████████▋ | 43776/50176 [05:06<01:03, 101.54it/s]
 88%|████████▊ | 44032/50176 [05:07<00:54, 113.63it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.51it/s]
 89%|████████▉ | 44544/50176 [05:11<00:43, 129.94it/s]
 89%|████████▉ | 44800/50176 [05:13<00:40, 134.12it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.30it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.24it/s]
 91%|█████████ | 45568/50176 [05:19<00:35, 128.11it/s]
 91%|█████████▏| 45824/50176 [05:20<00:32, 135.59it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.58it/s]
 92%|█████████▏| 46336/50176 [05:24<00:26, 144.96it/s]
 93%|█████████▎| 46592/50176 [05:25<00:24, 147.88it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.58it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 145.11it/s]
 94%|█████████▍| 47360/50176 [05:31<00:19, 146.39it/s]
 95%|█████████▍| 47616/50176 [05:32<00:17, 145.33it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.21it/s]
 96%|█████████▌| 48128/50176 [05:36<00:13, 150.23it/s]
 96%|█████████▋| 48384/50176 [05:37<00:11, 152.00it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.64it/s]
 97%|█████████▋| 48896/50176 [05:41<00:08, 154.76it/s]
 98%|█████████▊| 49152/50176 [05:42<00:06, 150.73it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.25it/s]
 99%|█████████▉| 49664/50176 [05:46<00:03, 149.88it/s]
 99%|█████████▉| 49920/50176 [05:48<00:01, 142.13it/s]
100%|██████████| 50176/50176 [05:48<00:00, 186.34it/s]
100%|██████████| 50176/50176 [05:48<00:00, 143.88it/s]
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.2664	imagenet-zeroshot-val-top5: 0.5124
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt
new file mode 100644
index 0000000000000000000000000000000000000000..42cd2af7bb381cffee286bf9c378481c10f31e24
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cc2b5a62dff040e778a5ff122c653e3a2c29ce5981525dd9af398af68b4b65b
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4937862f78dd26eb6641a7f4236f1ecd229b1c32
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt).
+2022-12-21,10:01:05 | INFO | Model:
+2022-12-21,10:01:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:05 | INFO | Params:
+2022-12-21,10:01:05 | INFO |   batch_size: 256
+2022-12-21,10:01:05 | INFO |   beta1: 0.9
+2022-12-21,10:01:05 | INFO |   beta2: 0.98
+2022-12-21,10:01:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/checkpoints
+2022-12-21,10:01:05 | INFO |   copy_codebase: False
+2022-12-21,10:01:05 | INFO |   csv_caption_key: title
+2022-12-21,10:01:05 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:05 | INFO |   csv_separator: 	
+2022-12-21,10:01:05 | INFO |   dataset_resampled: False
+2022-12-21,10:01:05 | INFO |   dataset_type: auto
+2022-12-21,10:01:05 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:05 | INFO |   debug: False
+2022-12-21,10:01:05 | INFO |   device: cuda:0
+2022-12-21,10:01:05 | INFO |   dist_backend: nccl
+2022-12-21,10:01:05 | INFO |   dist_url: env://
+2022-12-21,10:01:05 | INFO |   distributed: False
+2022-12-21,10:01:05 | INFO |   epochs: 32
+2022-12-21,10:01:05 | INFO |   eps: 1e-06
+2022-12-21,10:01:05 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:05 | INFO |   gather_with_grad: False
+2022-12-21,10:01:05 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:05 | INFO |   horovod: False
+2022-12-21,10:01:05 | INFO |   image_mean: None
+2022-12-21,10:01:05 | INFO |   image_std: None
+2022-12-21,10:01:05 | INFO |   imagenet_v2: None
+2022-12-21,10:01:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:05 | INFO |   local_loss: False
+2022-12-21,10:01:05 | INFO |   local_rank: 0
+2022-12-21,10:01:05 | INFO |   lock_image: False
+2022-12-21,10:01:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:05 | INFO |   log_level: 20
+2022-12-21,10:01:05 | INFO |   log_local: False
+2022-12-21,10:01:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/out.log
+2022-12-21,10:01:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:05 | INFO |   lr: 0.0005
+2022-12-21,10:01:05 | INFO |   model: ViT-B-16
+2022-12-21,10:01:05 | INFO |   name: epoch_20.pt
+2022-12-21,10:01:05 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:05 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:05 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt
+2022-12-21,10:01:05 | INFO |   pretrained_image: False
+2022-12-21,10:01:05 | INFO |   rank: 0
+2022-12-21,10:01:05 | INFO |   report_to: 
+2022-12-21,10:01:05 | INFO |   resume: None
+2022-12-21,10:01:05 | INFO |   save_frequency: 1
+2022-12-21,10:01:05 | INFO |   save_most_recent: False
+2022-12-21,10:01:05 | INFO |   seed: 0
+2022-12-21,10:01:05 | INFO |   skip_scheduler: False
+2022-12-21,10:01:05 | INFO |   tensorboard: False
+2022-12-21,10:01:05 | INFO |   tensorboard_path: 
+2022-12-21,10:01:05 | INFO |   torchscript: False
+2022-12-21,10:01:05 | INFO |   trace: False
+2022-12-21,10:01:05 | INFO |   train_data: None
+2022-12-21,10:01:05 | INFO |   train_num_samples: None
+2022-12-21,10:01:05 | INFO |   use_bn_sync: False
+2022-12-21,10:01:05 | INFO |   val_data: None
+2022-12-21,10:01:05 | INFO |   val_frequency: 1
+2022-12-21,10:01:05 | INFO |   val_num_samples: None
+2022-12-21,10:01:05 | INFO |   wandb: False
+2022-12-21,10:01:05 | INFO |   wandb_notes: 
+2022-12-21,10:01:05 | INFO |   warmup: 10000
+2022-12-21,10:01:05 | INFO |   wd: 0.2
+2022-12-21,10:01:05 | INFO |   workers: 1
+2022-12-21,10:01:05 | INFO |   world_size: 1
+2022-12-21,10:01:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:05 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<15:32,  1.07it/s]
  1%|          | 6/1000 [00:01<02:12,  7.52it/s]
  1%|          | 11/1000 [00:01<01:09, 14.19it/s]
  2%|▏         | 17/1000 [00:01<00:44, 22.14it/s]
  2%|▏         | 23/1000 [00:01<00:33, 29.27it/s]
  3%|▎         | 29/1000 [00:01<00:27, 35.39it/s]
  4%|▎         | 35/1000 [00:01<00:24, 39.83it/s]
  4%|▍         | 41/1000 [00:01<00:21, 44.30it/s]
  5%|▍         | 47/1000 [00:01<00:19, 47.75it/s]
  5%|▌         | 53/1000 [00:01<00:18, 49.91it/s]
  6%|▌         | 59/1000 [00:02<00:18, 51.28it/s]
  6%|▋         | 65/1000 [00:02<00:17, 52.20it/s]
  7%|▋         | 71/1000 [00:02<00:17, 52.85it/s]
  8%|▊         | 77/1000 [00:02<00:17, 53.42it/s]
  8%|▊         | 83/1000 [00:02<00:17, 53.89it/s]
  9%|▉         | 89/1000 [00:02<00:16, 55.18it/s]
 10%|▉         | 95/1000 [00:02<00:16, 56.05it/s]
 10%|█         | 101/1000 [00:02<00:15, 56.66it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.23it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.55it/s]
 12%|█▏        | 119/1000 [00:03<00:15, 57.68it/s]
 12%|█▎        | 125/1000 [00:03<00:15, 57.75it/s]
 13%|█▎        | 131/1000 [00:03<00:15, 57.85it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.83it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.81it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.87it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.96it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.88it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.54it/s]
 17%|█▋        | 173/1000 [00:04<00:14, 57.60it/s]
 18%|█▊        | 179/1000 [00:04<00:14, 57.66it/s]
 18%|█▊        | 185/1000 [00:04<00:14, 57.56it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.59it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.56it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.46it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.39it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.41it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.59it/s]
 23%|██▎       | 233/1000 [00:05<00:13, 57.65it/s]
 24%|██▍       | 239/1000 [00:05<00:13, 57.50it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.53it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.61it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.79it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.77it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.54it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.77it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.87it/s]
 29%|██▉       | 293/1000 [00:06<00:12, 58.10it/s]
 30%|██▉       | 299/1000 [00:06<00:12, 58.07it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.08it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.18it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.16it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.12it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.99it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.03it/s]
 35%|███▍      | 347/1000 [00:07<00:11, 58.15it/s]
 35%|███▌      | 353/1000 [00:07<00:11, 57.87it/s]
 36%|███▌      | 359/1000 [00:07<00:11, 58.00it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 57.95it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.94it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.74it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.71it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.94it/s]
 41%|████      | 407/1000 [00:08<00:10, 58.04it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 58.08it/s]
 42%|████▏     | 419/1000 [00:08<00:09, 58.12it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.06it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.20it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.20it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.00it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.02it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.04it/s]
 47%|████▋     | 467/1000 [00:09<00:09, 58.02it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 58.16it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.06it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.04it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.99it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.00it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.90it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.96it/s]
 52%|█████▏    | 521/1000 [00:10<00:08, 58.00it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 58.11it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 58.02it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.12it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.05it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.03it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.11it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.01it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.93it/s]
 58%|█████▊    | 581/1000 [00:11<00:07, 58.08it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 58.00it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.95it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.07it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.99it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.05it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.10it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.14it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.99it/s]
 64%|██████▍   | 641/1000 [00:12<00:06, 57.87it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 58.09it/s]
 65%|██████▌   | 653/1000 [00:12<00:05, 58.03it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.11it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.12it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.13it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.15it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.19it/s]
 70%|██████▉   | 695/1000 [00:13<00:05, 58.06it/s]
 70%|███████   | 701/1000 [00:13<00:05, 58.13it/s]
 71%|███████   | 707/1000 [00:13<00:05, 58.14it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.13it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.14it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.09it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.06it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.09it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.04it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.12it/s]
 76%|███████▌  | 755/1000 [00:14<00:04, 58.25it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 58.06it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 58.12it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.10it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.15it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.18it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.98it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.98it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.01it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.00it/s]
 82%|████████▏ | 815/1000 [00:15<00:03, 57.97it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 58.03it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 58.04it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.22it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.29it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.15it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.20it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.12it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.15it/s]
 87%|████████▋ | 869/1000 [00:16<00:02, 58.03it/s]
 88%|████████▊ | 875/1000 [00:16<00:02, 58.04it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 58.09it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.96it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.06it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.11it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.09it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.06it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.95it/s]
 93%|█████████▎| 929/1000 [00:17<00:01, 58.08it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 58.17it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 58.16it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.17it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.20it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.16it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.28it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.30it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.32it/s]
 99%|█████████▉| 989/1000 [00:18<00:00, 58.25it/s]
100%|█████████▉| 995/1000 [00:18<00:00, 58.12it/s]
100%|██████████| 1000/1000 [00:18<00:00, 54.76it/s]
+2022-12-21,10:01:23 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:26, 111.74it/s]
  1%|          | 512/50176 [00:04<06:33, 126.35it/s]
  2%|▏         | 768/50176 [00:06<06:25, 128.24it/s]
  2%|▏         | 1024/50176 [00:07<06:08, 133.46it/s]
  3%|▎         | 1280/50176 [00:09<05:55, 137.49it/s]
  3%|▎         | 1536/50176 [00:11<06:03, 133.74it/s]
  4%|▎         | 1792/50176 [00:13<05:56, 135.68it/s]
  4%|▍         | 2048/50176 [00:15<05:47, 138.54it/s]
  5%|▍         | 2304/50176 [00:17<05:43, 139.31it/s]
  5%|▌         | 2560/50176 [00:18<05:37, 141.22it/s]
  6%|▌         | 2816/50176 [00:20<05:37, 140.52it/s]
  6%|▌         | 3072/50176 [00:22<05:28, 143.30it/s]
  7%|▋         | 3328/50176 [00:24<05:24, 144.37it/s]
  7%|▋         | 3584/50176 [00:25<05:17, 146.88it/s]
  8%|▊         | 3840/50176 [00:27<05:26, 141.86it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.32it/s]
  9%|▊         | 4352/50176 [00:31<05:19, 143.37it/s]
  9%|▉         | 4608/50176 [00:33<05:15, 144.58it/s]
 10%|▉         | 4864/50176 [00:34<05:09, 146.58it/s]
 10%|█         | 5120/50176 [00:36<05:05, 147.56it/s]
 11%|█         | 5376/50176 [00:38<05:02, 148.10it/s]
 11%|█         | 5632/50176 [00:39<04:57, 149.73it/s]
 12%|█▏        | 5888/50176 [00:41<04:55, 149.80it/s]
 12%|█▏        | 6144/50176 [00:43<05:20, 137.49it/s]
 13%|█▎        | 6400/50176 [00:45<05:10, 140.93it/s]
 13%|█▎        | 6656/50176 [00:47<05:03, 143.29it/s]
 14%|█▍        | 6912/50176 [00:48<05:01, 143.68it/s]
 14%|█▍        | 7168/50176 [00:50<04:56, 145.03it/s]
 15%|█▍        | 7424/50176 [00:52<04:50, 147.39it/s]
 15%|█▌        | 7680/50176 [00:54<04:50, 146.04it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 148.98it/s]
 16%|█▋        | 8192/50176 [00:57<04:44, 147.66it/s]
 17%|█▋        | 8448/50176 [00:59<04:40, 148.67it/s]
 17%|█▋        | 8704/50176 [01:00<04:37, 149.35it/s]
 18%|█▊        | 8960/50176 [01:02<04:44, 144.75it/s]
 18%|█▊        | 9216/50176 [01:04<04:37, 147.50it/s]
 19%|█▉        | 9472/50176 [01:06<04:36, 147.23it/s]
 19%|█▉        | 9728/50176 [01:08<04:38, 145.08it/s]
 20%|█▉        | 9984/50176 [01:09<04:32, 147.66it/s]
 20%|██        | 10240/50176 [01:11<04:26, 149.59it/s]
 21%|██        | 10496/50176 [01:13<04:24, 150.02it/s]
 21%|██▏       | 10752/50176 [01:14<04:30, 145.99it/s]
 22%|██▏       | 11008/50176 [01:16<04:24, 148.08it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.45it/s]
 23%|██▎       | 11520/50176 [01:19<04:15, 151.50it/s]
 23%|██▎       | 11776/50176 [01:21<04:14, 151.09it/s]
 24%|██▍       | 12032/50176 [01:23<04:12, 151.09it/s]
 24%|██▍       | 12288/50176 [01:24<04:08, 152.20it/s]
 25%|██▌       | 12544/50176 [01:26<04:06, 152.73it/s]
 26%|██▌       | 12800/50176 [01:28<04:04, 152.76it/s]
 26%|██▌       | 13056/50176 [01:29<04:02, 152.86it/s]
 27%|██▋       | 13312/50176 [01:31<04:06, 149.79it/s]
 27%|██▋       | 13568/50176 [01:33<04:14, 143.89it/s]
 28%|██▊       | 13824/50176 [01:35<04:19, 140.21it/s]
 28%|██▊       | 14080/50176 [01:37<04:15, 141.30it/s]
 29%|██▊       | 14336/50176 [01:39<04:19, 138.19it/s]
 29%|██▉       | 14592/50176 [01:41<04:08, 143.05it/s]
 30%|██▉       | 14848/50176 [01:43<04:20, 135.61it/s]
 30%|███       | 15104/50176 [01:45<04:18, 135.42it/s]
 31%|███       | 15360/50176 [01:46<04:12, 137.86it/s]
 31%|███       | 15616/50176 [01:48<04:15, 135.09it/s]
 32%|███▏      | 15872/50176 [01:50<04:15, 134.28it/s]
 32%|███▏      | 16128/50176 [01:52<04:07, 137.69it/s]
 33%|███▎      | 16384/50176 [01:54<04:04, 138.15it/s]
 33%|███▎      | 16640/50176 [01:56<04:09, 134.15it/s]
 34%|███▎      | 16896/50176 [01:58<03:58, 139.35it/s]
 34%|███▍      | 17152/50176 [01:59<03:50, 142.99it/s]
 35%|███▍      | 17408/50176 [02:01<03:46, 144.52it/s]
 35%|███▌      | 17664/50176 [02:03<03:41, 146.62it/s]
 36%|███▌      | 17920/50176 [02:04<03:37, 148.15it/s]
 36%|███▌      | 18176/50176 [02:06<03:38, 146.69it/s]
 37%|███▋      | 18432/50176 [02:08<03:34, 147.71it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.05it/s]
 38%|███▊      | 18944/50176 [02:12<03:40, 141.69it/s]
 38%|███▊      | 19200/50176 [02:14<03:50, 134.59it/s]
 39%|███▉      | 19456/50176 [02:16<03:52, 132.38it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.27it/s]
 40%|███▉      | 19968/50176 [02:19<03:43, 135.17it/s]
 40%|████      | 20224/50176 [02:21<03:32, 140.94it/s]
 41%|████      | 20480/50176 [02:23<03:26, 143.65it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 143.88it/s]
 42%|████▏     | 20992/50176 [02:26<03:19, 146.25it/s]
 42%|████▏     | 21248/50176 [02:28<03:23, 142.30it/s]
 43%|████▎     | 21504/50176 [02:30<03:16, 146.11it/s]
 43%|████▎     | 21760/50176 [02:31<03:12, 147.39it/s]
 44%|████▍     | 22016/50176 [02:33<03:07, 150.32it/s]
 44%|████▍     | 22272/50176 [02:35<03:06, 149.77it/s]
 45%|████▍     | 22528/50176 [02:36<03:03, 150.91it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.09it/s]
 46%|████▌     | 23040/50176 [02:40<03:09, 142.90it/s]
 46%|████▋     | 23296/50176 [02:42<03:12, 139.73it/s]
 47%|████▋     | 23552/50176 [02:44<03:15, 136.46it/s]
 47%|████▋     | 23808/50176 [02:46<03:07, 140.99it/s]
 48%|████▊     | 24064/50176 [02:48<03:02, 142.78it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.24it/s]
 49%|████▉     | 24576/50176 [02:51<02:56, 145.34it/s]
 49%|████▉     | 24832/50176 [02:53<02:50, 148.48it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 148.98it/s]
 51%|█████     | 25344/50176 [02:56<02:46, 149.54it/s]
 51%|█████     | 25600/50176 [02:58<02:48, 145.81it/s]
 52%|█████▏    | 25856/50176 [03:00<02:41, 150.42it/s]
 52%|█████▏    | 26112/50176 [03:01<02:43, 147.31it/s]
 53%|█████▎    | 26368/50176 [03:03<02:46, 143.35it/s]
 53%|█████▎    | 26624/50176 [03:05<02:42, 145.04it/s]
 54%|█████▎    | 26880/50176 [03:07<02:40, 144.91it/s]
 54%|█████▍    | 27136/50176 [03:08<02:38, 145.67it/s]
 55%|█████▍    | 27392/50176 [03:10<02:35, 146.37it/s]
 55%|█████▌    | 27648/50176 [03:12<02:28, 152.15it/s]
 56%|█████▌    | 27904/50176 [03:14<02:36, 142.06it/s]
 56%|█████▌    | 28160/50176 [03:16<02:33, 143.63it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.15it/s]
 57%|█████▋    | 28672/50176 [03:19<02:33, 140.26it/s]
 58%|█████▊    | 28928/50176 [03:21<02:28, 142.75it/s]
 58%|█████▊    | 29184/50176 [03:23<02:25, 144.73it/s]
 59%|█████▊    | 29440/50176 [03:25<02:32, 136.16it/s]
 59%|█████▉    | 29696/50176 [03:27<02:34, 132.86it/s]
 60%|█████▉    | 29952/50176 [03:29<02:32, 133.02it/s]
 60%|██████    | 30208/50176 [03:30<02:25, 137.46it/s]
 61%|██████    | 30464/50176 [03:32<02:17, 143.09it/s]
 61%|██████    | 30720/50176 [03:34<02:16, 142.73it/s]
 62%|██████▏   | 30976/50176 [03:36<02:11, 145.61it/s]
 62%|██████▏   | 31232/50176 [03:37<02:08, 148.00it/s]
 63%|██████▎   | 31488/50176 [03:39<02:05, 149.34it/s]
 63%|██████▎   | 31744/50176 [03:41<02:03, 149.49it/s]
 64%|██████▍   | 32000/50176 [03:42<02:02, 148.35it/s]
 64%|██████▍   | 32256/50176 [03:44<02:00, 148.30it/s]
 65%|██████▍   | 32512/50176 [03:46<02:02, 144.68it/s]
 65%|██████▌   | 32768/50176 [03:48<01:58, 147.23it/s]
 66%|██████▌   | 33024/50176 [03:49<01:57, 145.46it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 146.85it/s]
 67%|██████▋   | 33536/50176 [03:53<01:56, 143.28it/s]
 67%|██████▋   | 33792/50176 [03:55<01:51, 146.33it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 150.28it/s]
 68%|██████▊   | 34304/50176 [03:58<01:45, 150.97it/s]
 69%|██████▉   | 34560/50176 [04:00<01:47, 145.52it/s]
 69%|██████▉   | 34816/50176 [04:02<01:45, 145.78it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.98it/s]
 70%|███████   | 35328/50176 [04:05<01:40, 147.73it/s]
 71%|███████   | 35584/50176 [04:07<01:40, 145.40it/s]
 71%|███████▏  | 35840/50176 [04:09<01:41, 140.95it/s]
 72%|███████▏  | 36096/50176 [04:11<01:38, 143.41it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.55it/s]
 73%|███████▎  | 36608/50176 [04:14<01:33, 144.52it/s]
 73%|███████▎  | 36864/50176 [04:16<01:32, 143.19it/s]
 74%|███████▍  | 37120/50176 [04:18<01:30, 144.56it/s]
 74%|███████▍  | 37376/50176 [04:19<01:26, 148.77it/s]
 75%|███████▌  | 37632/50176 [04:21<01:23, 151.01it/s]
 76%|███████▌  | 37888/50176 [04:23<01:22, 148.45it/s]
 76%|███████▌  | 38144/50176 [04:24<01:20, 149.90it/s]
 77%|███████▋  | 38400/50176 [04:26<01:19, 148.40it/s]
 77%|███████▋  | 38656/50176 [04:28<01:18, 146.98it/s]
 78%|███████▊  | 38912/50176 [04:30<01:17, 145.40it/s]
 78%|███████▊  | 39168/50176 [04:32<01:17, 141.76it/s]
 79%|███████▊  | 39424/50176 [04:33<01:14, 143.67it/s]
 79%|███████▉  | 39680/50176 [04:35<01:12, 145.26it/s]
 80%|███████▉  | 39936/50176 [04:37<01:09, 147.77it/s]
 80%|████████  | 40192/50176 [04:38<01:06, 150.03it/s]
 81%|████████  | 40448/50176 [04:40<01:07, 143.25it/s]
 81%|████████  | 40704/50176 [04:42<01:08, 137.99it/s]
 82%|████████▏ | 40960/50176 [04:44<01:04, 143.47it/s]
 82%|████████▏ | 41216/50176 [04:46<01:01, 146.80it/s]
 83%|████████▎ | 41472/50176 [04:47<00:58, 148.91it/s]
 83%|████████▎ | 41728/50176 [04:49<00:55, 151.66it/s]
 84%|████████▎ | 41984/50176 [04:51<00:53, 151.82it/s]
 84%|████████▍ | 42240/50176 [04:52<00:53, 148.69it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 149.79it/s]
 85%|████████▌ | 42752/50176 [04:56<00:49, 149.32it/s]
 86%|████████▌ | 43008/50176 [04:57<00:47, 150.81it/s]
 86%|████████▌ | 43264/50176 [04:59<00:45, 150.53it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.75it/s]
 87%|████████▋ | 43776/50176 [05:03<00:43, 146.35it/s]
 88%|████████▊ | 44032/50176 [05:04<00:41, 147.98it/s]
 88%|████████▊ | 44288/50176 [05:06<00:39, 150.48it/s]
 89%|████████▉ | 44544/50176 [05:08<00:38, 147.77it/s]
 89%|████████▉ | 44800/50176 [05:10<00:36, 146.37it/s]
 90%|████████▉ | 45056/50176 [05:11<00:34, 148.45it/s]
 90%|█████████ | 45312/50176 [05:13<00:34, 141.80it/s]
 91%|█████████ | 45568/50176 [05:15<00:31, 146.60it/s]
 91%|█████████▏| 45824/50176 [05:17<00:29, 148.24it/s]
 92%|█████████▏| 46080/50176 [05:18<00:27, 148.54it/s]
 92%|█████████▏| 46336/50176 [05:20<00:25, 149.42it/s]
 93%|█████████▎| 46592/50176 [05:22<00:24, 148.74it/s]
 93%|█████████▎| 46848/50176 [05:24<00:22, 148.18it/s]
 94%|█████████▍| 47104/50176 [05:25<00:21, 143.71it/s]
 94%|█████████▍| 47360/50176 [05:27<00:19, 145.42it/s]
 95%|█████████▍| 47616/50176 [05:29<00:17, 143.66it/s]
 95%|█████████▌| 47872/50176 [05:31<00:15, 146.55it/s]
 96%|█████████▌| 48128/50176 [05:32<00:13, 146.99it/s]
 96%|█████████▋| 48384/50176 [05:34<00:11, 149.73it/s]
 97%|█████████▋| 48640/50176 [05:36<00:10, 151.90it/s]
 97%|█████████▋| 48896/50176 [05:37<00:08, 151.92it/s]
 98%|█████████▊| 49152/50176 [05:39<00:06, 147.66it/s]
 98%|█████████▊| 49408/50176 [05:41<00:05, 145.41it/s]
 99%|█████████▉| 49664/50176 [05:43<00:03, 145.51it/s]
 99%|█████████▉| 49920/50176 [05:45<00:01, 138.63it/s]
100%|██████████| 50176/50176 [05:45<00:00, 181.03it/s]
100%|██████████| 50176/50176 [05:45<00:00, 145.13it/s]
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5645	imagenet-zeroshot-val-top5: 0.8353
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50d2370fff1af0fd647e48cf3c11c7648f29b025
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4c216a92478cd7ea89e726cfbb187bf4a715a0f3c9141cfd597a85bc6e049731
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1c8adf2d254138429b3e07ddbf8fa43e396b8c1f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt).
+2022-12-21,10:01:07 | INFO | Model:
+2022-12-21,10:01:07 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:07 | INFO | Params:
+2022-12-21,10:01:07 | INFO |   batch_size: 256
+2022-12-21,10:01:07 | INFO |   beta1: 0.9
+2022-12-21,10:01:07 | INFO |   beta2: 0.98
+2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/checkpoints
+2022-12-21,10:01:07 | INFO |   copy_codebase: False
+2022-12-21,10:01:07 | INFO |   csv_caption_key: title
+2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:07 | INFO |   csv_separator: 	
+2022-12-21,10:01:07 | INFO |   dataset_resampled: False
+2022-12-21,10:01:07 | INFO |   dataset_type: auto
+2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:07 | INFO |   debug: False
+2022-12-21,10:01:07 | INFO |   device: cuda:0
+2022-12-21,10:01:07 | INFO |   dist_backend: nccl
+2022-12-21,10:01:07 | INFO |   dist_url: env://
+2022-12-21,10:01:07 | INFO |   distributed: False
+2022-12-21,10:01:07 | INFO |   epochs: 32
+2022-12-21,10:01:07 | INFO |   eps: 1e-06
+2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:07 | INFO |   gather_with_grad: False
+2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:07 | INFO |   horovod: False
+2022-12-21,10:01:07 | INFO |   image_mean: None
+2022-12-21,10:01:07 | INFO |   image_std: None
+2022-12-21,10:01:07 | INFO |   imagenet_v2: None
+2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:07 | INFO |   local_loss: False
+2022-12-21,10:01:07 | INFO |   local_rank: 0
+2022-12-21,10:01:07 | INFO |   lock_image: False
+2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:07 | INFO |   log_level: 20
+2022-12-21,10:01:07 | INFO |   log_local: False
+2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/out.log
+2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:07 | INFO |   lr: 0.0005
+2022-12-21,10:01:07 | INFO |   model: ViT-B-16
+2022-12-21,10:01:07 | INFO |   name: epoch_21.pt
+2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt
+2022-12-21,10:01:07 | INFO |   pretrained_image: False
+2022-12-21,10:01:07 | INFO |   rank: 0
+2022-12-21,10:01:07 | INFO |   report_to: 
+2022-12-21,10:01:07 | INFO |   resume: None
+2022-12-21,10:01:07 | INFO |   save_frequency: 1
+2022-12-21,10:01:07 | INFO |   save_most_recent: False
+2022-12-21,10:01:07 | INFO |   seed: 0
+2022-12-21,10:01:07 | INFO |   skip_scheduler: False
+2022-12-21,10:01:07 | INFO |   tensorboard: False
+2022-12-21,10:01:07 | INFO |   tensorboard_path: 
+2022-12-21,10:01:07 | INFO |   torchscript: False
+2022-12-21,10:01:07 | INFO |   trace: False
+2022-12-21,10:01:07 | INFO |   train_data: None
+2022-12-21,10:01:07 | INFO |   train_num_samples: None
+2022-12-21,10:01:07 | INFO |   use_bn_sync: False
+2022-12-21,10:01:07 | INFO |   val_data: None
+2022-12-21,10:01:07 | INFO |   val_frequency: 1
+2022-12-21,10:01:07 | INFO |   val_num_samples: None
+2022-12-21,10:01:07 | INFO |   wandb: False
+2022-12-21,10:01:07 | INFO |   wandb_notes: 
+2022-12-21,10:01:07 | INFO |   warmup: 10000
+2022-12-21,10:01:07 | INFO |   wd: 0.2
+2022-12-21,10:01:07 | INFO |   workers: 1
+2022-12-21,10:01:07 | INFO |   world_size: 1
+2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:07 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:56,  1.52it/s]
  1%|          | 7/1000 [00:00<01:25, 11.56it/s]
  1%|▏         | 13/1000 [00:00<00:48, 20.38it/s]
  2%|▏         | 19/1000 [00:00<00:34, 28.69it/s]
  2%|▎         | 25/1000 [00:01<00:27, 35.66it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.36it/s]
  4%|▎         | 37/1000 [00:01<00:21, 45.75it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.05it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.46it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.17it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.41it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.36it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.05it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.64it/s]
  8%|▊         | 85/1000 [00:02<00:17, 51.56it/s]
  9%|▉         | 91/1000 [00:02<00:17, 53.35it/s]
 10%|▉         | 97/1000 [00:02<00:16, 54.85it/s]
 10%|█         | 103/1000 [00:02<00:16, 55.84it/s]
 11%|█         | 109/1000 [00:02<00:15, 56.44it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 57.08it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.36it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 57.58it/s]
 13%|█▎        | 133/1000 [00:03<00:15, 57.78it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.88it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.00it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.09it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.06it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.11it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.91it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.94it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.88it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.89it/s]
 19%|█▉        | 193/1000 [00:04<00:13, 57.88it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.81it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.64it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.65it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.65it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.82it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.79it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.61it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.78it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.65it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.83it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.85it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.72it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.86it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 58.03it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.17it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.34it/s]
 30%|███       | 301/1000 [00:05<00:11, 58.30it/s]
 31%|███       | 307/1000 [00:06<00:11, 58.24it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.43it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.45it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.26it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.11it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.25it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.31it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.27it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 58.19it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.24it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.29it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.37it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.13it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.93it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 58.00it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.07it/s]
 40%|████      | 403/1000 [00:07<00:10, 58.19it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.26it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.32it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.33it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.34it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.47it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.44it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.16it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.27it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.29it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.34it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.26it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.41it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.22it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.33it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.27it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.19it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.25it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.29it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.26it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.28it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.34it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.43it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.44it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.32it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.37it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.16it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.22it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.31it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.32it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.23it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.35it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.22it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.27it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.40it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.34it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.34it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.25it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.23it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.32it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.32it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.36it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.34it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.39it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.39it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.31it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.36it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.38it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 58.35it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.17it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.17it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.24it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.27it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.35it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.39it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.25it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.33it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.32it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.34it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.36it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.21it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.15it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.21it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.22it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.33it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.48it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.50it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.37it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.38it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.37it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.33it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.37it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.32it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.25it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.36it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.20it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 57.97it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.19it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.32it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.37it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.32it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.32it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.46it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.34it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.44it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.55it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.56it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.54it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.42it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 58.20it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.88it/s]
+2022-12-21,10:01:25 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:08, 135.42it/s]
  1%|          | 512/50176 [00:03<05:15, 157.34it/s]
  2%|▏         | 768/50176 [00:04<04:50, 170.14it/s]
  2%|▏         | 1024/50176 [00:06<04:50, 169.46it/s]
  3%|▎         | 1280/50176 [00:08<05:08, 158.58it/s]
  3%|▎         | 1536/50176 [00:10<05:34, 145.37it/s]
  4%|▎         | 1792/50176 [00:11<05:36, 143.96it/s]
  4%|▍         | 2048/50176 [00:13<05:33, 144.35it/s]
  5%|▍         | 2304/50176 [00:15<05:33, 143.46it/s]
  5%|▌         | 2560/50176 [00:17<05:30, 143.86it/s]
  6%|▌         | 2816/50176 [00:19<05:32, 142.64it/s]
  6%|▌         | 3072/50176 [00:20<05:25, 144.65it/s]
  7%|▋         | 3328/50176 [00:22<05:22, 145.26it/s]
  7%|▋         | 3584/50176 [00:24<05:15, 147.59it/s]
  8%|▊         | 3840/50176 [00:26<05:27, 141.65it/s]
  8%|▊         | 4096/50176 [00:27<05:25, 141.78it/s]
  9%|▊         | 4352/50176 [00:29<05:19, 143.35it/s]
  9%|▉         | 4608/50176 [00:31<05:16, 144.07it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.65it/s]
 10%|█         | 5120/50176 [00:34<05:06, 147.08it/s]
 11%|█         | 5376/50176 [00:36<05:01, 148.83it/s]
 11%|█         | 5632/50176 [00:38<04:57, 149.76it/s]
 12%|█▏        | 5888/50176 [00:39<04:55, 149.82it/s]
 12%|█▏        | 6144/50176 [00:42<05:19, 137.67it/s]
 13%|█▎        | 6400/50176 [00:43<05:09, 141.22it/s]
 13%|█▎        | 6656/50176 [00:45<05:03, 143.27it/s]
 14%|█▍        | 6912/50176 [00:47<05:01, 143.58it/s]
 14%|█▍        | 7168/50176 [00:49<04:56, 145.03it/s]
 15%|█▍        | 7424/50176 [00:50<04:51, 146.60it/s]
 15%|█▌        | 7680/50176 [00:52<04:50, 146.22it/s]
 16%|█▌        | 7936/50176 [00:54<04:43, 149.07it/s]
 16%|█▋        | 8192/50176 [00:55<04:44, 147.80it/s]
 17%|█▋        | 8448/50176 [00:57<04:40, 148.69it/s]
 17%|█▋        | 8704/50176 [00:59<04:37, 149.48it/s]
 18%|█▊        | 8960/50176 [01:01<04:46, 144.07it/s]
 18%|█▊        | 9216/50176 [01:02<04:37, 147.71it/s]
 19%|█▉        | 9472/50176 [01:04<04:35, 147.98it/s]
 19%|█▉        | 9728/50176 [01:06<04:40, 144.30it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 147.78it/s]
 20%|██        | 10240/50176 [01:09<04:26, 149.71it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.10it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.41it/s]
 22%|██▏       | 11008/50176 [01:14<04:24, 148.31it/s]
 22%|██▏       | 11264/50176 [01:16<04:21, 148.75it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.85it/s]
 23%|██▎       | 11776/50176 [01:20<04:13, 151.37it/s]
 24%|██▍       | 12032/50176 [01:21<04:11, 151.88it/s]
 24%|██▍       | 12288/50176 [01:23<04:09, 152.16it/s]
 25%|██▌       | 12544/50176 [01:25<04:06, 152.67it/s]
 26%|██▌       | 12800/50176 [01:26<04:04, 152.79it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.83it/s]
 27%|██▋       | 13312/50176 [01:30<04:06, 149.77it/s]
 27%|██▋       | 13568/50176 [01:32<04:15, 143.31it/s]
 28%|██▊       | 13824/50176 [01:34<04:17, 141.08it/s]
 28%|██▊       | 14080/50176 [01:35<04:15, 141.24it/s]
 29%|██▊       | 14336/50176 [01:37<04:20, 137.63it/s]
 29%|██▉       | 14592/50176 [01:39<04:08, 143.17it/s]
 30%|██▉       | 14848/50176 [01:41<04:20, 135.80it/s]
 30%|███       | 15104/50176 [01:43<04:19, 134.89it/s]
 31%|███       | 15360/50176 [01:45<04:11, 138.52it/s]
 31%|███       | 15616/50176 [01:47<04:14, 135.58it/s]
 32%|███▏      | 15872/50176 [01:49<04:15, 134.05it/s]
 32%|███▏      | 16128/50176 [01:50<04:07, 137.45it/s]
 33%|███▎      | 16384/50176 [01:52<04:04, 138.24it/s]
 33%|███▎      | 16640/50176 [01:54<04:10, 134.07it/s]
 34%|███▎      | 16896/50176 [01:56<03:58, 139.42it/s]
 34%|███▍      | 17152/50176 [01:58<03:51, 142.89it/s]
 35%|███▍      | 17408/50176 [01:59<03:46, 144.47it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.60it/s]
 36%|███▌      | 17920/50176 [02:03<03:37, 148.17it/s]
 36%|███▌      | 18176/50176 [02:04<03:37, 147.15it/s]
 37%|███▋      | 18432/50176 [02:06<03:35, 147.62it/s]
 37%|███▋      | 18688/50176 [02:08<03:34, 146.88it/s]
 38%|███▊      | 18944/50176 [02:10<03:40, 141.95it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.48it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.21it/s]
 39%|███▉      | 19712/50176 [02:16<03:50, 132.07it/s]
 40%|███▉      | 19968/50176 [02:18<03:43, 135.21it/s]
 40%|████      | 20224/50176 [02:19<03:32, 140.93it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.76it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.78it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.51it/s]
 42%|████▏     | 21248/50176 [02:27<03:24, 141.56it/s]
 43%|████▎     | 21504/50176 [02:28<03:15, 146.30it/s]
 43%|████▎     | 21760/50176 [02:30<03:12, 147.47it/s]
 44%|████▍     | 22016/50176 [02:31<03:07, 150.00it/s]
 44%|████▍     | 22272/50176 [02:33<03:06, 149.72it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.17it/s]
 45%|████▌     | 22784/50176 [02:37<03:09, 144.52it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.35it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.61it/s]
 47%|████▋     | 23552/50176 [02:42<03:14, 137.22it/s]
 47%|████▋     | 23808/50176 [02:44<03:07, 140.94it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 142.73it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.22it/s]
 49%|████▉     | 24576/50176 [02:49<02:56, 145.45it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.49it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.97it/s]
 51%|█████     | 25344/50176 [02:54<02:46, 149.46it/s]
 51%|█████     | 25600/50176 [02:56<02:48, 145.81it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.33it/s]
 52%|█████▏    | 26112/50176 [03:00<02:43, 147.15it/s]
 53%|█████▎    | 26368/50176 [03:02<02:46, 143.33it/s]
 53%|█████▎    | 26624/50176 [03:03<02:42, 145.16it/s]
 54%|█████▎    | 26880/50176 [03:05<02:41, 144.37it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 145.90it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.46it/s]
 55%|█████▌    | 27648/50176 [03:10<02:28, 151.54it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 142.36it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.89it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.22it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.83it/s]
 58%|█████▊    | 28928/50176 [03:19<02:29, 142.53it/s]
 58%|█████▊    | 29184/50176 [03:21<02:24, 144.99it/s]
 59%|█████▊    | 29440/50176 [03:23<02:32, 135.88it/s]
 59%|█████▉    | 29696/50176 [03:25<02:34, 132.72it/s]
 60%|█████▉    | 29952/50176 [03:27<02:31, 133.52it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.39it/s]
 61%|██████    | 30464/50176 [03:30<02:17, 143.31it/s]
 61%|██████    | 30720/50176 [03:32<02:16, 142.68it/s]
 62%|██████▏   | 30976/50176 [03:34<02:12, 145.31it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.91it/s]
 63%|██████▎   | 31488/50176 [03:37<02:05, 149.25it/s]
 63%|██████▎   | 31744/50176 [03:39<02:03, 149.49it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.29it/s]
 64%|██████▍   | 32256/50176 [03:42<02:00, 149.11it/s]
 65%|██████▍   | 32512/50176 [03:44<02:02, 143.71it/s]
 65%|██████▌   | 32768/50176 [03:46<01:58, 147.41it/s]
 66%|██████▌   | 33024/50176 [03:48<01:58, 145.35it/s]
 66%|██████▋   | 33280/50176 [03:50<01:54, 146.94it/s]
 67%|██████▋   | 33536/50176 [03:51<01:56, 142.62it/s]
 67%|██████▋   | 33792/50176 [03:53<01:51, 146.67it/s]
 68%|██████▊   | 34048/50176 [03:55<01:47, 150.30it/s]
 68%|██████▊   | 34304/50176 [03:56<01:44, 151.84it/s]
 69%|██████▉   | 34560/50176 [03:58<01:47, 144.68it/s]
 69%|██████▉   | 34816/50176 [04:00<01:45, 145.92it/s]
 70%|██████▉   | 35072/50176 [04:02<01:44, 144.94it/s]
 70%|███████   | 35328/50176 [04:03<01:40, 147.87it/s]
 71%|███████   | 35584/50176 [04:05<01:40, 144.73it/s]
 71%|███████▏  | 35840/50176 [04:07<01:41, 141.61it/s]
 72%|███████▏  | 36096/50176 [04:09<01:38, 143.27it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 141.67it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 144.49it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.36it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 144.50it/s]
 74%|███████▍  | 37376/50176 [04:18<01:25, 148.86it/s]
 75%|███████▌  | 37632/50176 [04:19<01:23, 151.04it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 148.89it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.65it/s]
 77%|███████▋  | 38400/50176 [04:24<01:19, 148.43it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.22it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.30it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.81it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.53it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.13it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.88it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.89it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.79it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 137.73it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.35it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.64it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.89it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.46it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 151.77it/s]
 84%|████████▍ | 42240/50176 [04:51<00:53, 148.64it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.85it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.25it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.91it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.36it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.52it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 145.94it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 148.03it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.67it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 148.03it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 146.47it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 148.54it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 141.90it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.81it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 148.14it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.63it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.29it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.42it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 147.51it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.42it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.39it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.81it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 146.56it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 147.24it/s]
 96%|█████████▋| 48384/50176 [05:32<00:11, 149.68it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 151.98it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 151.73it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.72it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.30it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.63it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.84it/s]
100%|██████████| 50176/50176 [05:44<00:00, 180.99it/s]
100%|██████████| 50176/50176 [05:44<00:00, 145.82it/s]
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5724	imagenet-zeroshot-val-top5: 0.8406
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt
new file mode 100644
index 0000000000000000000000000000000000000000..2140a101d81fb070bf2eb8a3d8e8ea50d1082e7c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d3f5710f609fd28b51cd4dde5a1d1e3cd14bb38908120409bcfe530eaae31dd
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..87b2b96b5874e9f02e5f05f6354c50613e0c0cbf
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:00:58 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:58 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt).
+2022-12-21,10:01:07 | INFO | Model:
+2022-12-21,10:01:07 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:07 | INFO | Params:
+2022-12-21,10:01:07 | INFO |   batch_size: 256
+2022-12-21,10:01:07 | INFO |   beta1: 0.9
+2022-12-21,10:01:07 | INFO |   beta2: 0.98
+2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/checkpoints
+2022-12-21,10:01:07 | INFO |   copy_codebase: False
+2022-12-21,10:01:07 | INFO |   csv_caption_key: title
+2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:07 | INFO |   csv_separator: 	
+2022-12-21,10:01:07 | INFO |   dataset_resampled: False
+2022-12-21,10:01:07 | INFO |   dataset_type: auto
+2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:07 | INFO |   debug: False
+2022-12-21,10:01:07 | INFO |   device: cuda:0
+2022-12-21,10:01:07 | INFO |   dist_backend: nccl
+2022-12-21,10:01:07 | INFO |   dist_url: env://
+2022-12-21,10:01:07 | INFO |   distributed: False
+2022-12-21,10:01:07 | INFO |   epochs: 32
+2022-12-21,10:01:07 | INFO |   eps: 1e-06
+2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:07 | INFO |   gather_with_grad: False
+2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:07 | INFO |   horovod: False
+2022-12-21,10:01:07 | INFO |   image_mean: None
+2022-12-21,10:01:07 | INFO |   image_std: None
+2022-12-21,10:01:07 | INFO |   imagenet_v2: None
+2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:07 | INFO |   local_loss: False
+2022-12-21,10:01:07 | INFO |   local_rank: 0
+2022-12-21,10:01:07 | INFO |   lock_image: False
+2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:07 | INFO |   log_level: 20
+2022-12-21,10:01:07 | INFO |   log_local: False
+2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/out.log
+2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:07 | INFO |   lr: 0.0005
+2022-12-21,10:01:07 | INFO |   model: ViT-B-16
+2022-12-21,10:01:07 | INFO |   name: epoch_22.pt
+2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt
+2022-12-21,10:01:07 | INFO |   pretrained_image: False
+2022-12-21,10:01:07 | INFO |   rank: 0
+2022-12-21,10:01:07 | INFO |   report_to: 
+2022-12-21,10:01:07 | INFO |   resume: None
+2022-12-21,10:01:07 | INFO |   save_frequency: 1
+2022-12-21,10:01:07 | INFO |   save_most_recent: False
+2022-12-21,10:01:07 | INFO |   seed: 0
+2022-12-21,10:01:07 | INFO |   skip_scheduler: False
+2022-12-21,10:01:07 | INFO |   tensorboard: False
+2022-12-21,10:01:07 | INFO |   tensorboard_path: 
+2022-12-21,10:01:07 | INFO |   torchscript: False
+2022-12-21,10:01:07 | INFO |   trace: False
+2022-12-21,10:01:07 | INFO |   train_data: None
+2022-12-21,10:01:07 | INFO |   train_num_samples: None
+2022-12-21,10:01:07 | INFO |   use_bn_sync: False
+2022-12-21,10:01:07 | INFO |   val_data: None
+2022-12-21,10:01:07 | INFO |   val_frequency: 1
+2022-12-21,10:01:07 | INFO |   val_num_samples: None
+2022-12-21,10:01:07 | INFO |   wandb: False
+2022-12-21,10:01:07 | INFO |   wandb_notes: 
+2022-12-21,10:01:07 | INFO |   warmup: 10000
+2022-12-21,10:01:07 | INFO |   wd: 0.2
+2022-12-21,10:01:07 | INFO |   workers: 1
+2022-12-21,10:01:07 | INFO |   world_size: 1
+2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:07 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:57,  1.39it/s]
  1%|          | 6/1000 [00:00<01:45,  9.44it/s]
  1%|          | 12/1000 [00:00<00:53, 18.47it/s]
  2%|▏         | 18/1000 [00:01<00:36, 26.91it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.15it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.05it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.66it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.07it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.73it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.62it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.04it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.05it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.78it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.29it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.86it/s]
  9%|▉         | 90/1000 [00:02<00:17, 52.46it/s]
 10%|▉         | 96/1000 [00:02<00:16, 54.13it/s]
 10%|█         | 102/1000 [00:02<00:16, 55.20it/s]
 11%|█         | 108/1000 [00:02<00:15, 56.18it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 56.76it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.11it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.33it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.56it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.66it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.79it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.92it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.01it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.90it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.79it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.80it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.76it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.68it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.66it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.42it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.40it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.62it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.60it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.49it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.56it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.65it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.76it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.78it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.66it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.86it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.91it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.11it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.10it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.09it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.13it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.09it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.18it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.96it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.02it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.11it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.66it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.74it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.85it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.00it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.09it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.11it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.26it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.14it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.08it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.22it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.12it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.11it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.03it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.09it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.00it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.09it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.17it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.10it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.21it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.21it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.10it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.11it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.03it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.97it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.12it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 594/1000 [00:11<00:06, 58.02it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.13it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.05it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.17it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.26it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.08it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.01it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.15it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.10it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.18it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.19it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.21it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.10it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.18it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.13it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.11it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.12it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.09it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.13it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.11it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.20it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.06it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.93it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.09it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.06it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.11it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.18it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.99it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.99it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.05it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.02it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.03it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.13it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.27it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.29it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.20it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.21it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.21it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.14it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.20it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.18it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.05it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.16it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.94it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.97it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.99it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.12it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.19it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.17it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.21it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.24it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.19it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.32it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.34it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.33it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.28it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.10it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.55it/s]
+2022-12-21,10:01:25 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:04, 137.06it/s]
  1%|          | 512/50176 [00:03<05:12, 158.79it/s]
  2%|▏         | 768/50176 [00:04<04:47, 172.08it/s]
  2%|▏         | 1024/50176 [00:06<04:52, 167.78it/s]
  3%|▎         | 1280/50176 [00:08<05:10, 157.46it/s]
  3%|▎         | 1536/50176 [00:10<05:36, 144.69it/s]
  4%|▎         | 1792/50176 [00:11<05:37, 143.24it/s]
  4%|▍         | 2048/50176 [00:13<05:34, 143.97it/s]
  5%|▍         | 2304/50176 [00:15<05:34, 143.13it/s]
  5%|▌         | 2560/50176 [00:17<05:30, 144.20it/s]
  6%|▌         | 2816/50176 [00:19<05:32, 142.39it/s]
  6%|▌         | 3072/50176 [00:20<05:25, 144.52it/s]
  7%|▋         | 3328/50176 [00:22<05:22, 145.21it/s]
  7%|▋         | 3584/50176 [00:24<05:15, 147.50it/s]
  8%|▊         | 3840/50176 [00:26<05:25, 142.19it/s]
  8%|▊         | 4096/50176 [00:27<05:25, 141.36it/s]
  9%|▊         | 4352/50176 [00:29<05:19, 143.35it/s]
  9%|▉         | 4608/50176 [00:31<05:14, 144.76it/s]
 10%|▉         | 4864/50176 [00:33<05:09, 146.53it/s]
 10%|█         | 5120/50176 [00:34<05:05, 147.57it/s]
 11%|█         | 5376/50176 [00:36<05:01, 148.58it/s]
 11%|█         | 5632/50176 [00:38<04:57, 149.53it/s]
 12%|█▏        | 5888/50176 [00:39<04:56, 149.61it/s]
 12%|█▏        | 6144/50176 [00:42<05:20, 137.47it/s]
 13%|█▎        | 6400/50176 [00:43<05:10, 141.12it/s]
 13%|█▎        | 6656/50176 [00:45<05:04, 143.08it/s]
 14%|█▍        | 6912/50176 [00:47<05:01, 143.62it/s]
 14%|█▍        | 7168/50176 [00:49<04:56, 145.08it/s]
 15%|█▍        | 7424/50176 [00:50<04:50, 147.35it/s]
 15%|█▌        | 7680/50176 [00:52<04:51, 146.03it/s]
 16%|█▌        | 7936/50176 [00:54<04:43, 148.91it/s]
 16%|█▋        | 8192/50176 [00:55<04:44, 147.76it/s]
 17%|█▋        | 8448/50176 [00:57<04:40, 148.69it/s]
 17%|█▋        | 8704/50176 [00:59<04:37, 149.41it/s]
 18%|█▊        | 8960/50176 [01:01<04:44, 144.68it/s]
 18%|█▊        | 9216/50176 [01:02<04:37, 147.43it/s]
 19%|█▉        | 9472/50176 [01:04<04:36, 147.19it/s]
 19%|█▉        | 9728/50176 [01:06<04:38, 145.04it/s]
 20%|█▉        | 9984/50176 [01:08<04:32, 147.61it/s]
 20%|██        | 10240/50176 [01:09<04:26, 149.58it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.06it/s]
 21%|██▏       | 10752/50176 [01:13<04:30, 146.00it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 148.11it/s]
 22%|██▏       | 11264/50176 [01:16<04:21, 148.66it/s]
 23%|██▎       | 11520/50176 [01:18<04:15, 151.40it/s]
 23%|██▎       | 11776/50176 [01:20<04:14, 151.09it/s]
 24%|██▍       | 12032/50176 [01:21<04:11, 151.77it/s]
 24%|██▍       | 12288/50176 [01:23<04:09, 152.03it/s]
 25%|██▌       | 12544/50176 [01:25<04:06, 152.61it/s]
 26%|██▌       | 12800/50176 [01:26<04:04, 152.70it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.80it/s]
 27%|██▋       | 13312/50176 [01:30<04:06, 149.63it/s]
 27%|██▋       | 13568/50176 [01:32<04:15, 143.55it/s]
 28%|██▊       | 13824/50176 [01:34<04:18, 140.88it/s]
 28%|██▊       | 14080/50176 [01:35<04:15, 141.19it/s]
 29%|██▊       | 14336/50176 [01:37<04:20, 137.78it/s]
 29%|██▉       | 14592/50176 [01:39<04:08, 142.98it/s]
 30%|██▉       | 14848/50176 [01:41<04:20, 135.71it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.04it/s]
 31%|███       | 15360/50176 [01:45<04:11, 138.31it/s]
 31%|███       | 15616/50176 [01:47<04:15, 135.23it/s]
 32%|███▏      | 15872/50176 [01:49<04:15, 134.05it/s]
 32%|███▏      | 16128/50176 [01:50<04:07, 137.67it/s]
 33%|███▎      | 16384/50176 [01:52<04:04, 138.29it/s]
 33%|███▎      | 16640/50176 [01:54<04:09, 134.23it/s]
 34%|███▎      | 16896/50176 [01:56<03:58, 139.45it/s]
 34%|███▍      | 17152/50176 [01:58<03:51, 142.89it/s]
 35%|███▍      | 17408/50176 [01:59<03:46, 144.51it/s]
 35%|███▌      | 17664/50176 [02:01<03:42, 146.43it/s]
 36%|███▌      | 17920/50176 [02:03<03:37, 148.24it/s]
 36%|███▌      | 18176/50176 [02:04<03:37, 147.09it/s]
 37%|███▋      | 18432/50176 [02:06<03:35, 147.03it/s]
 37%|███▋      | 18688/50176 [02:08<03:34, 147.00it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.52it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.43it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.16it/s]
 39%|███▉      | 19712/50176 [02:16<03:50, 132.43it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.61it/s]
 40%|████      | 20224/50176 [02:19<03:32, 140.74it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.78it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.97it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.38it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 141.83it/s]
 43%|████▎     | 21504/50176 [02:28<03:16, 146.21it/s]
 43%|████▎     | 21760/50176 [02:30<03:12, 147.54it/s]
 44%|████▍     | 22016/50176 [02:31<03:07, 150.50it/s]
 44%|████▍     | 22272/50176 [02:33<03:06, 149.75it/s]
 45%|████▍     | 22528/50176 [02:35<03:03, 150.94it/s]
 45%|████▌     | 22784/50176 [02:37<03:09, 144.53it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.24it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.85it/s]
 47%|████▋     | 23552/50176 [02:43<03:15, 136.52it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.03it/s]
 48%|████▊     | 24064/50176 [02:46<03:03, 142.22it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.51it/s]
 49%|████▉     | 24576/50176 [02:49<02:55, 145.58it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.55it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 149.22it/s]
 51%|█████     | 25344/50176 [02:54<02:46, 149.50it/s]
 51%|█████     | 25600/50176 [02:56<02:48, 145.89it/s]
 52%|█████▏    | 25856/50176 [02:58<02:42, 149.96it/s]
 52%|█████▏    | 26112/50176 [03:00<02:43, 147.41it/s]
 53%|█████▎    | 26368/50176 [03:02<02:46, 142.94it/s]
 53%|█████▎    | 26624/50176 [03:03<02:42, 145.38it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.01it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 145.93it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.11it/s]
 55%|█████▌    | 27648/50176 [03:10<02:28, 152.19it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 141.93it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.76it/s]
 57%|█████▋    | 28416/50176 [03:16<02:31, 144.10it/s]
 57%|█████▋    | 28672/50176 [03:18<02:33, 140.13it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 142.69it/s]
 58%|█████▊    | 29184/50176 [03:21<02:24, 145.04it/s]
 59%|█████▊    | 29440/50176 [03:23<02:32, 136.01it/s]
 59%|█████▉    | 29696/50176 [03:25<02:34, 132.82it/s]
 60%|█████▉    | 29952/50176 [03:27<02:31, 133.45it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.43it/s]
 61%|██████    | 30464/50176 [03:30<02:17, 143.42it/s]
 61%|██████    | 30720/50176 [03:32<02:16, 142.74it/s]
 62%|██████▏   | 30976/50176 [03:34<02:12, 145.44it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.82it/s]
 63%|██████▎   | 31488/50176 [03:37<02:05, 149.10it/s]
 63%|██████▎   | 31744/50176 [03:39<02:03, 149.59it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.39it/s]
 64%|██████▍   | 32256/50176 [03:42<02:00, 149.11it/s]
 65%|██████▍   | 32512/50176 [03:44<02:02, 143.99it/s]
 65%|██████▌   | 32768/50176 [03:46<01:58, 147.32it/s]
 66%|██████▌   | 33024/50176 [03:48<01:58, 145.31it/s]
 66%|██████▋   | 33280/50176 [03:50<01:55, 146.23it/s]
 67%|██████▋   | 33536/50176 [03:51<01:57, 142.14it/s]
 67%|██████▋   | 33792/50176 [03:53<01:52, 146.13it/s]
 68%|██████▊   | 34048/50176 [03:55<01:47, 149.97it/s]
 68%|██████▊   | 34304/50176 [03:56<01:44, 151.59it/s]
 69%|██████▉   | 34560/50176 [03:58<01:47, 145.36it/s]
 69%|██████▉   | 34816/50176 [04:00<01:45, 145.60it/s]
 70%|██████▉   | 35072/50176 [04:02<01:44, 144.90it/s]
 70%|███████   | 35328/50176 [04:03<01:40, 148.13it/s]
 71%|███████   | 35584/50176 [04:05<01:40, 145.61it/s]
 71%|███████▏  | 35840/50176 [04:07<01:41, 141.13it/s]
 72%|███████▏  | 36096/50176 [04:09<01:38, 143.46it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 141.59it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 144.60it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.28it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 144.91it/s]
 74%|███████▍  | 37376/50176 [04:18<01:26, 148.75it/s]
 75%|███████▌  | 37632/50176 [04:19<01:23, 150.91it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 148.82it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.60it/s]
 77%|███████▋  | 38400/50176 [04:25<01:19, 148.60it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.14it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.23it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.73it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.49it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.13it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.71it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.90it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.86it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 137.74it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.44it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.73it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.22it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.74it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 151.92it/s]
 84%|████████▍ | 42240/50176 [04:51<00:53, 149.05it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.85it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.27it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.77it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.37it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.07it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.34it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 147.97it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.69it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 148.05it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 146.39it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 148.41it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 142.15it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.65it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 148.07it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.66it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.21it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.49it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 148.05it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.25it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.19it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.71it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 146.69it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 146.65it/s]
 96%|█████████▋| 48384/50176 [05:32<00:11, 149.85it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 152.06it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 151.97it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.77it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.41it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.65it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.57it/s]
100%|██████████| 50176/50176 [05:44<00:00, 181.80it/s]
100%|██████████| 50176/50176 [05:44<00:00, 145.82it/s]
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5654	imagenet-zeroshot-val-top5: 0.8378
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt
new file mode 100644
index 0000000000000000000000000000000000000000..01841a3e2fef1f662afe9e80a4cf338512304ed4
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8af949c18ef1ecd578e8913d80ad17053fb83fbcbe68cb859d20c2bcae7063e8
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..78170ac8666bebd46420f417af05eafa9be6d7fb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt).
+2022-12-21,10:01:07 | INFO | Model:
+2022-12-21,10:01:07 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:07 | INFO | Params:
+2022-12-21,10:01:07 | INFO |   batch_size: 256
+2022-12-21,10:01:07 | INFO |   beta1: 0.9
+2022-12-21,10:01:07 | INFO |   beta2: 0.98
+2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/checkpoints
+2022-12-21,10:01:07 | INFO |   copy_codebase: False
+2022-12-21,10:01:07 | INFO |   csv_caption_key: title
+2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:07 | INFO |   csv_separator: 	
+2022-12-21,10:01:07 | INFO |   dataset_resampled: False
+2022-12-21,10:01:07 | INFO |   dataset_type: auto
+2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:07 | INFO |   debug: False
+2022-12-21,10:01:07 | INFO |   device: cuda:0
+2022-12-21,10:01:07 | INFO |   dist_backend: nccl
+2022-12-21,10:01:07 | INFO |   dist_url: env://
+2022-12-21,10:01:07 | INFO |   distributed: False
+2022-12-21,10:01:07 | INFO |   epochs: 32
+2022-12-21,10:01:07 | INFO |   eps: 1e-06
+2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:07 | INFO |   gather_with_grad: False
+2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:07 | INFO |   horovod: False
+2022-12-21,10:01:07 | INFO |   image_mean: None
+2022-12-21,10:01:07 | INFO |   image_std: None
+2022-12-21,10:01:07 | INFO |   imagenet_v2: None
+2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:07 | INFO |   local_loss: False
+2022-12-21,10:01:07 | INFO |   local_rank: 0
+2022-12-21,10:01:07 | INFO |   lock_image: False
+2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:07 | INFO |   log_level: 20
+2022-12-21,10:01:07 | INFO |   log_local: False
+2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/out.log
+2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:07 | INFO |   lr: 0.0005
+2022-12-21,10:01:07 | INFO |   model: ViT-B-16
+2022-12-21,10:01:07 | INFO |   name: epoch_23.pt
+2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt
+2022-12-21,10:01:07 | INFO |   pretrained_image: False
+2022-12-21,10:01:07 | INFO |   rank: 0
+2022-12-21,10:01:07 | INFO |   report_to: 
+2022-12-21,10:01:07 | INFO |   resume: None
+2022-12-21,10:01:07 | INFO |   save_frequency: 1
+2022-12-21,10:01:07 | INFO |   save_most_recent: False
+2022-12-21,10:01:07 | INFO |   seed: 0
+2022-12-21,10:01:07 | INFO |   skip_scheduler: False
+2022-12-21,10:01:07 | INFO |   tensorboard: False
+2022-12-21,10:01:07 | INFO |   tensorboard_path: 
+2022-12-21,10:01:07 | INFO |   torchscript: False
+2022-12-21,10:01:07 | INFO |   trace: False
+2022-12-21,10:01:07 | INFO |   train_data: None
+2022-12-21,10:01:07 | INFO |   train_num_samples: None
+2022-12-21,10:01:07 | INFO |   use_bn_sync: False
+2022-12-21,10:01:07 | INFO |   val_data: None
+2022-12-21,10:01:07 | INFO |   val_frequency: 1
+2022-12-21,10:01:07 | INFO |   val_num_samples: None
+2022-12-21,10:01:07 | INFO |   wandb: False
+2022-12-21,10:01:07 | INFO |   wandb_notes: 
+2022-12-21,10:01:07 | INFO |   warmup: 10000
+2022-12-21,10:01:07 | INFO |   wd: 0.2
+2022-12-21,10:01:07 | INFO |   workers: 1
+2022-12-21,10:01:07 | INFO |   world_size: 1
+2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:07 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:23,  1.46it/s]
  1%|          | 6/1000 [00:00<01:41,  9.84it/s]
  1%|          | 11/1000 [00:00<00:55, 17.77it/s]
  2%|▏         | 17/1000 [00:00<00:37, 26.53it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.01it/s]
  3%|▎         | 29/1000 [00:01<00:24, 40.03it/s]
  4%|▎         | 35/1000 [00:01<00:21, 44.64it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.19it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.62it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.60it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.84it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.86it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.46it/s]
  8%|▊         | 77/1000 [00:02<00:16, 55.97it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.41it/s]
  9%|▉         | 89/1000 [00:02<00:16, 56.80it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.01it/s]
 10%|█         | 101/1000 [00:02<00:17, 51.99it/s]
 11%|█         | 107/1000 [00:02<00:16, 53.67it/s]
 11%|█▏        | 113/1000 [00:02<00:16, 54.87it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 55.66it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 56.16it/s]
 13%|█▎        | 131/1000 [00:03<00:15, 56.63it/s]
 14%|█▎        | 137/1000 [00:03<00:15, 56.82it/s]
 14%|█▍        | 143/1000 [00:03<00:15, 57.06it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.08it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.29it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.24it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.21it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.26it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.30it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.17it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.17it/s]
 20%|█▉        | 197/1000 [00:04<00:14, 57.13it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.01it/s]
 21%|██        | 209/1000 [00:04<00:13, 56.86it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 56.91it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 56.92it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.11it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.19it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.02it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.04it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 56.94it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.17it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.24it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.16it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.29it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.61it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.60it/s]
 30%|███       | 305/1000 [00:06<00:12, 57.61it/s]
 31%|███       | 311/1000 [00:06<00:11, 57.72it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 57.72it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 57.67it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.46it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.51it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 57.57it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 57.70it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.51it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 57.63it/s]
 36%|███▋      | 365/1000 [00:07<00:11, 57.59it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 57.71it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.55it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.13it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 56.88it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.06it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.25it/s]
 41%|████      | 407/1000 [00:07<00:10, 57.42it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 57.50it/s]
 42%|████▏     | 419/1000 [00:08<00:10, 57.54it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.52it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.69it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 57.72it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.52it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 57.57it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 57.65it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 57.63it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 57.61it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 57.73it/s]
 48%|████▊     | 479/1000 [00:09<00:09, 57.63it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 57.62it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.60it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.50it/s]
 50%|█████     | 503/1000 [00:09<00:08, 57.64it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.51it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 57.61it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 57.72it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 57.63it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 57.73it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 57.76it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.63it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.55it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 57.65it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.55it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.48it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 57.64it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 57.56it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.50it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 57.61it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.51it/s]
 61%|██████    | 611/1000 [00:11<00:06, 57.58it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 57.66it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 57.67it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 57.74it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.60it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.47it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 57.69it/s]
 65%|██████▌   | 653/1000 [00:12<00:06, 57.65it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 57.76it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 57.74it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 57.70it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 57.74it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 57.72it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.77it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.65it/s]
 70%|███████   | 701/1000 [00:12<00:05, 57.76it/s]
 71%|███████   | 707/1000 [00:13<00:05, 57.75it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 57.75it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 57.77it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 57.69it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 57.63it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 57.69it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 57.64it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 57.73it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 57.85it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 57.69it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 57.77it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 57.77it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 57.60it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 57.68it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.52it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.54it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.58it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.55it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 57.56it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 57.62it/s]
 83%|████████▎ | 827/1000 [00:15<00:03, 57.63it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 57.82it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 57.87it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 57.72it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 57.78it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 57.72it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 57.79it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 57.71it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 57.72it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 57.76it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.61it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 57.71it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 57.74it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 57.73it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 57.43it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 57.53it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.55it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 57.71it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 57.82it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 57.82it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 57.81it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 57.84it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 57.78it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 57.54it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 57.71it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 57.82it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 57.89it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 57.85it/s]
100%|█████████▉| 995/1000 [00:18<00:00, 57.75it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.21it/s]
+2022-12-21,10:01:25 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:12, 133.90it/s]
  1%|          | 512/50176 [00:03<05:25, 152.35it/s]
  2%|▏         | 768/50176 [00:04<04:54, 167.53it/s]
  2%|▏         | 1024/50176 [00:06<04:47, 171.03it/s]
  3%|▎         | 1280/50176 [00:08<05:07, 159.08it/s]
  3%|▎         | 1536/50176 [00:10<05:33, 145.88it/s]
  4%|▎         | 1792/50176 [00:11<05:35, 144.10it/s]
  4%|▍         | 2048/50176 [00:13<05:32, 144.65it/s]
  5%|▍         | 2304/50176 [00:15<05:34, 143.28it/s]
  5%|▌         | 2560/50176 [00:17<05:30, 144.08it/s]
  6%|▌         | 2816/50176 [00:19<05:31, 142.69it/s]
  6%|▌         | 3072/50176 [00:20<05:25, 144.54it/s]
  7%|▋         | 3328/50176 [00:22<05:22, 145.25it/s]
  7%|▋         | 3584/50176 [00:24<05:15, 147.57it/s]
  8%|▊         | 3840/50176 [00:26<05:27, 141.62it/s]
  8%|▊         | 4096/50176 [00:27<05:25, 141.71it/s]
  9%|▊         | 4352/50176 [00:29<05:19, 143.57it/s]
  9%|▉         | 4608/50176 [00:31<05:14, 144.88it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.68it/s]
 10%|█         | 5120/50176 [00:34<05:06, 147.01it/s]
 11%|█         | 5376/50176 [00:36<05:00, 148.86it/s]
 11%|█         | 5632/50176 [00:38<04:57, 149.82it/s]
 12%|█▏        | 5888/50176 [00:39<04:55, 149.71it/s]
 12%|█▏        | 6144/50176 [00:42<05:20, 137.54it/s]
 13%|█▎        | 6400/50176 [00:43<05:10, 141.09it/s]
 13%|█▎        | 6656/50176 [00:45<05:04, 143.03it/s]
 14%|█▍        | 6912/50176 [00:47<05:01, 143.72it/s]
 14%|█▍        | 7168/50176 [00:49<04:56, 145.13it/s]
 15%|█▍        | 7424/50176 [00:50<04:51, 146.69it/s]
 15%|█▌        | 7680/50176 [00:52<04:50, 146.26it/s]
 16%|█▌        | 7936/50176 [00:54<04:43, 149.07it/s]
 16%|█▋        | 8192/50176 [00:55<04:44, 147.81it/s]
 17%|█▋        | 8448/50176 [00:57<04:40, 148.74it/s]
 17%|█▋        | 8704/50176 [00:59<04:37, 149.42it/s]
 18%|█▊        | 8960/50176 [01:01<04:45, 144.22it/s]
 18%|█▊        | 9216/50176 [01:02<04:37, 147.56it/s]
 19%|█▉        | 9472/50176 [01:04<04:35, 147.92it/s]
 19%|█▉        | 9728/50176 [01:06<04:40, 144.35it/s]
 20%|█▉        | 9984/50176 [01:08<04:32, 147.72it/s]
 20%|██        | 10240/50176 [01:09<04:26, 149.75it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.23it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.31it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 148.21it/s]
 22%|██▏       | 11264/50176 [01:16<04:21, 148.76it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.91it/s]
 23%|██▎       | 11776/50176 [01:20<04:13, 151.38it/s]
 24%|██▍       | 12032/50176 [01:21<04:11, 151.83it/s]
 24%|██▍       | 12288/50176 [01:23<04:09, 152.14it/s]
 25%|██▌       | 12544/50176 [01:25<04:06, 152.61it/s]
 26%|██▌       | 12800/50176 [01:26<04:04, 152.87it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.85it/s]
 27%|██▋       | 13312/50176 [01:30<04:06, 149.75it/s]
 27%|██▋       | 13568/50176 [01:32<04:15, 143.22it/s]
 28%|██▊       | 13824/50176 [01:34<04:17, 141.08it/s]
 28%|██▊       | 14080/50176 [01:35<04:15, 141.25it/s]
 29%|██▊       | 14336/50176 [01:37<04:20, 137.57it/s]
 29%|██▉       | 14592/50176 [01:39<04:08, 143.07it/s]
 30%|██▉       | 14848/50176 [01:41<04:20, 135.77it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.13it/s]
 31%|███       | 15360/50176 [01:45<04:11, 138.37it/s]
 31%|███       | 15616/50176 [01:47<04:14, 135.66it/s]
 32%|███▏      | 15872/50176 [01:49<04:16, 133.95it/s]
 32%|███▏      | 16128/50176 [01:50<04:07, 137.54it/s]
 33%|███▎      | 16384/50176 [01:52<04:04, 138.21it/s]
 33%|███▎      | 16640/50176 [01:54<04:10, 134.12it/s]
 34%|███▎      | 16896/50176 [01:56<03:58, 139.43it/s]
 34%|███▍      | 17152/50176 [01:58<03:51, 142.85it/s]
 35%|███▍      | 17408/50176 [01:59<03:46, 144.43it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.54it/s]
 36%|███▌      | 17920/50176 [02:03<03:37, 148.10it/s]
 36%|███▌      | 18176/50176 [02:04<03:37, 147.20it/s]
 37%|███▋      | 18432/50176 [02:06<03:35, 147.51it/s]
 37%|███▋      | 18688/50176 [02:08<03:34, 146.86it/s]
 38%|███▊      | 18944/50176 [02:10<03:40, 141.91it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.57it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.17it/s]
 39%|███▉      | 19712/50176 [02:16<03:50, 132.01it/s]
 40%|███▉      | 19968/50176 [02:18<03:43, 135.21it/s]
 40%|████      | 20224/50176 [02:19<03:32, 141.02it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.70it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.90it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.59it/s]
 42%|████▏     | 21248/50176 [02:27<03:24, 141.53it/s]
 43%|████▎     | 21504/50176 [02:28<03:16, 146.26it/s]
 43%|████▎     | 21760/50176 [02:30<03:12, 147.70it/s]
 44%|████▍     | 22016/50176 [02:31<03:07, 150.46it/s]
 44%|████▍     | 22272/50176 [02:33<03:07, 149.06it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.15it/s]
 45%|████▌     | 22784/50176 [02:37<03:09, 144.43it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.36it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.56it/s]
 47%|████▋     | 23552/50176 [02:43<03:13, 137.29it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.06it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 142.77it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.28it/s]
 49%|████▉     | 24576/50176 [02:49<02:56, 145.39it/s]
 49%|████▉     | 24832/50176 [02:51<02:51, 147.70it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 149.21it/s]
 51%|█████     | 25344/50176 [02:54<02:45, 149.67it/s]
 51%|█████     | 25600/50176 [02:56<02:49, 145.31it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.61it/s]
 52%|█████▏    | 26112/50176 [03:00<02:43, 146.80it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 143.59it/s]
 53%|█████▎    | 26624/50176 [03:03<02:41, 145.41it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.03it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 145.91it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.27it/s]
 55%|█████▌    | 27648/50176 [03:10<02:28, 152.17it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 141.94it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.74it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.16it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.69it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 142.63it/s]
 58%|█████▊    | 29184/50176 [03:21<02:24, 145.01it/s]
 59%|█████▊    | 29440/50176 [03:23<02:32, 135.84it/s]
 59%|█████▉    | 29696/50176 [03:25<02:34, 132.60it/s]
 60%|█████▉    | 29952/50176 [03:27<02:31, 133.47it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.49it/s]
 61%|██████    | 30464/50176 [03:30<02:17, 143.41it/s]
 61%|██████    | 30720/50176 [03:32<02:16, 142.74it/s]
 62%|██████▏   | 30976/50176 [03:34<02:12, 145.43it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.75it/s]
 63%|██████▎   | 31488/50176 [03:37<02:05, 148.39it/s]
 63%|██████▎   | 31744/50176 [03:39<02:03, 149.83it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.47it/s]
 64%|██████▍   | 32256/50176 [03:42<02:00, 149.18it/s]
 65%|██████▍   | 32512/50176 [03:44<02:02, 143.87it/s]
 65%|██████▌   | 32768/50176 [03:46<01:58, 147.45it/s]
 66%|██████▌   | 33024/50176 [03:48<01:57, 145.42it/s]
 66%|██████▋   | 33280/50176 [03:50<01:55, 146.49it/s]
 67%|██████▋   | 33536/50176 [03:51<01:56, 142.66it/s]
 67%|██████▋   | 33792/50176 [03:53<01:52, 146.15it/s]
 68%|██████▊   | 34048/50176 [03:55<01:47, 149.94it/s]
 68%|██████▊   | 34304/50176 [03:56<01:44, 151.58it/s]
 69%|██████▉   | 34560/50176 [03:58<01:47, 145.11it/s]
 69%|██████▉   | 34816/50176 [04:00<01:45, 145.68it/s]
 70%|██████▉   | 35072/50176 [04:02<01:44, 144.24it/s]
 70%|███████   | 35328/50176 [04:03<01:40, 147.96it/s]
 71%|███████   | 35584/50176 [04:05<01:40, 145.71it/s]
 71%|███████▏  | 35840/50176 [04:07<01:41, 141.82it/s]
 72%|███████▏  | 36096/50176 [04:09<01:38, 143.26it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 141.56it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 144.68it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 143.09it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 144.82it/s]
 74%|███████▍  | 37376/50176 [04:18<01:26, 148.79it/s]
 75%|███████▌  | 37632/50176 [04:19<01:23, 150.95it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 148.90it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.60it/s]
 77%|███████▋  | 38400/50176 [04:25<01:19, 148.59it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.15it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.26it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.63it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.50it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.15it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.73it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.27it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 144.05it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 137.92it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.54it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.67it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.99it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.46it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 151.80it/s]
 84%|████████▍ | 42240/50176 [04:51<00:53, 148.99it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.83it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 148.56it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.95it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.50it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.16it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.24it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 147.45it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.90it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 148.17it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 146.64it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 148.44it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 141.90it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.75it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 148.21it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.78it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.56it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.44it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 148.04it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.06it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.28it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.64it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 146.68it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 147.21it/s]
 96%|█████████▋| 48384/50176 [05:32<00:11, 149.68it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 152.02it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 151.78it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.63it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.31it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.55it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.68it/s]
100%|██████████| 50176/50176 [05:44<00:00, 181.77it/s]
100%|██████████| 50176/50176 [05:44<00:00, 145.82it/s]
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5731	imagenet-zeroshot-val-top5: 0.8431
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt
new file mode 100644
index 0000000000000000000000000000000000000000..df56d8eef35c510d8fcacbc61ffb10997fcd43d4
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90a6047d8b86f2d7dc8180513aadc52c1ce9ddff3715402e679a8d9b31363b29
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1ad0e74a7ac670c7a28a371ad75348f3c7da4fa1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt).
+2022-12-21,10:07:24 | INFO | Model:
+2022-12-21,10:07:24 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:24 | INFO | Params:
+2022-12-21,10:07:24 | INFO |   batch_size: 256
+2022-12-21,10:07:24 | INFO |   beta1: 0.9
+2022-12-21,10:07:24 | INFO |   beta2: 0.98
+2022-12-21,10:07:24 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/checkpoints
+2022-12-21,10:07:24 | INFO |   copy_codebase: False
+2022-12-21,10:07:24 | INFO |   csv_caption_key: title
+2022-12-21,10:07:24 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:24 | INFO |   csv_separator: 	
+2022-12-21,10:07:24 | INFO |   dataset_resampled: False
+2022-12-21,10:07:24 | INFO |   dataset_type: auto
+2022-12-21,10:07:24 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:24 | INFO |   debug: False
+2022-12-21,10:07:24 | INFO |   device: cuda:0
+2022-12-21,10:07:24 | INFO |   dist_backend: nccl
+2022-12-21,10:07:24 | INFO |   dist_url: env://
+2022-12-21,10:07:24 | INFO |   distributed: False
+2022-12-21,10:07:24 | INFO |   epochs: 32
+2022-12-21,10:07:24 | INFO |   eps: 1e-06
+2022-12-21,10:07:24 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:24 | INFO |   gather_with_grad: False
+2022-12-21,10:07:24 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:24 | INFO |   horovod: False
+2022-12-21,10:07:24 | INFO |   image_mean: None
+2022-12-21,10:07:24 | INFO |   image_std: None
+2022-12-21,10:07:24 | INFO |   imagenet_v2: None
+2022-12-21,10:07:24 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:24 | INFO |   local_loss: False
+2022-12-21,10:07:24 | INFO |   local_rank: 0
+2022-12-21,10:07:24 | INFO |   lock_image: False
+2022-12-21,10:07:24 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:24 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:24 | INFO |   log_level: 20
+2022-12-21,10:07:24 | INFO |   log_local: False
+2022-12-21,10:07:24 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/out.log
+2022-12-21,10:07:24 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:24 | INFO |   lr: 0.0005
+2022-12-21,10:07:24 | INFO |   model: ViT-B-16
+2022-12-21,10:07:24 | INFO |   name: epoch_24.pt
+2022-12-21,10:07:24 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:24 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:24 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:24 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt
+2022-12-21,10:07:24 | INFO |   pretrained_image: False
+2022-12-21,10:07:24 | INFO |   rank: 0
+2022-12-21,10:07:24 | INFO |   report_to: 
+2022-12-21,10:07:24 | INFO |   resume: None
+2022-12-21,10:07:24 | INFO |   save_frequency: 1
+2022-12-21,10:07:24 | INFO |   save_most_recent: False
+2022-12-21,10:07:24 | INFO |   seed: 0
+2022-12-21,10:07:24 | INFO |   skip_scheduler: False
+2022-12-21,10:07:24 | INFO |   tensorboard: False
+2022-12-21,10:07:24 | INFO |   tensorboard_path: 
+2022-12-21,10:07:24 | INFO |   torchscript: False
+2022-12-21,10:07:24 | INFO |   trace: False
+2022-12-21,10:07:24 | INFO |   train_data: None
+2022-12-21,10:07:24 | INFO |   train_num_samples: None
+2022-12-21,10:07:24 | INFO |   use_bn_sync: False
+2022-12-21,10:07:24 | INFO |   val_data: None
+2022-12-21,10:07:24 | INFO |   val_frequency: 1
+2022-12-21,10:07:24 | INFO |   val_num_samples: None
+2022-12-21,10:07:24 | INFO |   wandb: False
+2022-12-21,10:07:24 | INFO |   wandb_notes: 
+2022-12-21,10:07:24 | INFO |   warmup: 10000
+2022-12-21,10:07:24 | INFO |   wd: 0.2
+2022-12-21,10:07:24 | INFO |   workers: 1
+2022-12-21,10:07:24 | INFO |   world_size: 1
+2022-12-21,10:07:24 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:25 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:29,  1.59it/s]
  1%|          | 6/1000 [00:00<01:35, 10.39it/s]
  1%|          | 12/1000 [00:00<00:49, 19.83it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.32it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.47it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.21it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.43it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.67it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.11it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.90it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.18it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.14it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.78it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.24it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.76it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.02it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.39it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.52it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.74it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.88it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.79it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.84it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.81it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.84it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.91it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.96it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.69it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.62it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.58it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.52it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.49it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.43it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.26it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.16it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.33it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.28it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.46it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.45it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.39it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.45it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.54it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.59it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.41it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.65it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.88it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.87it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.84it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.01it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.99it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.92it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.74it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.83it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.95it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.04it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.82it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.80it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.88it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.96it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.52it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.58it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.69it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.78it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.80it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.84it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.88it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.89it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.01it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.02it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.81it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.89it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.96it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.94it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.88it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.00it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.91it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.90it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.92it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.80it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.86it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.79it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.86it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.91it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.99it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.94it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.04it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.05it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.94it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.94it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.02it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.88it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.59it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.77it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.79it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.74it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.87it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.79it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.81it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.93it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.03it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.86it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.79it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.95it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 57.91it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.98it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.04it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.97it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.96it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.97it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.89it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.97it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.98it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.97it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.99it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.05it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.01it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.08it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.09it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.94it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.05it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.97it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.95it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.01it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.81it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.80it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.85it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.83it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.83it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.85it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 57.95it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.05it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.12it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.04it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.10it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.04it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.07it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.01it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.04it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.03it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.90it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.00it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.02it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.94it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.04it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.08it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.08it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.11it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.85it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.81it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.76it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.91it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.02it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.08it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.07it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.88it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.72it/s]
+2022-12-21,10:07:43 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:57, 139.57it/s]
  1%|          | 512/50176 [00:03<05:18, 155.78it/s]
  2%|▏         | 768/50176 [00:04<05:14, 156.94it/s]
  2%|▏         | 1024/50176 [00:06<05:25, 151.21it/s]
  3%|▎         | 1280/50176 [00:08<05:32, 147.06it/s]
  3%|▎         | 1536/50176 [00:10<05:51, 138.46it/s]
  4%|▎         | 1792/50176 [00:12<05:51, 137.75it/s]
  4%|▍         | 2048/50176 [00:14<05:44, 139.80it/s]
  5%|▍         | 2304/50176 [00:16<05:43, 139.22it/s]
  5%|▌         | 2560/50176 [00:17<05:36, 141.69it/s]
  6%|▌         | 2816/50176 [00:19<05:36, 140.65it/s]
  6%|▌         | 3072/50176 [00:21<05:31, 141.92it/s]
  7%|▋         | 3328/50176 [00:23<05:25, 143.91it/s]
  7%|▋         | 3584/50176 [00:24<05:18, 146.31it/s]
  8%|▊         | 3840/50176 [00:26<05:30, 140.39it/s]
  8%|▊         | 4096/50176 [00:28<05:27, 140.49it/s]
  9%|▊         | 4352/50176 [00:30<05:21, 142.40it/s]
  9%|▉         | 4608/50176 [00:32<05:16, 144.01it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.87it/s]
 10%|█         | 5120/50176 [00:35<05:05, 147.59it/s]
 11%|█         | 5376/50176 [00:37<05:03, 147.84it/s]
 11%|█         | 5632/50176 [00:39<05:01, 147.70it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.42it/s]
 12%|█▏        | 6144/50176 [00:42<05:22, 136.50it/s]
 13%|█▎        | 6400/50176 [00:44<05:12, 140.30it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.57it/s]
 14%|█▍        | 6912/50176 [00:48<05:02, 142.98it/s]
 14%|█▍        | 7168/50176 [00:49<04:57, 144.73it/s]
 15%|█▍        | 7424/50176 [00:51<04:50, 147.05it/s]
 15%|█▌        | 7680/50176 [00:53<04:49, 147.00it/s]
 16%|█▌        | 7936/50176 [00:54<04:39, 150.98it/s]
 16%|█▋        | 8192/50176 [00:56<04:38, 151.00it/s]
 17%|█▋        | 8448/50176 [00:58<04:35, 151.64it/s]
 17%|█▋        | 8704/50176 [00:59<04:32, 152.10it/s]
 18%|█▊        | 8960/50176 [01:01<04:39, 147.24it/s]
 18%|█▊        | 9216/50176 [01:03<04:32, 150.36it/s]
 19%|█▉        | 9472/50176 [01:05<04:31, 149.99it/s]
 19%|█▉        | 9728/50176 [01:06<04:36, 146.15it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 148.27it/s]
 20%|██        | 10240/50176 [01:10<04:27, 149.48it/s]
 21%|██        | 10496/50176 [01:12<04:24, 150.29it/s]
 21%|██▏       | 10752/50176 [01:13<04:32, 144.93it/s]
 22%|██▏       | 11008/50176 [01:15<04:27, 146.54it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.12it/s]
 23%|██▎       | 11520/50176 [01:18<04:17, 150.39it/s]
 23%|██▎       | 11776/50176 [01:20<04:15, 150.23it/s]
 24%|██▍       | 12032/50176 [01:22<04:12, 150.94it/s]
 24%|██▍       | 12288/50176 [01:23<04:08, 152.64it/s]
 25%|██▌       | 12544/50176 [01:25<04:04, 153.73it/s]
 26%|██▌       | 12800/50176 [01:27<04:02, 154.06it/s]
 26%|██▌       | 13056/50176 [01:28<03:59, 154.98it/s]
 27%|██▋       | 13312/50176 [01:30<04:02, 152.08it/s]
 27%|██▋       | 13568/50176 [01:32<04:10, 145.89it/s]
 28%|██▊       | 13824/50176 [01:34<04:14, 142.90it/s]
 28%|██▊       | 14080/50176 [01:36<04:43, 127.51it/s]
 29%|██▊       | 14336/50176 [01:38<04:38, 128.82it/s]
 29%|██▉       | 14592/50176 [01:40<04:19, 137.20it/s]
 30%|██▉       | 14848/50176 [01:42<04:27, 131.96it/s]
 30%|███       | 15104/50176 [01:44<04:26, 131.69it/s]
 31%|███       | 15360/50176 [01:46<04:15, 136.42it/s]
 31%|███       | 15616/50176 [01:48<04:08, 138.90it/s]
 32%|███▏      | 15872/50176 [01:49<04:11, 136.26it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.68it/s]
 33%|███▎      | 16384/50176 [01:53<04:02, 139.23it/s]
 33%|███▎      | 16640/50176 [01:55<04:06, 136.19it/s]
 34%|███▎      | 16896/50176 [01:57<03:57, 140.15it/s]
 34%|███▍      | 17152/50176 [01:58<03:48, 144.33it/s]
 35%|███▍      | 17408/50176 [02:00<03:44, 145.96it/s]
 35%|███▌      | 17664/50176 [02:02<03:39, 147.79it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.44it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.24it/s]
 37%|███▋      | 18432/50176 [02:07<03:32, 149.51it/s]
 37%|███▋      | 18688/50176 [02:09<03:31, 148.89it/s]
 38%|███▊      | 18944/50176 [02:11<03:38, 142.99it/s]
 38%|███▊      | 19200/50176 [02:13<03:49, 134.82it/s]
 39%|███▉      | 19456/50176 [02:15<03:52, 132.19it/s]
 39%|███▉      | 19712/50176 [02:17<03:50, 132.32it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.63it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.54it/s]
 41%|████      | 20480/50176 [02:22<03:25, 144.84it/s]
 41%|████▏     | 20736/50176 [02:24<03:23, 145.02it/s]
 42%|████▏     | 20992/50176 [02:25<03:17, 147.53it/s]
 42%|████▏     | 21248/50176 [02:27<03:21, 143.28it/s]
 43%|████▎     | 21504/50176 [02:29<03:14, 147.57it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.31it/s]
 44%|████▍     | 22016/50176 [02:32<03:06, 150.97it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.59it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.61it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.69it/s]
 46%|████▌     | 23040/50176 [02:39<03:07, 144.40it/s]
 46%|████▋     | 23296/50176 [02:41<03:10, 140.75it/s]
 47%|████▋     | 23552/50176 [02:43<03:12, 138.37it/s]
 47%|████▋     | 23808/50176 [02:45<03:05, 142.01it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 143.14it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.54it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.84it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 149.04it/s]
 50%|█████     | 25088/50176 [02:53<02:47, 149.64it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.88it/s]
 51%|█████     | 25600/50176 [02:57<02:46, 147.53it/s]
 52%|█████▏    | 25856/50176 [02:58<02:40, 151.16it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.39it/s]
 53%|█████▎    | 26368/50176 [03:02<02:44, 144.80it/s]
 53%|█████▎    | 26624/50176 [03:04<02:40, 146.84it/s]
 54%|█████▎    | 26880/50176 [03:05<02:38, 147.03it/s]
 54%|█████▍    | 27136/50176 [03:07<02:35, 147.77it/s]
 55%|█████▍    | 27392/50176 [03:09<02:33, 148.16it/s]
 55%|█████▌    | 27648/50176 [03:10<02:25, 155.23it/s]
 56%|█████▌    | 27904/50176 [03:12<02:34, 143.81it/s]
 56%|█████▌    | 28160/50176 [03:14<02:30, 145.82it/s]
 57%|█████▋    | 28416/50176 [03:16<02:29, 145.56it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.29it/s]
 58%|█████▊    | 28928/50176 [03:20<02:27, 144.10it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 146.43it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.01it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.39it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.22it/s]
 60%|██████    | 30208/50176 [03:29<02:24, 138.23it/s]
 61%|██████    | 30464/50176 [03:30<02:16, 144.56it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 143.61it/s]
 62%|██████▏   | 30976/50176 [03:34<02:11, 145.79it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.98it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.13it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.55it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.66it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 149.86it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 145.40it/s]
 65%|██████▌   | 32768/50176 [03:46<01:57, 148.49it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 146.66it/s]
 66%|██████▋   | 33280/50176 [03:49<01:54, 148.10it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 144.90it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.80it/s]
 68%|██████▊   | 34048/50176 [03:55<01:45, 152.31it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 155.39it/s]
 69%|██████▉   | 34560/50176 [03:58<01:46, 147.03it/s]
 69%|██████▉   | 34816/50176 [04:00<01:44, 146.75it/s]
 70%|██████▉   | 35072/50176 [04:02<01:43, 146.10it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 148.64it/s]
 71%|███████   | 35584/50176 [04:05<01:39, 147.23it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.57it/s]
 72%|███████▏  | 36096/50176 [04:09<01:36, 145.58it/s]
 72%|███████▏  | 36352/50176 [04:10<01:37, 142.49it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.24it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 142.43it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 143.62it/s]
 74%|███████▍  | 37376/50176 [04:17<01:26, 148.53it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.29it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 149.19it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.81it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.34it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.44it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 144.79it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.48it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.64it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.33it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 146.97it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.65it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 144.15it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.57it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.10it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.65it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.88it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 152.06it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 152.42it/s]
 84%|████████▍ | 42240/50176 [04:50<00:53, 148.90it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 150.10it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.20it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 151.39it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 151.03it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.10it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.28it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.08it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.31it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.30it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 145.63it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.36it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 140.62it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.55it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 147.39it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 147.30it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.50it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.69it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 148.32it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.21it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 145.58it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.74it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.32it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 147.04it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 147.49it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.11it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 150.29it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 146.50it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 144.81it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.52it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.22it/s]
100%|██████████| 50176/50176 [05:43<00:00, 181.19it/s]
100%|██████████| 50176/50176 [05:43<00:00, 145.88it/s]
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5730	imagenet-zeroshot-val-top5: 0.8418
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt
new file mode 100644
index 0000000000000000000000000000000000000000..854c9d46eea2d49c4223396ddb1fd96fd597c748
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f510973f6c6b2dcd96e71cad23646313e1451c68912be2c02e0075cb8e05021
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..29fd2397ee889dc061e16941af88cb40b523397e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt).
+2022-12-21,10:07:24 | INFO | Model:
+2022-12-21,10:07:24 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:24 | INFO | Params:
+2022-12-21,10:07:24 | INFO |   batch_size: 256
+2022-12-21,10:07:24 | INFO |   beta1: 0.9
+2022-12-21,10:07:24 | INFO |   beta2: 0.98
+2022-12-21,10:07:24 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/checkpoints
+2022-12-21,10:07:24 | INFO |   copy_codebase: False
+2022-12-21,10:07:24 | INFO |   csv_caption_key: title
+2022-12-21,10:07:24 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:24 | INFO |   csv_separator: 	
+2022-12-21,10:07:24 | INFO |   dataset_resampled: False
+2022-12-21,10:07:24 | INFO |   dataset_type: auto
+2022-12-21,10:07:24 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:24 | INFO |   debug: False
+2022-12-21,10:07:24 | INFO |   device: cuda:0
+2022-12-21,10:07:24 | INFO |   dist_backend: nccl
+2022-12-21,10:07:24 | INFO |   dist_url: env://
+2022-12-21,10:07:24 | INFO |   distributed: False
+2022-12-21,10:07:24 | INFO |   epochs: 32
+2022-12-21,10:07:24 | INFO |   eps: 1e-06
+2022-12-21,10:07:24 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:24 | INFO |   gather_with_grad: False
+2022-12-21,10:07:24 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:24 | INFO |   horovod: False
+2022-12-21,10:07:24 | INFO |   image_mean: None
+2022-12-21,10:07:24 | INFO |   image_std: None
+2022-12-21,10:07:24 | INFO |   imagenet_v2: None
+2022-12-21,10:07:24 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:24 | INFO |   local_loss: False
+2022-12-21,10:07:24 | INFO |   local_rank: 0
+2022-12-21,10:07:24 | INFO |   lock_image: False
+2022-12-21,10:07:24 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:24 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:24 | INFO |   log_level: 20
+2022-12-21,10:07:24 | INFO |   log_local: False
+2022-12-21,10:07:24 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/out.log
+2022-12-21,10:07:24 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:24 | INFO |   lr: 0.0005
+2022-12-21,10:07:24 | INFO |   model: ViT-B-16
+2022-12-21,10:07:24 | INFO |   name: epoch_25.pt
+2022-12-21,10:07:24 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:24 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:24 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:24 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt
+2022-12-21,10:07:24 | INFO |   pretrained_image: False
+2022-12-21,10:07:24 | INFO |   rank: 0
+2022-12-21,10:07:24 | INFO |   report_to: 
+2022-12-21,10:07:24 | INFO |   resume: None
+2022-12-21,10:07:24 | INFO |   save_frequency: 1
+2022-12-21,10:07:24 | INFO |   save_most_recent: False
+2022-12-21,10:07:24 | INFO |   seed: 0
+2022-12-21,10:07:24 | INFO |   skip_scheduler: False
+2022-12-21,10:07:24 | INFO |   tensorboard: False
+2022-12-21,10:07:24 | INFO |   tensorboard_path: 
+2022-12-21,10:07:24 | INFO |   torchscript: False
+2022-12-21,10:07:24 | INFO |   trace: False
+2022-12-21,10:07:24 | INFO |   train_data: None
+2022-12-21,10:07:24 | INFO |   train_num_samples: None
+2022-12-21,10:07:24 | INFO |   use_bn_sync: False
+2022-12-21,10:07:24 | INFO |   val_data: None
+2022-12-21,10:07:24 | INFO |   val_frequency: 1
+2022-12-21,10:07:24 | INFO |   val_num_samples: None
+2022-12-21,10:07:24 | INFO |   wandb: False
+2022-12-21,10:07:24 | INFO |   wandb_notes: 
+2022-12-21,10:07:24 | INFO |   warmup: 10000
+2022-12-21,10:07:24 | INFO |   wd: 0.2
+2022-12-21,10:07:24 | INFO |   workers: 1
+2022-12-21,10:07:24 | INFO |   world_size: 1
+2022-12-21,10:07:24 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:24 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:24 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<09:58,  1.67it/s]
  1%|          | 7/1000 [00:00<01:19, 12.43it/s]
  1%|▏         | 13/1000 [00:00<00:45, 21.67it/s]
  2%|▏         | 19/1000 [00:00<00:32, 30.06it/s]
  2%|▎         | 25/1000 [00:01<00:26, 36.91it/s]
  3%|▎         | 31/1000 [00:01<00:22, 42.46it/s]
  4%|▎         | 37/1000 [00:01<00:20, 46.63it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.82it/s]
  5%|▍         | 49/1000 [00:01<00:18, 52.13it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.70it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.92it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.68it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.21it/s]
  8%|▊         | 79/1000 [00:01<00:16, 56.50it/s]
  8%|▊         | 85/1000 [00:02<00:16, 56.93it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.16it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.40it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.58it/s]
 11%|█         | 109/1000 [00:02<00:15, 57.82it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.01it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.93it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 57.93it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 57.93it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.90it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 57.95it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 57.99it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 57.75it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 57.79it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.58it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.60it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.62it/s]
 19%|█▉        | 193/1000 [00:03<00:14, 57.60it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.51it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.35it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.17it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.36it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.36it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.55it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.52it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.36it/s]
 25%|██▌       | 253/1000 [00:05<00:15, 49.55it/s]
 26%|██▌       | 259/1000 [00:05<00:14, 51.80it/s]
 26%|██▋       | 265/1000 [00:05<00:13, 53.43it/s]
 27%|██▋       | 271/1000 [00:05<00:13, 54.50it/s]
 28%|██▊       | 277/1000 [00:05<00:13, 55.50it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 56.27it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 56.84it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 57.30it/s]
 30%|███       | 301/1000 [00:05<00:12, 57.52it/s]
 31%|███       | 307/1000 [00:05<00:12, 57.61it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 57.91it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.01it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 57.89it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 57.78it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 57.94it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 57.97it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 57.91it/s]
 36%|███▌      | 361/1000 [00:06<00:11, 57.91it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 57.97it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 57.85it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.69it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.75it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.93it/s]
 41%|████      | 409/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.08it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.09it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.21it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.17it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 57.91it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.05it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.10it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.02it/s]
 48%|████▊     | 475/1000 [00:08<00:09, 58.19it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.01it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.05it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 57.96it/s]
 50%|█████     | 505/1000 [00:09<00:08, 57.92it/s]
 51%|█████     | 511/1000 [00:09<00:08, 57.85it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.00it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 57.95it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.02it/s]
 54%|█████▎    | 535/1000 [00:09<00:08, 58.08it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.13it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.05it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.03it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 57.94it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.12it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.02it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.14it/s]
 61%|██████    | 607/1000 [00:11<00:06, 57.99it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.17it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.14it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.12it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.03it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.11it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.11it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.14it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.20it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.18it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.10it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.16it/s]
 71%|███████   | 709/1000 [00:12<00:05, 58.18it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 58.15it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.07it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.05it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.11it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.16it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.20it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.06it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.15it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.14it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.15it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.14it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 57.92it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 57.90it/s]
 80%|████████  | 805/1000 [00:14<00:03, 57.96it/s]
 81%|████████  | 811/1000 [00:14<00:03, 57.93it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 57.99it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.11it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.27it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.14it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.20it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.18it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.19it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.15it/s]
 89%|████████▉ | 889/1000 [00:16<00:01, 58.07it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.08it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.12it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.19it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.18it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.18it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.35it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.19it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 57.96it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.82it/s]
+2022-12-21,10:07:42 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:23, 112.65it/s]
  1%|          | 512/50176 [00:04<06:27, 128.24it/s]
  2%|▏         | 768/50176 [00:05<06:05, 135.17it/s]
  2%|▏         | 1024/50176 [00:07<05:54, 138.77it/s]
  3%|▎         | 1280/50176 [00:09<05:50, 139.32it/s]
  3%|▎         | 1536/50176 [00:11<06:04, 133.59it/s]
  4%|▎         | 1792/50176 [00:13<05:58, 134.91it/s]
  4%|▍         | 2048/50176 [00:15<05:48, 137.92it/s]
  5%|▍         | 2304/50176 [00:16<05:47, 137.89it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.17it/s]
  6%|▌         | 2816/50176 [00:20<05:39, 139.61it/s]
  6%|▌         | 3072/50176 [00:22<05:31, 141.99it/s]
  7%|▋         | 3328/50176 [00:24<05:27, 143.14it/s]
  7%|▋         | 3584/50176 [00:25<05:19, 145.78it/s]
  8%|▊         | 3840/50176 [00:27<05:29, 140.56it/s]
  8%|▊         | 4096/50176 [00:29<05:29, 139.97it/s]
  9%|▊         | 4352/50176 [00:31<05:22, 142.16it/s]
  9%|▉         | 4608/50176 [00:33<05:16, 143.83it/s]
 10%|▉         | 4864/50176 [00:34<05:08, 146.72it/s]
 10%|█         | 5120/50176 [00:36<05:05, 147.39it/s]
 11%|█         | 5376/50176 [00:38<05:04, 147.14it/s]
 11%|█         | 5632/50176 [00:39<04:59, 148.77it/s]
 12%|█▏        | 5888/50176 [00:41<04:58, 148.26it/s]
 12%|█▏        | 6144/50176 [00:43<05:22, 136.40it/s]
 13%|█▎        | 6400/50176 [00:45<05:11, 140.33it/s]
 13%|█▎        | 6656/50176 [00:47<05:05, 142.63it/s]
 14%|█▍        | 6912/50176 [00:49<05:02, 142.90it/s]
 14%|█▍        | 7168/50176 [00:50<04:57, 144.69it/s]
 15%|█▍        | 7424/50176 [00:52<04:50, 147.03it/s]
 15%|█▌        | 7680/50176 [00:54<04:49, 146.95it/s]
 16%|█▌        | 7936/50176 [00:55<04:39, 150.94it/s]
 16%|█▋        | 8192/50176 [00:57<04:38, 151.02it/s]
 17%|█▋        | 8448/50176 [00:59<04:35, 151.68it/s]
 17%|█▋        | 8704/50176 [01:00<04:31, 152.75it/s]
 18%|█▊        | 8960/50176 [01:02<04:40, 146.90it/s]
 18%|█▊        | 9216/50176 [01:04<04:32, 150.17it/s]
 19%|█▉        | 9472/50176 [01:06<04:31, 149.84it/s]
 19%|█▉        | 9728/50176 [01:07<04:36, 146.13it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.29it/s]
 20%|██        | 10240/50176 [01:11<04:28, 148.77it/s]
 21%|██        | 10496/50176 [01:12<04:23, 150.53it/s]
 21%|██▏       | 10752/50176 [01:14<04:32, 144.64it/s]
 22%|██▏       | 11008/50176 [01:16<04:26, 146.79it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.29it/s]
 23%|██▎       | 11520/50176 [01:19<04:16, 150.46it/s]
 23%|██▎       | 11776/50176 [01:21<04:15, 150.33it/s]
 24%|██▍       | 12032/50176 [01:23<04:12, 150.99it/s]
 24%|██▍       | 12288/50176 [01:24<04:08, 152.66it/s]
 25%|██▌       | 12544/50176 [01:26<04:04, 153.84it/s]
 26%|██▌       | 12800/50176 [01:28<04:02, 154.15it/s]
 26%|██▌       | 13056/50176 [01:29<03:59, 154.94it/s]
 27%|██▋       | 13312/50176 [01:31<04:02, 152.05it/s]
 27%|██▋       | 13568/50176 [01:33<04:10, 146.12it/s]
 28%|██▊       | 13824/50176 [01:35<04:14, 142.84it/s]
 28%|██▊       | 14080/50176 [01:37<04:43, 127.32it/s]
 29%|██▊       | 14336/50176 [01:39<04:37, 129.31it/s]
 29%|██▉       | 14592/50176 [01:41<04:19, 137.02it/s]
 30%|██▉       | 14848/50176 [01:43<04:28, 131.75it/s]
 30%|███       | 15104/50176 [01:45<04:25, 132.27it/s]
 31%|███       | 15360/50176 [01:47<04:15, 136.25it/s]
 31%|███       | 15616/50176 [01:48<04:08, 139.21it/s]
 32%|███▏      | 15872/50176 [01:50<04:11, 136.55it/s]
 32%|███▏      | 16128/50176 [01:52<04:04, 139.25it/s]
 33%|███▎      | 16384/50176 [01:54<04:02, 139.54it/s]
 33%|███▎      | 16640/50176 [01:56<04:08, 135.22it/s]
 34%|███▎      | 16896/50176 [01:58<03:56, 140.73it/s]
 34%|███▍      | 17152/50176 [01:59<03:49, 144.15it/s]
 35%|███▍      | 17408/50176 [02:01<03:44, 145.81it/s]
 35%|███▌      | 17664/50176 [02:03<03:40, 147.71it/s]
 36%|███▌      | 17920/50176 [02:04<03:35, 149.50it/s]
 36%|███▌      | 18176/50176 [02:06<03:35, 148.45it/s]
 37%|███▋      | 18432/50176 [02:08<03:32, 149.33it/s]
 37%|███▋      | 18688/50176 [02:10<03:31, 148.67it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.35it/s]
 38%|███▊      | 19200/50176 [02:14<03:49, 134.91it/s]
 39%|███▉      | 19456/50176 [02:16<03:51, 132.55it/s]
 39%|███▉      | 19712/50176 [02:18<03:49, 132.67it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.50it/s]
 40%|████      | 20224/50176 [02:21<03:31, 141.43it/s]
 41%|████      | 20480/50176 [02:23<03:25, 144.76it/s]
 41%|████▏     | 20736/50176 [02:24<03:23, 144.48it/s]
 42%|████▏     | 20992/50176 [02:26<03:17, 147.64it/s]
 42%|████▏     | 21248/50176 [02:28<03:22, 142.76it/s]
 43%|████▎     | 21504/50176 [02:30<03:13, 147.82it/s]
 43%|████▎     | 21760/50176 [02:31<03:11, 148.50it/s]
 44%|████▍     | 22016/50176 [02:33<03:06, 151.17it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.68it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.70it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.64it/s]
 46%|████▌     | 23040/50176 [02:40<03:07, 144.51it/s]
 46%|████▋     | 23296/50176 [02:42<03:10, 141.21it/s]
 47%|████▋     | 23552/50176 [02:44<03:13, 137.94it/s]
 47%|████▋     | 23808/50176 [02:46<03:05, 142.13it/s]
 48%|████▊     | 24064/50176 [02:47<03:02, 143.21it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.62it/s]
 49%|████▉     | 24576/50176 [02:51<02:55, 145.91it/s]
 49%|████▉     | 24832/50176 [02:52<02:49, 149.14it/s]
 50%|█████     | 25088/50176 [02:54<02:47, 149.67it/s]
 51%|█████     | 25344/50176 [02:56<02:45, 149.91it/s]
 51%|█████     | 25600/50176 [02:58<02:46, 147.47it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.24it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.53it/s]
 53%|█████▎    | 26368/50176 [03:03<02:44, 144.71it/s]
 53%|█████▎    | 26624/50176 [03:05<02:40, 147.00it/s]
 54%|█████▎    | 26880/50176 [03:06<02:38, 146.92it/s]
 54%|█████▍    | 27136/50176 [03:08<02:35, 147.76it/s]
 55%|█████▍    | 27392/50176 [03:10<02:33, 148.18it/s]
 55%|█████▌    | 27648/50176 [03:11<02:25, 155.15it/s]
 56%|█████▌    | 27904/50176 [03:13<02:34, 143.76it/s]
 56%|█████▌    | 28160/50176 [03:15<02:30, 145.87it/s]
 57%|█████▋    | 28416/50176 [03:17<02:29, 145.51it/s]
 57%|█████▋    | 28672/50176 [03:19<02:31, 141.61it/s]
 58%|█████▊    | 28928/50176 [03:20<02:27, 143.97it/s]
 58%|█████▊    | 29184/50176 [03:22<02:23, 146.35it/s]
 59%|█████▊    | 29440/50176 [03:24<02:30, 138.07it/s]
 59%|█████▉    | 29696/50176 [03:26<02:30, 135.72it/s]
 60%|█████▉    | 29952/50176 [03:28<02:28, 135.95it/s]
 60%|██████    | 30208/50176 [03:30<02:24, 138.15it/s]
 61%|██████    | 30464/50176 [03:31<02:16, 144.55it/s]
 61%|██████    | 30720/50176 [03:33<02:15, 143.59it/s]
 62%|██████▏   | 30976/50176 [03:35<02:11, 145.79it/s]
 62%|██████▏   | 31232/50176 [03:37<02:08, 147.97it/s]
 63%|██████▎   | 31488/50176 [03:38<02:04, 150.17it/s]
 63%|██████▎   | 31744/50176 [03:40<02:02, 150.49it/s]
 64%|██████▍   | 32000/50176 [03:42<02:02, 148.65it/s]
 64%|██████▍   | 32256/50176 [03:43<01:59, 150.00it/s]
 65%|██████▍   | 32512/50176 [03:45<02:01, 144.89it/s]
 65%|██████▌   | 32768/50176 [03:47<01:57, 148.63it/s]
 66%|██████▌   | 33024/50176 [03:49<01:56, 147.06it/s]
 66%|██████▋   | 33280/50176 [03:50<01:53, 148.41it/s]
 67%|██████▋   | 33536/50176 [03:52<01:54, 145.07it/s]
 67%|██████▋   | 33792/50176 [03:54<01:50, 147.94it/s]
 68%|██████▊   | 34048/50176 [03:55<01:45, 152.52it/s]
 68%|██████▊   | 34304/50176 [03:57<01:42, 154.98it/s]
 69%|██████▉   | 34560/50176 [03:59<01:45, 147.82it/s]
 69%|██████▉   | 34816/50176 [04:01<01:44, 146.80it/s]
 70%|██████▉   | 35072/50176 [04:02<01:43, 145.60it/s]
 70%|███████   | 35328/50176 [04:04<01:40, 148.26it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 147.12it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 141.94it/s]
 72%|███████▏  | 36096/50176 [04:09<01:36, 145.50it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 142.51it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 145.24it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 142.88it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.58it/s]
 74%|███████▍  | 37376/50176 [04:18<01:25, 149.58it/s]
 75%|███████▌  | 37632/50176 [04:20<01:22, 151.35it/s]
 76%|███████▌  | 37888/50176 [04:22<01:22, 148.67it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.95it/s]
 77%|███████▋  | 38400/50176 [04:25<01:19, 148.96it/s]
 77%|███████▋  | 38656/50176 [04:27<01:18, 147.25it/s]
 78%|███████▊  | 38912/50176 [04:29<01:17, 144.62it/s]
 78%|███████▊  | 39168/50176 [04:31<01:17, 141.47it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.55it/s]
 79%|███████▉  | 39680/50176 [04:34<01:12, 145.19it/s]
 80%|███████▉  | 39936/50176 [04:36<01:09, 146.99it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.67it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.60it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 138.73it/s]
 82%|████████▏ | 40960/50176 [04:43<01:04, 143.36it/s]
 82%|████████▏ | 41216/50176 [04:45<01:00, 146.94it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.76it/s]
 83%|████████▎ | 41728/50176 [04:48<00:55, 152.17it/s]
 84%|████████▎ | 41984/50176 [04:50<00:53, 152.52it/s]
 84%|████████▍ | 42240/50176 [04:51<00:53, 148.94it/s]
 85%|████████▍ | 42496/50176 [04:53<00:51, 150.07it/s]
 85%|████████▌ | 42752/50176 [04:55<00:49, 149.20it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 151.27it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.29it/s]
 87%|████████▋ | 43520/50176 [05:00<00:44, 149.26it/s]
 87%|████████▋ | 43776/50176 [05:02<00:43, 146.34it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 148.19it/s]
 88%|████████▊ | 44288/50176 [05:05<00:39, 150.35it/s]
 89%|████████▉ | 44544/50176 [05:07<00:38, 147.31it/s]
 89%|████████▉ | 44800/50176 [05:09<00:36, 145.61it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 147.46it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 140.64it/s]
 91%|█████████ | 45568/50176 [05:14<00:31, 145.46it/s]
 91%|█████████▏| 45824/50176 [05:16<00:29, 146.78it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.18it/s]
 92%|█████████▏| 46336/50176 [05:19<00:25, 149.45it/s]
 93%|█████████▎| 46592/50176 [05:21<00:23, 149.68it/s]
 93%|█████████▎| 46848/50176 [05:23<00:22, 148.37it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.93it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.41it/s]
 95%|█████████▍| 47616/50176 [05:28<00:17, 142.98it/s]
 95%|█████████▌| 47872/50176 [05:30<00:15, 145.03it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 146.96it/s]
 96%|█████████▋| 48384/50176 [05:33<00:12, 148.23it/s]
 97%|█████████▋| 48640/50176 [05:35<00:10, 149.93it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 150.16it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 146.40it/s]
 98%|█████████▊| 49408/50176 [05:40<00:05, 144.69it/s]
 99%|█████████▉| 49664/50176 [05:42<00:03, 145.43it/s]
 99%|█████████▉| 49920/50176 [05:44<00:01, 137.97it/s]
100%|██████████| 50176/50176 [05:44<00:00, 180.93it/s]
100%|██████████| 50176/50176 [05:44<00:00, 145.50it/s]
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5733	imagenet-zeroshot-val-top5: 0.8413
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt
new file mode 100644
index 0000000000000000000000000000000000000000..613741f88b42f87154c9e7bc3aae29712e50af23
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7ada228f719770c24962fa833e0c1e2fc52914f3b3d395b4562f97bb89aa738
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..abd05dc1046e61a661a5183c01aeaf8a6d05ed54
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt).
+2022-12-21,10:07:25 | INFO | Model:
+2022-12-21,10:07:25 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:25 | INFO | Params:
+2022-12-21,10:07:25 | INFO |   batch_size: 256
+2022-12-21,10:07:25 | INFO |   beta1: 0.9
+2022-12-21,10:07:25 | INFO |   beta2: 0.98
+2022-12-21,10:07:25 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/checkpoints
+2022-12-21,10:07:25 | INFO |   copy_codebase: False
+2022-12-21,10:07:25 | INFO |   csv_caption_key: title
+2022-12-21,10:07:25 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:25 | INFO |   csv_separator: 	
+2022-12-21,10:07:25 | INFO |   dataset_resampled: False
+2022-12-21,10:07:25 | INFO |   dataset_type: auto
+2022-12-21,10:07:25 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:25 | INFO |   debug: False
+2022-12-21,10:07:25 | INFO |   device: cuda:0
+2022-12-21,10:07:25 | INFO |   dist_backend: nccl
+2022-12-21,10:07:25 | INFO |   dist_url: env://
+2022-12-21,10:07:25 | INFO |   distributed: False
+2022-12-21,10:07:25 | INFO |   epochs: 32
+2022-12-21,10:07:25 | INFO |   eps: 1e-06
+2022-12-21,10:07:25 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:25 | INFO |   gather_with_grad: False
+2022-12-21,10:07:25 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:25 | INFO |   horovod: False
+2022-12-21,10:07:25 | INFO |   image_mean: None
+2022-12-21,10:07:25 | INFO |   image_std: None
+2022-12-21,10:07:25 | INFO |   imagenet_v2: None
+2022-12-21,10:07:25 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:25 | INFO |   local_loss: False
+2022-12-21,10:07:25 | INFO |   local_rank: 0
+2022-12-21,10:07:25 | INFO |   lock_image: False
+2022-12-21,10:07:25 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:25 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:25 | INFO |   log_level: 20
+2022-12-21,10:07:25 | INFO |   log_local: False
+2022-12-21,10:07:25 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/out.log
+2022-12-21,10:07:25 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:25 | INFO |   lr: 0.0005
+2022-12-21,10:07:25 | INFO |   model: ViT-B-16
+2022-12-21,10:07:25 | INFO |   name: epoch_26.pt
+2022-12-21,10:07:25 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:25 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:25 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:25 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt
+2022-12-21,10:07:25 | INFO |   pretrained_image: False
+2022-12-21,10:07:25 | INFO |   rank: 0
+2022-12-21,10:07:25 | INFO |   report_to: 
+2022-12-21,10:07:25 | INFO |   resume: None
+2022-12-21,10:07:25 | INFO |   save_frequency: 1
+2022-12-21,10:07:25 | INFO |   save_most_recent: False
+2022-12-21,10:07:25 | INFO |   seed: 0
+2022-12-21,10:07:25 | INFO |   skip_scheduler: False
+2022-12-21,10:07:25 | INFO |   tensorboard: False
+2022-12-21,10:07:25 | INFO |   tensorboard_path: 
+2022-12-21,10:07:25 | INFO |   torchscript: False
+2022-12-21,10:07:25 | INFO |   trace: False
+2022-12-21,10:07:25 | INFO |   train_data: None
+2022-12-21,10:07:25 | INFO |   train_num_samples: None
+2022-12-21,10:07:25 | INFO |   use_bn_sync: False
+2022-12-21,10:07:25 | INFO |   val_data: None
+2022-12-21,10:07:25 | INFO |   val_frequency: 1
+2022-12-21,10:07:25 | INFO |   val_num_samples: None
+2022-12-21,10:07:25 | INFO |   wandb: False
+2022-12-21,10:07:25 | INFO |   wandb_notes: 
+2022-12-21,10:07:25 | INFO |   warmup: 10000
+2022-12-21,10:07:25 | INFO |   wd: 0.2
+2022-12-21,10:07:25 | INFO |   workers: 1
+2022-12-21,10:07:25 | INFO |   world_size: 1
+2022-12-21,10:07:25 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:25 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:38,  1.22it/s]
  1%|          | 6/1000 [00:00<01:57,  8.45it/s]
  1%|          | 12/1000 [00:01<00:58, 16.90it/s]
  2%|▏         | 18/1000 [00:01<00:39, 24.95it/s]
  2%|▏         | 24/1000 [00:01<00:30, 32.18it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.29it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.07it/s]
  4%|▍         | 42/1000 [00:01<00:20, 46.86it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.66it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.80it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.33it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.29it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.89it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.48it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.81it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.18it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.37it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.66it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.75it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.55it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.51it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.61it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.58it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.64it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.73it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.80it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.71it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.61it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.59it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.51it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.38it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.36it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.33it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.16it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.08it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.19it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.15it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.37it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.32it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.19it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.30it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.39it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.47it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.53it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.38it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.43it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.56it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.67it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.85it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.79it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.75it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.89it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.91it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.79it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.60it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.71it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.81it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.89it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.68it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.72it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.82it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.90it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.78it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.54it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.78it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.81it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.81it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.82it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.85it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.95it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.94it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.72it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.79it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.86it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.82it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.78it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.92it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.85it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.86it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.85it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.78it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.81it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.69it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.77it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.79it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.88it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.76it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.87it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.87it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.78it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.77it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.85it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.75it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.72it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.87it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.85it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.74it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.88it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.76it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.77it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.89it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.90it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.96it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.80it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.74it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.89it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.82it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.89it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.85it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.74it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.83it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.88it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.85it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.86it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.88it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.90it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.93it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.86it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.89it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.93it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.91it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.00it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.03it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.87it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.97it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.95it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.00it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.04it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.80it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.81it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.87it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.84it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.79it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.81it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.94it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.10it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.14it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.05it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.07it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.02it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.03it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.95it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 58.01it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.00it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.87it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.99it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.98it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.89it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.92it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.93it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.92it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.99it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 58.02it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.02it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.04it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.03it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.93it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.97it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.03it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.04it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.05it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.03it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.68it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.10it/s]
+2022-12-21,10:07:43 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:04, 137.14it/s]
  1%|          | 512/50176 [00:03<05:12, 159.02it/s]
  2%|▏         | 768/50176 [00:04<04:46, 172.60it/s]
  2%|▏         | 1024/50176 [00:06<05:01, 162.76it/s]
  3%|▎         | 1280/50176 [00:08<05:17, 153.97it/s]
  3%|▎         | 1536/50176 [00:10<05:41, 142.25it/s]
  4%|▎         | 1792/50176 [00:12<05:44, 140.27it/s]
  4%|▍         | 2048/50176 [00:13<05:39, 141.58it/s]
  5%|▍         | 2304/50176 [00:15<05:39, 141.05it/s]
  5%|▌         | 2560/50176 [00:17<05:34, 142.29it/s]
  6%|▌         | 2816/50176 [00:19<05:36, 140.83it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 143.00it/s]
  7%|▋         | 3328/50176 [00:22<05:25, 144.00it/s]
  7%|▋         | 3584/50176 [00:24<05:18, 146.46it/s]
  8%|▊         | 3840/50176 [00:26<05:29, 140.56it/s]
  8%|▊         | 4096/50176 [00:28<05:27, 140.52it/s]
  9%|▊         | 4352/50176 [00:30<05:21, 142.48it/s]
  9%|▉         | 4608/50176 [00:31<05:16, 144.18it/s]
 10%|▉         | 4864/50176 [00:33<05:09, 146.57it/s]
 10%|█         | 5120/50176 [00:35<05:05, 147.34it/s]
 11%|█         | 5376/50176 [00:36<05:03, 147.79it/s]
 11%|█         | 5632/50176 [00:38<04:59, 148.65it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.42it/s]
 12%|█▏        | 6144/50176 [00:42<05:22, 136.36it/s]
 13%|█▎        | 6400/50176 [00:44<05:12, 140.26it/s]
 13%|█▎        | 6656/50176 [00:45<05:06, 142.01it/s]
 14%|█▍        | 6912/50176 [00:47<05:02, 143.02it/s]
 14%|█▍        | 7168/50176 [00:49<04:57, 144.79it/s]
 15%|█▍        | 7424/50176 [00:51<04:50, 147.08it/s]
 15%|█▌        | 7680/50176 [00:52<04:50, 146.41it/s]
 16%|█▌        | 7936/50176 [00:54<04:39, 151.24it/s]
 16%|█▋        | 8192/50176 [00:56<04:37, 151.13it/s]
 17%|█▋        | 8448/50176 [00:57<04:34, 151.77it/s]
 17%|█▋        | 8704/50176 [00:59<04:31, 152.81it/s]
 18%|█▊        | 8960/50176 [01:01<04:40, 147.11it/s]
 18%|█▊        | 9216/50176 [01:03<04:32, 150.21it/s]
 19%|█▉        | 9472/50176 [01:04<04:31, 149.89it/s]
 19%|█▉        | 9728/50176 [01:06<04:36, 146.15it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 148.15it/s]
 20%|██        | 10240/50176 [01:09<04:27, 149.34it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.00it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.02it/s]
 22%|██▏       | 11008/50176 [01:15<04:26, 146.73it/s]
 22%|██▏       | 11264/50176 [01:16<04:22, 148.28it/s]
 23%|██▎       | 11520/50176 [01:18<04:17, 150.27it/s]
 23%|██▎       | 11776/50176 [01:20<04:15, 150.22it/s]
 24%|██▍       | 12032/50176 [01:21<04:12, 150.93it/s]
 24%|██▍       | 12288/50176 [01:23<04:08, 152.63it/s]
 25%|██▌       | 12544/50176 [01:25<04:04, 153.79it/s]
 26%|██▌       | 12800/50176 [01:26<04:02, 154.08it/s]
 26%|██▌       | 13056/50176 [01:28<03:59, 155.08it/s]
 27%|██▋       | 13312/50176 [01:30<04:02, 152.08it/s]
 27%|██▋       | 13568/50176 [01:32<04:11, 145.63it/s]
 28%|██▊       | 13824/50176 [01:34<04:14, 142.78it/s]
 28%|██▊       | 14080/50176 [01:36<04:43, 127.48it/s]
 29%|██▊       | 14336/50176 [01:38<04:37, 129.36it/s]
 29%|██▉       | 14592/50176 [01:40<04:19, 137.14it/s]
 30%|██▉       | 14848/50176 [01:42<04:28, 131.80it/s]
 30%|███       | 15104/50176 [01:44<04:25, 132.03it/s]
 31%|███       | 15360/50176 [01:45<04:15, 136.27it/s]
 31%|███       | 15616/50176 [01:47<04:08, 139.11it/s]
 32%|███▏      | 15872/50176 [01:49<04:11, 136.21it/s]
 32%|███▏      | 16128/50176 [01:51<04:04, 139.31it/s]
 33%|███▎      | 16384/50176 [01:53<04:01, 139.82it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.48it/s]
 34%|███▎      | 16896/50176 [01:56<03:56, 140.78it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.16it/s]
 35%|███▍      | 17408/50176 [02:00<03:44, 145.80it/s]
 35%|███▌      | 17664/50176 [02:01<03:40, 147.62it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.56it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.85it/s]
 37%|███▋      | 18432/50176 [02:06<03:32, 149.56it/s]
 37%|███▋      | 18688/50176 [02:08<03:31, 148.63it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.59it/s]
 38%|███▊      | 19200/50176 [02:12<03:49, 135.11it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.19it/s]
 39%|███▉      | 19712/50176 [02:16<03:49, 132.63it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.68it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.52it/s]
 41%|████      | 20480/50176 [02:21<03:24, 144.92it/s]
 41%|████▏     | 20736/50176 [02:23<03:23, 145.01it/s]
 42%|████▏     | 20992/50176 [02:25<03:17, 147.52it/s]
 42%|████▏     | 21248/50176 [02:27<03:22, 142.71it/s]
 43%|████▎     | 21504/50176 [02:28<03:14, 147.68it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.42it/s]
 44%|████▍     | 22016/50176 [02:32<03:06, 151.09it/s]
 44%|████▍     | 22272/50176 [02:33<03:05, 150.58it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.69it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.28it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 144.17it/s]
 46%|████▋     | 23296/50176 [02:41<03:10, 140.84it/s]
 47%|████▋     | 23552/50176 [02:43<03:13, 137.55it/s]
 47%|████▋     | 23808/50176 [02:44<03:05, 141.89it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 143.21it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.56it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.92it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 149.02it/s]
 50%|█████     | 25088/50176 [02:53<02:47, 149.70it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.20it/s]
 51%|█████     | 25600/50176 [02:56<02:46, 147.72it/s]
 52%|█████▏    | 25856/50176 [02:58<02:40, 151.32it/s]
 52%|█████▏    | 26112/50176 [03:00<02:41, 148.55it/s]
 53%|█████▎    | 26368/50176 [03:02<02:44, 144.87it/s]
 53%|█████▎    | 26624/50176 [03:03<02:40, 146.99it/s]
 54%|█████▎    | 26880/50176 [03:05<02:38, 146.85it/s]
 54%|█████▍    | 27136/50176 [03:07<02:36, 147.62it/s]
 55%|█████▍    | 27392/50176 [03:08<02:33, 148.10it/s]
 55%|█████▌    | 27648/50176 [03:10<02:25, 155.31it/s]
 56%|█████▌    | 27904/50176 [03:12<02:34, 143.83it/s]
 56%|█████▌    | 28160/50176 [03:14<02:31, 145.46it/s]
 57%|█████▋    | 28416/50176 [03:15<02:29, 145.75it/s]
 57%|█████▋    | 28672/50176 [03:17<02:31, 141.84it/s]
 58%|█████▊    | 28928/50176 [03:19<02:27, 144.01it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 146.45it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.21it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 135.81it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.70it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.68it/s]
 61%|██████    | 30464/50176 [03:30<02:16, 144.54it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 143.59it/s]
 62%|██████▏   | 30976/50176 [03:34<02:11, 145.77it/s]
 62%|██████▏   | 31232/50176 [03:35<02:08, 147.33it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.33it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.68it/s]
 64%|██████▍   | 32000/50176 [03:40<02:02, 148.73it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 150.04it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 144.89it/s]
 65%|██████▌   | 32768/50176 [03:46<01:57, 148.76it/s]
 66%|██████▌   | 33024/50176 [03:47<01:56, 146.90it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 148.33it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 144.96it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.86it/s]
 68%|██████▊   | 34048/50176 [03:54<01:46, 151.92it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 155.35it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.08it/s]
 69%|██████▉   | 34816/50176 [03:59<01:44, 147.16it/s]
 70%|██████▉   | 35072/50176 [04:01<01:43, 146.17it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 148.81it/s]
 71%|███████   | 35584/50176 [04:05<01:39, 147.19it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.27it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.49it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 142.80it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.35it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 142.62it/s]
 74%|███████▍  | 37120/50176 [04:15<01:30, 143.66it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 149.30it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.30it/s]
 76%|███████▌  | 37888/50176 [04:20<01:22, 148.62it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.88it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.26it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.42it/s]
 78%|███████▊  | 38912/50176 [04:27<01:17, 144.81it/s]
 78%|███████▊  | 39168/50176 [04:29<01:18, 140.93it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.78it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 144.99it/s]
 80%|███████▉  | 39936/50176 [04:34<01:09, 147.25it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.86it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.70it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.79it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.16it/s]
 82%|████████▏ | 41216/50176 [04:43<01:01, 146.58it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 149.01it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.90it/s]
 84%|████████▎ | 41984/50176 [04:48<00:53, 152.30it/s]
 84%|████████▍ | 42240/50176 [04:50<00:53, 148.84it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 150.11it/s]
 85%|████████▌ | 42752/50176 [04:53<00:49, 149.17it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 151.15it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.80it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.91it/s]
 87%|████████▋ | 43776/50176 [05:00<00:43, 146.33it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.10it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.32it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.24it/s]
 89%|████████▉ | 44800/50176 [05:07<00:37, 145.15it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.08it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 141.07it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.50it/s]
 91%|█████████▏| 45824/50176 [05:14<00:29, 147.56it/s]
 92%|█████████▏| 46080/50176 [05:16<00:27, 148.01it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.39it/s]
 93%|█████████▎| 46592/50176 [05:19<00:23, 149.63it/s]
 93%|█████████▎| 46848/50176 [05:21<00:22, 148.11it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 143.32it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 145.43it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.05it/s]
 95%|█████████▌| 47872/50176 [05:28<00:15, 145.06it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 146.91it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.13it/s]
 97%|█████████▋| 48640/50176 [05:33<00:10, 149.89it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 150.13it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 146.39it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 144.72it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.44it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.23it/s]
100%|██████████| 50176/50176 [05:43<00:00, 180.35it/s]
100%|██████████| 50176/50176 [05:43<00:00, 146.03it/s]
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5748	imagenet-zeroshot-val-top5: 0.8455
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ea0a3206710817f563cf7f074e79f21f49e956c3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e38037946c6a96355ccafdeb43d67ff31269a7435ed05d1a6b2f854ebb7dff9f
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5b49f5c01e68617afd3d5e95293c0f75cda6c67f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt).
+2022-12-21,10:07:25 | INFO | Model:
+2022-12-21,10:07:25 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:25 | INFO | Params:
+2022-12-21,10:07:25 | INFO |   batch_size: 256
+2022-12-21,10:07:25 | INFO |   beta1: 0.9
+2022-12-21,10:07:25 | INFO |   beta2: 0.98
+2022-12-21,10:07:25 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/checkpoints
+2022-12-21,10:07:25 | INFO |   copy_codebase: False
+2022-12-21,10:07:25 | INFO |   csv_caption_key: title
+2022-12-21,10:07:25 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:25 | INFO |   csv_separator: 	
+2022-12-21,10:07:25 | INFO |   dataset_resampled: False
+2022-12-21,10:07:25 | INFO |   dataset_type: auto
+2022-12-21,10:07:25 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:25 | INFO |   debug: False
+2022-12-21,10:07:25 | INFO |   device: cuda:0
+2022-12-21,10:07:25 | INFO |   dist_backend: nccl
+2022-12-21,10:07:25 | INFO |   dist_url: env://
+2022-12-21,10:07:25 | INFO |   distributed: False
+2022-12-21,10:07:25 | INFO |   epochs: 32
+2022-12-21,10:07:25 | INFO |   eps: 1e-06
+2022-12-21,10:07:25 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:25 | INFO |   gather_with_grad: False
+2022-12-21,10:07:25 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:25 | INFO |   horovod: False
+2022-12-21,10:07:25 | INFO |   image_mean: None
+2022-12-21,10:07:25 | INFO |   image_std: None
+2022-12-21,10:07:25 | INFO |   imagenet_v2: None
+2022-12-21,10:07:25 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:25 | INFO |   local_loss: False
+2022-12-21,10:07:25 | INFO |   local_rank: 0
+2022-12-21,10:07:25 | INFO |   lock_image: False
+2022-12-21,10:07:25 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:25 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:25 | INFO |   log_level: 20
+2022-12-21,10:07:25 | INFO |   log_local: False
+2022-12-21,10:07:25 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/out.log
+2022-12-21,10:07:25 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:25 | INFO |   lr: 0.0005
+2022-12-21,10:07:25 | INFO |   model: ViT-B-16
+2022-12-21,10:07:25 | INFO |   name: epoch_27.pt
+2022-12-21,10:07:25 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:25 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:25 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:25 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt
+2022-12-21,10:07:25 | INFO |   pretrained_image: False
+2022-12-21,10:07:25 | INFO |   rank: 0
+2022-12-21,10:07:25 | INFO |   report_to: 
+2022-12-21,10:07:25 | INFO |   resume: None
+2022-12-21,10:07:25 | INFO |   save_frequency: 1
+2022-12-21,10:07:25 | INFO |   save_most_recent: False
+2022-12-21,10:07:25 | INFO |   seed: 0
+2022-12-21,10:07:25 | INFO |   skip_scheduler: False
+2022-12-21,10:07:25 | INFO |   tensorboard: False
+2022-12-21,10:07:25 | INFO |   tensorboard_path: 
+2022-12-21,10:07:25 | INFO |   torchscript: False
+2022-12-21,10:07:25 | INFO |   trace: False
+2022-12-21,10:07:25 | INFO |   train_data: None
+2022-12-21,10:07:25 | INFO |   train_num_samples: None
+2022-12-21,10:07:25 | INFO |   use_bn_sync: False
+2022-12-21,10:07:25 | INFO |   val_data: None
+2022-12-21,10:07:25 | INFO |   val_frequency: 1
+2022-12-21,10:07:25 | INFO |   val_num_samples: None
+2022-12-21,10:07:25 | INFO |   wandb: False
+2022-12-21,10:07:25 | INFO |   wandb_notes: 
+2022-12-21,10:07:25 | INFO |   warmup: 10000
+2022-12-21,10:07:25 | INFO |   wd: 0.2
+2022-12-21,10:07:25 | INFO |   workers: 1
+2022-12-21,10:07:25 | INFO |   world_size: 1
+2022-12-21,10:07:25 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:25 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:04,  1.50it/s]
  1%|          | 6/1000 [00:00<01:38, 10.05it/s]
  1%|          | 12/1000 [00:00<00:51, 19.37it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.75it/s]
  2%|▏         | 24/1000 [00:01<00:27, 34.91it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.65it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.03it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.37it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.01it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.97it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.37it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.05it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.69it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.08it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.58it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.88it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.25it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.44it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.70it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.79it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.55it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.52it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.10it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.22it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.19it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.40it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.20it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.20it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.20it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.20it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.27it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 56.77it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 56.91it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 56.63it/s]
 20%|██        | 204/1000 [00:04<00:14, 56.62it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.43it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.71it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.49it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 56.83it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 56.96it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.55it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 56.77it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 54.99it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 55.80it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 56.32it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 56.51it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 56.80it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.17it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.41it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.71it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.75it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.76it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.95it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.95it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.86it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.67it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.75it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.83it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.94it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.73it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.75it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.82it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.91it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.80it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.48it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.56it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.84it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.93it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.92it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.93it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.96it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.08it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.07it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.85it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.91it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.93it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.92it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.88it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.03it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.86it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.86it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.86it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.78it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.84it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.75it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.82it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.87it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.96it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.89it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.00it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.01it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.91it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.92it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.97it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.86it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.80it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.94it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.92it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.84it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.95it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.85it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.87it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.00it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.99it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.07it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.87it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.81it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.96it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.91it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.01it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.05it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.99it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.05it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.05it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.07it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.96it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.06it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.06it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.05it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.02it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.04it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.02it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.09it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.00it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.10it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.05it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.08it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.10it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.89it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.86it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.92it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.89it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.87it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.90it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.01it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.15it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.19it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.10it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.14it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.09it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.10it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.09it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.08it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.96it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.06it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.06it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.03it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.03it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.00it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.10it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.14it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.12it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.87it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.81it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.63it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.65it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.68it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.70it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.64it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.62it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.24it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.50it/s]
+2022-12-21,10:07:43 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:01, 138.14it/s]
  1%|          | 512/50176 [00:03<05:13, 158.18it/s]
  2%|▏         | 768/50176 [00:04<04:59, 165.03it/s]
  2%|▏         | 1024/50176 [00:06<05:16, 155.36it/s]
  3%|▎         | 1280/50176 [00:08<05:28, 148.83it/s]
  3%|▎         | 1536/50176 [00:10<05:46, 140.24it/s]
  4%|▎         | 1792/50176 [00:12<05:47, 139.04it/s]
  4%|▍         | 2048/50176 [00:14<05:43, 140.19it/s]
  5%|▍         | 2304/50176 [00:15<05:41, 140.29it/s]
  5%|▌         | 2560/50176 [00:17<05:35, 141.90it/s]
  6%|▌         | 2816/50176 [00:19<05:36, 140.90it/s]
  6%|▌         | 3072/50176 [00:21<05:30, 142.53it/s]
  7%|▋         | 3328/50176 [00:23<05:27, 143.11it/s]
  7%|▋         | 3584/50176 [00:24<05:18, 146.28it/s]
  8%|▊         | 3840/50176 [00:26<05:29, 140.59it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.38it/s]
  9%|▊         | 4352/50176 [00:30<05:21, 142.37it/s]
  9%|▉         | 4608/50176 [00:32<05:16, 143.86it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.81it/s]
 10%|█         | 5120/50176 [00:35<05:04, 147.77it/s]
 11%|█         | 5376/50176 [00:37<05:02, 147.92it/s]
 11%|█         | 5632/50176 [00:38<04:59, 148.68it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.34it/s]
 12%|█▏        | 6144/50176 [00:42<05:23, 136.22it/s]
 13%|█▎        | 6400/50176 [00:44<05:12, 140.23it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.28it/s]
 14%|█▍        | 6912/50176 [00:47<05:02, 142.94it/s]
 14%|█▍        | 7168/50176 [00:49<04:57, 144.64it/s]
 15%|█▍        | 7424/50176 [00:51<04:50, 146.97it/s]
 15%|█▌        | 7680/50176 [00:53<04:49, 146.95it/s]
 16%|█▌        | 7936/50176 [00:54<04:39, 151.00it/s]
 16%|█▋        | 8192/50176 [00:56<04:38, 151.02it/s]
 17%|█▋        | 8448/50176 [00:58<04:35, 151.64it/s]
 17%|█▋        | 8704/50176 [00:59<04:31, 152.81it/s]
 18%|█▊        | 8960/50176 [01:01<04:40, 146.91it/s]
 18%|█▊        | 9216/50176 [01:03<04:32, 150.27it/s]
 19%|█▉        | 9472/50176 [01:04<04:32, 149.51it/s]
 19%|█▉        | 9728/50176 [01:06<04:36, 146.27it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 148.28it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.68it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.08it/s]
 21%|██▏       | 10752/50176 [01:13<04:32, 144.73it/s]
 22%|██▏       | 11008/50176 [01:15<04:26, 146.71it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.38it/s]
 23%|██▎       | 11520/50176 [01:18<04:17, 150.35it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.56it/s]
 24%|██▍       | 12032/50176 [01:22<04:12, 151.12it/s]
 24%|██▍       | 12288/50176 [01:23<04:08, 152.76it/s]
 25%|██▌       | 12544/50176 [01:25<04:04, 153.81it/s]
 26%|██▌       | 12800/50176 [01:27<04:02, 154.11it/s]
 26%|██▌       | 13056/50176 [01:28<03:59, 155.13it/s]
 27%|██▋       | 13312/50176 [01:30<04:02, 152.22it/s]
 27%|██▋       | 13568/50176 [01:32<04:12, 145.19it/s]
 28%|██▊       | 13824/50176 [01:34<04:14, 142.97it/s]
 28%|██▊       | 14080/50176 [01:36<04:43, 127.49it/s]
 29%|██▊       | 14336/50176 [01:38<04:36, 129.44it/s]
 29%|██▉       | 14592/50176 [01:40<04:19, 137.13it/s]
 30%|██▉       | 14848/50176 [01:42<04:28, 131.78it/s]
 30%|███       | 15104/50176 [01:44<04:25, 132.00it/s]
 31%|███       | 15360/50176 [01:46<04:15, 136.44it/s]
 31%|███       | 15616/50176 [01:47<04:08, 139.19it/s]
 32%|███▏      | 15872/50176 [01:49<04:11, 136.62it/s]
 32%|███▏      | 16128/50176 [01:51<04:04, 139.34it/s]
 33%|███▎      | 16384/50176 [01:53<04:02, 139.58it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.28it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.69it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.10it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.18it/s]
 35%|███▌      | 17664/50176 [02:02<03:39, 148.05it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.58it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.17it/s]
 37%|███▋      | 18432/50176 [02:07<03:32, 149.51it/s]
 37%|███▋      | 18688/50176 [02:08<03:31, 148.91it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.43it/s]
 38%|███▊      | 19200/50176 [02:13<03:49, 134.92it/s]
 39%|███▉      | 19456/50176 [02:15<03:51, 132.57it/s]
 39%|███▉      | 19712/50176 [02:17<03:49, 132.50it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.58it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.54it/s]
 41%|████      | 20480/50176 [02:22<03:25, 144.75it/s]
 41%|████▏     | 20736/50176 [02:23<03:22, 145.03it/s]
 42%|████▏     | 20992/50176 [02:25<03:17, 147.53it/s]
 42%|████▏     | 21248/50176 [02:27<03:22, 142.65it/s]
 43%|████▎     | 21504/50176 [02:29<03:14, 147.72it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.37it/s]
 44%|████▍     | 22016/50176 [02:32<03:06, 151.11it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.70it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.63it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.26it/s]
 46%|████▌     | 23040/50176 [02:39<03:07, 144.67it/s]
 46%|████▋     | 23296/50176 [02:41<03:09, 141.51it/s]
 47%|████▋     | 23552/50176 [02:43<03:12, 138.31it/s]
 47%|████▋     | 23808/50176 [02:45<03:05, 142.19it/s]
 48%|████▊     | 24064/50176 [02:46<03:01, 143.58it/s]
 48%|████▊     | 24320/50176 [02:48<02:59, 143.83it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 146.08it/s]
 49%|████▉     | 24832/50176 [02:51<02:49, 149.13it/s]
 50%|█████     | 25088/50176 [02:53<02:47, 149.57it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 150.01it/s]
 51%|█████     | 25600/50176 [02:57<02:46, 147.62it/s]
 52%|█████▏    | 25856/50176 [02:58<02:40, 151.27it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.38it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.26it/s]
 53%|█████▎    | 26624/50176 [03:03<02:40, 147.17it/s]
 54%|█████▎    | 26880/50176 [03:05<02:38, 146.99it/s]
 54%|█████▍    | 27136/50176 [03:07<02:35, 147.79it/s]
 55%|█████▍    | 27392/50176 [03:09<02:33, 148.10it/s]
 55%|█████▌    | 27648/50176 [03:10<02:25, 154.88it/s]
 56%|█████▌    | 27904/50176 [03:12<02:34, 144.14it/s]
 56%|█████▌    | 28160/50176 [03:14<02:30, 145.82it/s]
 57%|█████▋    | 28416/50176 [03:16<02:29, 145.71it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.20it/s]
 58%|█████▊    | 28928/50176 [03:19<02:27, 144.19it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 145.91it/s]
 59%|█████▊    | 29440/50176 [03:23<02:29, 138.35it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.04it/s]
 60%|█████▉    | 29952/50176 [03:27<02:28, 135.86it/s]
 60%|██████    | 30208/50176 [03:29<02:24, 138.17it/s]
 61%|██████    | 30464/50176 [03:30<02:16, 144.49it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 143.55it/s]
 62%|██████▏   | 30976/50176 [03:34<02:11, 145.75it/s]
 62%|██████▏   | 31232/50176 [03:35<02:08, 147.93it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.14it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.53it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.62it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 149.90it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 144.96it/s]
 65%|██████▌   | 32768/50176 [03:46<01:57, 148.40it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 146.97it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 148.35it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 144.99it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.88it/s]
 68%|██████▊   | 34048/50176 [03:54<01:45, 152.48it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 155.15it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.16it/s]
 69%|██████▉   | 34816/50176 [04:00<01:44, 146.60it/s]
 70%|██████▉   | 35072/50176 [04:01<01:43, 146.28it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 148.82it/s]
 71%|███████   | 35584/50176 [04:05<01:39, 147.21it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.31it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.49it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 142.77it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.35it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 142.60it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 143.67it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 149.10it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.37it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 149.10it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.78it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.16it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.39it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 144.78it/s]
 78%|███████▊  | 39168/50176 [04:30<01:18, 140.96it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.75it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 144.96it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.22it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.84it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.73it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.75it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.16it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.58it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.99it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.94it/s]
 84%|████████▎ | 41984/50176 [04:48<00:53, 152.29it/s]
 84%|████████▍ | 42240/50176 [04:50<00:53, 148.86it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 150.11it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.17it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 150.45it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 151.01it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.04it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.46it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.14it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.38it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.38it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 145.62it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.24it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 140.97it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.55it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 147.38it/s]
 92%|█████████▏| 46080/50176 [05:16<00:27, 147.89it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.29it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.54it/s]
 93%|█████████▎| 46848/50176 [05:21<00:22, 148.09it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 143.39it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 145.29it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.02it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.09it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 146.90it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.11it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 149.19it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 150.32it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 146.52it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 144.81it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.51it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.21it/s]
100%|██████████| 50176/50176 [05:43<00:00, 180.31it/s]
100%|██████████| 50176/50176 [05:43<00:00, 145.94it/s]
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5813	imagenet-zeroshot-val-top5: 0.8476
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5173b1f903a73635c7f4c2b0ce8dd75525097234
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eab48e604643e294001d8f7fbbaadc8cc2b4c804eee64c745470bc761b920a7d
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..bb820acc3570cfb88efffa4dadd38818d87f22e0
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:13:33 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:33 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:34 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt).
+2022-12-21,10:13:41 | INFO | Model:
+2022-12-21,10:13:41 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:42 | INFO | Params:
+2022-12-21,10:13:42 | INFO |   batch_size: 256
+2022-12-21,10:13:42 | INFO |   beta1: 0.9
+2022-12-21,10:13:42 | INFO |   beta2: 0.98
+2022-12-21,10:13:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/checkpoints
+2022-12-21,10:13:42 | INFO |   copy_codebase: False
+2022-12-21,10:13:42 | INFO |   csv_caption_key: title
+2022-12-21,10:13:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:42 | INFO |   csv_separator: 	
+2022-12-21,10:13:42 | INFO |   dataset_resampled: False
+2022-12-21,10:13:42 | INFO |   dataset_type: auto
+2022-12-21,10:13:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:42 | INFO |   debug: False
+2022-12-21,10:13:42 | INFO |   device: cuda:0
+2022-12-21,10:13:42 | INFO |   dist_backend: nccl
+2022-12-21,10:13:42 | INFO |   dist_url: env://
+2022-12-21,10:13:42 | INFO |   distributed: False
+2022-12-21,10:13:42 | INFO |   epochs: 32
+2022-12-21,10:13:42 | INFO |   eps: 1e-06
+2022-12-21,10:13:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:42 | INFO |   gather_with_grad: False
+2022-12-21,10:13:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:42 | INFO |   horovod: False
+2022-12-21,10:13:42 | INFO |   image_mean: None
+2022-12-21,10:13:42 | INFO |   image_std: None
+2022-12-21,10:13:42 | INFO |   imagenet_v2: None
+2022-12-21,10:13:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:42 | INFO |   local_loss: False
+2022-12-21,10:13:42 | INFO |   local_rank: 0
+2022-12-21,10:13:42 | INFO |   lock_image: False
+2022-12-21,10:13:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:42 | INFO |   log_level: 20
+2022-12-21,10:13:42 | INFO |   log_local: False
+2022-12-21,10:13:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/out.log
+2022-12-21,10:13:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:42 | INFO |   lr: 0.0005
+2022-12-21,10:13:42 | INFO |   model: ViT-B-16
+2022-12-21,10:13:42 | INFO |   name: epoch_28.pt
+2022-12-21,10:13:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt
+2022-12-21,10:13:42 | INFO |   pretrained_image: False
+2022-12-21,10:13:42 | INFO |   rank: 0
+2022-12-21,10:13:42 | INFO |   report_to: 
+2022-12-21,10:13:42 | INFO |   resume: None
+2022-12-21,10:13:42 | INFO |   save_frequency: 1
+2022-12-21,10:13:42 | INFO |   save_most_recent: False
+2022-12-21,10:13:42 | INFO |   seed: 0
+2022-12-21,10:13:42 | INFO |   skip_scheduler: False
+2022-12-21,10:13:42 | INFO |   tensorboard: False
+2022-12-21,10:13:42 | INFO |   tensorboard_path: 
+2022-12-21,10:13:42 | INFO |   torchscript: False
+2022-12-21,10:13:42 | INFO |   trace: False
+2022-12-21,10:13:42 | INFO |   train_data: None
+2022-12-21,10:13:42 | INFO |   train_num_samples: None
+2022-12-21,10:13:42 | INFO |   use_bn_sync: False
+2022-12-21,10:13:42 | INFO |   val_data: None
+2022-12-21,10:13:42 | INFO |   val_frequency: 1
+2022-12-21,10:13:42 | INFO |   val_num_samples: None
+2022-12-21,10:13:42 | INFO |   wandb: False
+2022-12-21,10:13:42 | INFO |   wandb_notes: 
+2022-12-21,10:13:42 | INFO |   warmup: 10000
+2022-12-21,10:13:42 | INFO |   wd: 0.2
+2022-12-21,10:13:42 | INFO |   workers: 1
+2022-12-21,10:13:42 | INFO |   world_size: 1
+2022-12-21,10:13:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:42 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:42 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:53,  1.40it/s]
  1%|          | 6/1000 [00:00<01:44,  9.49it/s]
  1%|          | 12/1000 [00:00<00:53, 18.53it/s]
  2%|▏         | 18/1000 [00:01<00:36, 26.86it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.07it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.99it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.44it/s]
  4%|▍         | 42/1000 [00:01<00:19, 47.95it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.49it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.42it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.79it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.82it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.54it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.04it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.56it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.84it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.21it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.40it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.63it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.75it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.52it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.52it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.59it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.57it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.52it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.64it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.69it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.59it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.46it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.48it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.45it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.38it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.34it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.29it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.11it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.00it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.11it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.07it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.30it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.29it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.20it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.28it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.39it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.49it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.53it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.35it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.39it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.54it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.78it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.79it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.79it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.93it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.92it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.82it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.63it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.72it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.83it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.92it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.69it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.67it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.73it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.81it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.69it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.36it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.43it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.55it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.68it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.76it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.76it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.77it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.80it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.93it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.92it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.70it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.81it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.78it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.73it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.87it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.77it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.75it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.72it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.64it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.70it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.63it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.71it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.74it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.83it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.77it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.88it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.88it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.76it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.75it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.82it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.70it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.64it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.78it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.77it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.70it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.80it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.71it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.71it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.84it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.84it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.93it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.75it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.70it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.84it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.77it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.86it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.88it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.80it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.84it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.86it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.89it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.77it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.85it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.84it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.83it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.85it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.81it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.82it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.83it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.63it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.74it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.88it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.74it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.87it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.84it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.90it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.96it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.75it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.73it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.79it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.77it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.74it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.76it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.86it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.99it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.03it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.94it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.98it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.92it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.92it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.86it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.91it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.88it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.76it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.86it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.86it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.82it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.85it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.84it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.59it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.67it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.79it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.84it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.91it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.96it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.92it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.00it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.08it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.11it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.83it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.52it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.20it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.35it/s]
+2022-12-21,10:14:00 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:55, 140.40it/s]
  1%|          | 512/50176 [00:03<05:13, 158.57it/s]
  2%|▏         | 768/50176 [00:04<04:46, 172.32it/s]
  2%|▏         | 1024/50176 [00:06<04:51, 168.41it/s]
  3%|▎         | 1280/50176 [00:07<05:08, 158.44it/s]
  3%|▎         | 1536/50176 [00:10<05:34, 145.37it/s]
  4%|▎         | 1792/50176 [00:11<05:38, 143.05it/s]
  4%|▍         | 2048/50176 [00:13<05:34, 143.71it/s]
  5%|▍         | 2304/50176 [00:15<05:36, 142.32it/s]
  5%|▌         | 2560/50176 [00:17<05:33, 142.60it/s]
  6%|▌         | 2816/50176 [00:19<05:38, 139.83it/s]
  6%|▌         | 3072/50176 [00:20<05:30, 142.52it/s]
  7%|▋         | 3328/50176 [00:22<05:28, 142.52it/s]
  7%|▋         | 3584/50176 [00:24<05:21, 144.87it/s]
  8%|▊         | 3840/50176 [00:26<05:31, 139.70it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.07it/s]
  9%|▊         | 4352/50176 [00:29<05:22, 142.01it/s]
  9%|▉         | 4608/50176 [00:31<05:17, 143.43it/s]
 10%|▉         | 4864/50176 [00:33<05:11, 145.62it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.41it/s]
 11%|█         | 5376/50176 [00:36<05:03, 147.39it/s]
 11%|█         | 5632/50176 [00:38<05:00, 148.21it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.14it/s]
 12%|█▏        | 6144/50176 [00:42<05:24, 135.86it/s]
 13%|█▎        | 6400/50176 [00:44<05:14, 139.23it/s]
 13%|█▎        | 6656/50176 [00:45<05:08, 141.24it/s]
 14%|█▍        | 6912/50176 [00:47<05:04, 141.90it/s]
 14%|█▍        | 7168/50176 [00:49<05:01, 142.87it/s]
 15%|█▍        | 7424/50176 [00:51<04:53, 145.72it/s]
 15%|█▌        | 7680/50176 [00:53<04:54, 144.42it/s]
 16%|█▌        | 7936/50176 [00:54<04:45, 148.10it/s]
 16%|█▋        | 8192/50176 [00:56<04:45, 147.21it/s]
 17%|█▋        | 8448/50176 [00:58<04:41, 148.43it/s]
 17%|█▋        | 8704/50176 [00:59<04:38, 149.06it/s]
 18%|█▊        | 8960/50176 [01:01<04:47, 143.57it/s]
 18%|█▊        | 9216/50176 [01:03<04:37, 147.65it/s]
 19%|█▉        | 9472/50176 [01:05<04:36, 147.33it/s]
 19%|█▉        | 9728/50176 [01:06<04:39, 144.65it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 147.90it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.71it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.24it/s]
 21%|██▏       | 10752/50176 [01:13<04:30, 145.86it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 147.82it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.40it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.50it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.81it/s]
 24%|██▍       | 12032/50176 [01:22<04:14, 150.07it/s]
 24%|██▍       | 12288/50176 [01:23<04:11, 150.58it/s]
 25%|██▌       | 12544/50176 [01:25<04:08, 151.45it/s]
 26%|██▌       | 12800/50176 [01:27<04:06, 151.42it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.78it/s]
 27%|██▋       | 13312/50176 [01:30<04:05, 150.05it/s]
 27%|██▋       | 13568/50176 [01:32<04:12, 144.87it/s]
 28%|██▊       | 13824/50176 [01:34<04:16, 141.74it/s]
 28%|██▊       | 14080/50176 [01:36<04:13, 142.21it/s]
 29%|██▊       | 14336/50176 [01:38<04:18, 138.87it/s]
 29%|██▉       | 14592/50176 [01:39<04:05, 144.73it/s]
 30%|██▉       | 14848/50176 [01:41<04:18, 136.63it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.37it/s]
 31%|███       | 15360/50176 [01:45<04:10, 138.86it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.40it/s]
 32%|███▏      | 15872/50176 [01:49<04:10, 137.01it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.65it/s]
 33%|███▎      | 16384/50176 [01:52<04:01, 139.70it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 134.93it/s]
 34%|███▎      | 16896/50176 [01:56<03:57, 139.96it/s]
 34%|███▍      | 17152/50176 [01:58<03:50, 143.10it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.40it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.89it/s]
 36%|███▌      | 17920/50176 [02:03<03:36, 149.18it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.91it/s]
 37%|███▋      | 18432/50176 [02:06<03:34, 147.83it/s]
 37%|███▋      | 18688/50176 [02:08<03:33, 147.44it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.05it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.36it/s]
 39%|███▉      | 19456/50176 [02:14<03:53, 131.46it/s]
 39%|███▉      | 19712/50176 [02:16<03:51, 131.67it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.66it/s]
 40%|████      | 20224/50176 [02:20<03:32, 141.25it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.77it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.89it/s]
 42%|████▏     | 20992/50176 [02:25<03:18, 147.22it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 142.20it/s]
 43%|████▎     | 21504/50176 [02:28<03:15, 146.89it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.49it/s]
 44%|████▍     | 22016/50176 [02:32<03:05, 151.58it/s]
 44%|████▍     | 22272/50176 [02:33<03:04, 150.94it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.51it/s]
 45%|████▌     | 22784/50176 [02:37<03:07, 145.91it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 143.95it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.64it/s]
 47%|████▋     | 23552/50176 [02:43<03:14, 136.90it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.02it/s]
 48%|████▊     | 24064/50176 [02:46<03:04, 141.87it/s]
 48%|████▊     | 24320/50176 [02:48<03:01, 142.61it/s]
 49%|████▉     | 24576/50176 [02:50<02:56, 145.13it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.66it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.70it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.74it/s]
 51%|█████     | 25600/50176 [02:56<02:48, 146.16it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.59it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.25it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.14it/s]
 53%|█████▎    | 26624/50176 [03:03<02:42, 145.31it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.29it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 146.20it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.69it/s]
 55%|█████▌    | 27648/50176 [03:10<02:27, 153.24it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 142.52it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.78it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.50it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.90it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 143.13it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 146.11it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.00it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.30it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.32it/s]
 60%|██████    | 30208/50176 [03:29<02:23, 138.90it/s]
 61%|██████    | 30464/50176 [03:30<02:15, 145.36it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 144.08it/s]
 62%|██████▏   | 30976/50176 [03:34<02:10, 146.95it/s]
 62%|██████▏   | 31232/50176 [03:35<02:07, 149.08it/s]
 63%|██████▎   | 31488/50176 [03:37<02:03, 150.96it/s]
 63%|██████▎   | 31744/50176 [03:39<02:01, 151.15it/s]
 64%|██████▍   | 32000/50176 [03:41<02:01, 149.96it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 150.43it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 145.75it/s]
 65%|██████▌   | 32768/50176 [03:46<01:56, 149.39it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 147.58it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 148.79it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 145.21it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.97it/s]
 68%|██████▊   | 34048/50176 [03:54<01:45, 152.19it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 154.37it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.61it/s]
 69%|██████▉   | 34816/50176 [03:59<01:43, 148.57it/s]
 70%|██████▉   | 35072/50176 [04:01<01:42, 147.34it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 149.83it/s]
 71%|███████   | 35584/50176 [04:05<01:38, 147.82it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.35it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.72it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 143.38it/s]
 73%|███████▎  | 36608/50176 [04:12<01:32, 146.06it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.74it/s]
 74%|███████▍  | 37120/50176 [04:15<01:29, 146.21it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 150.11it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.58it/s]
 76%|███████▌  | 37888/50176 [04:20<01:22, 149.01it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 150.13it/s]
 77%|███████▋  | 38400/50176 [04:24<01:19, 148.91it/s]
 77%|███████▋  | 38656/50176 [04:26<01:17, 147.90it/s]
 78%|███████▊  | 38912/50176 [04:27<01:17, 145.72it/s]
 78%|███████▊  | 39168/50176 [04:29<01:17, 142.01it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.70it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.37it/s]
 80%|███████▉  | 39936/50176 [04:34<01:09, 147.11it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.53it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.79it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.77it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.25it/s]
 82%|████████▏ | 41216/50176 [04:43<01:01, 146.40it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 147.97it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.04it/s]
 84%|████████▎ | 41984/50176 [04:48<00:54, 150.71it/s]
 84%|████████▍ | 42240/50176 [04:50<00:53, 147.99it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.75it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.00it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 150.33it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.59it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.74it/s]
 87%|████████▋ | 43776/50176 [05:00<00:43, 146.74it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.18it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.11it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.37it/s]
 89%|████████▉ | 44800/50176 [05:07<00:36, 145.85it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.44it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 141.68it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.77it/s]
 91%|█████████▏| 45824/50176 [05:15<00:30, 144.89it/s]
 92%|█████████▏| 46080/50176 [05:16<00:28, 144.77it/s]
 92%|█████████▏| 46336/50176 [05:18<00:26, 145.65it/s]
 93%|█████████▎| 46592/50176 [05:20<00:24, 147.01it/s]
 93%|█████████▎| 46848/50176 [05:21<00:22, 146.05it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 142.00it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 144.19it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.44it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.16it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 147.35it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.41it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.28it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 149.56it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 147.50it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.63it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.23it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 137.69it/s]
100%|██████████| 50176/50176 [05:43<00:00, 180.10it/s]
100%|██████████| 50176/50176 [05:43<00:00, 145.94it/s]
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5849	imagenet-zeroshot-val-top5: 0.8517
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8b40198de17dac688fb02e016d92b550c8d0c5cf
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5e7d8ae017bef5400398d0172ac746108d6b9e45f0132a456108fea10b30c9f
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..41be5935e5410a88f26214fcfe33dd68db6ace9b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt).
+2022-12-21,10:13:40 | INFO | Model:
+2022-12-21,10:13:40 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:40 | INFO | Params:
+2022-12-21,10:13:40 | INFO |   batch_size: 256
+2022-12-21,10:13:40 | INFO |   beta1: 0.9
+2022-12-21,10:13:40 | INFO |   beta2: 0.98
+2022-12-21,10:13:40 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/checkpoints
+2022-12-21,10:13:40 | INFO |   copy_codebase: False
+2022-12-21,10:13:40 | INFO |   csv_caption_key: title
+2022-12-21,10:13:40 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:40 | INFO |   csv_separator: 	
+2022-12-21,10:13:40 | INFO |   dataset_resampled: False
+2022-12-21,10:13:40 | INFO |   dataset_type: auto
+2022-12-21,10:13:40 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:40 | INFO |   debug: False
+2022-12-21,10:13:40 | INFO |   device: cuda:0
+2022-12-21,10:13:40 | INFO |   dist_backend: nccl
+2022-12-21,10:13:40 | INFO |   dist_url: env://
+2022-12-21,10:13:40 | INFO |   distributed: False
+2022-12-21,10:13:40 | INFO |   epochs: 32
+2022-12-21,10:13:40 | INFO |   eps: 1e-06
+2022-12-21,10:13:40 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:40 | INFO |   gather_with_grad: False
+2022-12-21,10:13:40 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:40 | INFO |   horovod: False
+2022-12-21,10:13:40 | INFO |   image_mean: None
+2022-12-21,10:13:40 | INFO |   image_std: None
+2022-12-21,10:13:40 | INFO |   imagenet_v2: None
+2022-12-21,10:13:40 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:40 | INFO |   local_loss: False
+2022-12-21,10:13:40 | INFO |   local_rank: 0
+2022-12-21,10:13:40 | INFO |   lock_image: False
+2022-12-21,10:13:40 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:40 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:40 | INFO |   log_level: 20
+2022-12-21,10:13:40 | INFO |   log_local: False
+2022-12-21,10:13:40 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/out.log
+2022-12-21,10:13:40 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:40 | INFO |   lr: 0.0005
+2022-12-21,10:13:40 | INFO |   model: ViT-B-16
+2022-12-21,10:13:40 | INFO |   name: epoch_29.pt
+2022-12-21,10:13:40 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:40 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:40 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:40 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt
+2022-12-21,10:13:40 | INFO |   pretrained_image: False
+2022-12-21,10:13:40 | INFO |   rank: 0
+2022-12-21,10:13:40 | INFO |   report_to: 
+2022-12-21,10:13:40 | INFO |   resume: None
+2022-12-21,10:13:40 | INFO |   save_frequency: 1
+2022-12-21,10:13:40 | INFO |   save_most_recent: False
+2022-12-21,10:13:40 | INFO |   seed: 0
+2022-12-21,10:13:40 | INFO |   skip_scheduler: False
+2022-12-21,10:13:40 | INFO |   tensorboard: False
+2022-12-21,10:13:40 | INFO |   tensorboard_path: 
+2022-12-21,10:13:40 | INFO |   torchscript: False
+2022-12-21,10:13:40 | INFO |   trace: False
+2022-12-21,10:13:40 | INFO |   train_data: None
+2022-12-21,10:13:40 | INFO |   train_num_samples: None
+2022-12-21,10:13:40 | INFO |   use_bn_sync: False
+2022-12-21,10:13:40 | INFO |   val_data: None
+2022-12-21,10:13:40 | INFO |   val_frequency: 1
+2022-12-21,10:13:40 | INFO |   val_num_samples: None
+2022-12-21,10:13:40 | INFO |   wandb: False
+2022-12-21,10:13:40 | INFO |   wandb_notes: 
+2022-12-21,10:13:40 | INFO |   warmup: 10000
+2022-12-21,10:13:40 | INFO |   wd: 0.2
+2022-12-21,10:13:40 | INFO |   workers: 1
+2022-12-21,10:13:40 | INFO |   world_size: 1
+2022-12-21,10:13:40 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:40 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:40 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<14:33,  1.14it/s]
  1%|          | 6/1000 [00:00<02:05,  7.94it/s]
  1%|          | 12/1000 [00:01<01:01, 16.11it/s]
  2%|▏         | 18/1000 [00:01<00:40, 24.18it/s]
  2%|▏         | 24/1000 [00:01<00:31, 31.40it/s]
  3%|▎         | 30/1000 [00:01<00:26, 36.99it/s]
  4%|▎         | 36/1000 [00:01<00:23, 41.31it/s]
  4%|▍         | 42/1000 [00:01<00:21, 44.77it/s]
  5%|▍         | 48/1000 [00:01<00:20, 47.24it/s]
  5%|▌         | 54/1000 [00:01<00:19, 49.22it/s]
  6%|▌         | 60/1000 [00:01<00:18, 51.34it/s]
  7%|▋         | 66/1000 [00:02<00:17, 53.19it/s]
  7%|▋         | 72/1000 [00:02<00:17, 54.53it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.41it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.10it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.55it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.03it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.28it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.56it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.72it/s]
 12%|█▏        | 120/1000 [00:03<00:15, 57.71it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.65it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.68it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.65it/s]
 14%|█▍        | 144/1000 [00:03<00:15, 54.15it/s]
 15%|█▌        | 150/1000 [00:03<00:15, 55.26it/s]
 16%|█▌        | 156/1000 [00:03<00:15, 56.04it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 56.45it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 56.51it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 56.82it/s]
 18%|█▊        | 180/1000 [00:04<00:14, 57.00it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.09it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.13it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.16it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.03it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.96it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.11it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.09it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.35it/s]
 23%|██▎       | 234/1000 [00:05<00:13, 57.34it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.24it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.30it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.40it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.50it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.54it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.38it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.66it/s]
 29%|██▉       | 294/1000 [00:06<00:12, 57.86it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.83it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.79it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.93it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.92it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.83it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.65it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.71it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.80it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.88it/s]
 35%|███▌      | 354/1000 [00:07<00:11, 57.60it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.62it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.72it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.82it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.75it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.52it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.63it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.77it/s]
 41%|████      | 408/1000 [00:08<00:10, 57.84it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.84it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.84it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.87it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.98it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.97it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.75it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.82it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.86it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.83it/s]
 47%|████▋     | 468/1000 [00:09<00:09, 57.80it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.95it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.86it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.84it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.82it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.74it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.82it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.73it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.80it/s]
 52%|█████▏    | 522/1000 [00:10<00:08, 57.83it/s]
 53%|█████▎    | 528/1000 [00:10<00:08, 57.91it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.80it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.92it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.95it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.84it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.87it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.95it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.82it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.77it/s]
 58%|█████▊    | 582/1000 [00:11<00:07, 57.89it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.87it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.79it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.91it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.81it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.81it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.95it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.93it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.01it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.80it/s]
 64%|██████▍   | 642/1000 [00:12<00:06, 57.75it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.92it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.85it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.95it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.99it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.91it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.96it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 696/1000 [00:13<00:05, 57.85it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.95it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.95it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.93it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.89it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.89it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.86it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.85it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.91it/s]
 76%|███████▌  | 756/1000 [00:14<00:04, 58.01it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.85it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.94it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.89it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.93it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.96it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.80it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.77it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.82it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.78it/s]
 82%|████████▏ | 816/1000 [00:15<00:03, 57.77it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.81it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.90it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.05it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.08it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.00it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.03it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.95it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.94it/s]
 87%|████████▋ | 870/1000 [00:16<00:02, 57.87it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.93it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.93it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.80it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.91it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.89it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.87it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.89it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.89it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.90it/s]
 93%|█████████▎| 930/1000 [00:17<00:01, 58.01it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 58.05it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.02it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.02it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.01it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.89it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.98it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.07it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.07it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.07it/s]
 99%|█████████▉| 990/1000 [00:18<00:00, 58.04it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.87it/s]
100%|██████████| 1000/1000 [00:18<00:00, 54.78it/s]
+2022-12-21,10:13:58 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:29, 111.01it/s]
  1%|          | 512/50176 [00:04<06:30, 127.17it/s]
  2%|▏         | 768/50176 [00:05<06:03, 135.91it/s]
  2%|▏         | 1024/50176 [00:07<05:52, 139.42it/s]
  3%|▎         | 1280/50176 [00:09<05:47, 140.83it/s]
  3%|▎         | 1536/50176 [00:11<05:59, 135.42it/s]
  4%|▎         | 1792/50176 [00:13<05:54, 136.47it/s]
  4%|▍         | 2048/50176 [00:15<05:46, 138.99it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 139.09it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.45it/s]
  6%|▌         | 2816/50176 [00:20<05:40, 139.10it/s]
  6%|▌         | 3072/50176 [00:22<05:33, 141.12it/s]
  7%|▋         | 3328/50176 [00:24<05:30, 141.83it/s]
  7%|▋         | 3584/50176 [00:25<05:23, 144.05it/s]
  8%|▊         | 3840/50176 [00:27<05:33, 138.81it/s]
  8%|▊         | 4096/50176 [00:29<05:29, 139.96it/s]
  9%|▊         | 4352/50176 [00:31<05:23, 141.68it/s]
  9%|▉         | 4608/50176 [00:33<05:18, 143.22it/s]
 10%|▉         | 4864/50176 [00:34<05:11, 145.44it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.36it/s]
 11%|█         | 5376/50176 [00:38<05:04, 147.18it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.26it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 148.08it/s]
 12%|█▏        | 6144/50176 [00:43<05:24, 135.85it/s]
 13%|█▎        | 6400/50176 [00:45<05:14, 139.21it/s]
 13%|█▎        | 6656/50176 [00:47<05:08, 141.25it/s]
 14%|█▍        | 6912/50176 [00:49<05:04, 141.91it/s]
 14%|█▍        | 7168/50176 [00:50<05:00, 143.31it/s]
 15%|█▍        | 7424/50176 [00:52<04:53, 145.56it/s]
 15%|█▌        | 7680/50176 [00:54<04:53, 144.61it/s]
 16%|█▌        | 7936/50176 [00:56<04:45, 147.90it/s]
 16%|█▋        | 8192/50176 [00:57<04:45, 147.31it/s]
 17%|█▋        | 8448/50176 [00:59<04:41, 148.26it/s]
 17%|█▋        | 8704/50176 [01:01<04:38, 148.84it/s]
 18%|█▊        | 8960/50176 [01:03<04:45, 144.19it/s]
 18%|█▊        | 9216/50176 [01:04<04:37, 147.42it/s]
 19%|█▉        | 9472/50176 [01:06<04:36, 147.16it/s]
 19%|█▉        | 9728/50176 [01:08<04:39, 144.53it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.04it/s]
 20%|██        | 10240/50176 [01:11<04:26, 149.68it/s]
 21%|██        | 10496/50176 [01:13<04:24, 150.20it/s]
 21%|██▏       | 10752/50176 [01:15<04:29, 146.08it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.62it/s]
 22%|██▏       | 11264/50176 [01:18<04:21, 148.52it/s]
 23%|██▎       | 11520/50176 [01:20<04:16, 150.42it/s]
 23%|██▎       | 11776/50176 [01:21<04:16, 149.97it/s]
 24%|██▍       | 12032/50176 [01:23<04:14, 150.10it/s]
 24%|██▍       | 12288/50176 [01:25<04:11, 150.75it/s]
 25%|██▌       | 12544/50176 [01:27<04:08, 151.22it/s]
 26%|██▌       | 12800/50176 [01:28<04:06, 151.63it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.61it/s]
 27%|██▋       | 13312/50176 [01:32<04:05, 150.10it/s]
 27%|██▋       | 13568/50176 [01:34<04:12, 144.75it/s]
 28%|██▊       | 13824/50176 [01:35<04:16, 141.61it/s]
 28%|██▊       | 14080/50176 [01:37<04:13, 142.17it/s]
 29%|██▊       | 14336/50176 [01:39<04:16, 139.57it/s]
 29%|██▉       | 14592/50176 [01:41<04:06, 144.15it/s]
 30%|██▉       | 14848/50176 [01:43<04:18, 136.60it/s]
 30%|███       | 15104/50176 [01:45<04:18, 135.53it/s]
 31%|███       | 15360/50176 [01:47<04:10, 138.88it/s]
 31%|███       | 15616/50176 [01:48<04:05, 140.99it/s]
 32%|███▏      | 15872/50176 [01:50<04:09, 137.67it/s]
 32%|███▏      | 16128/50176 [01:52<04:04, 139.48it/s]
 33%|███▎      | 16384/50176 [01:54<04:01, 139.66it/s]
 33%|███▎      | 16640/50176 [01:56<04:08, 135.03it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 139.92it/s]
 34%|███▍      | 17152/50176 [01:59<03:50, 143.01it/s]
 35%|███▍      | 17408/50176 [02:01<03:45, 145.28it/s]
 35%|███▌      | 17664/50176 [02:03<03:41, 146.70it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.25it/s]
 36%|███▌      | 18176/50176 [02:06<03:36, 147.59it/s]
 37%|███▋      | 18432/50176 [02:08<03:34, 148.08it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.11it/s]
 38%|███▊      | 18944/50176 [02:12<03:40, 141.67it/s]
 38%|███▊      | 19200/50176 [02:14<03:50, 134.46it/s]
 39%|███▉      | 19456/50176 [02:16<03:53, 131.53it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.18it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.68it/s]
 40%|████      | 20224/50176 [02:21<03:32, 141.17it/s]
 41%|████      | 20480/50176 [02:23<03:26, 143.74it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 143.86it/s]
 42%|████▏     | 20992/50176 [02:26<03:18, 147.19it/s]
 42%|████▏     | 21248/50176 [02:28<03:22, 142.55it/s]
 43%|████▎     | 21504/50176 [02:30<03:15, 146.72it/s]
 43%|████▎     | 21760/50176 [02:31<03:11, 148.36it/s]
 44%|████▍     | 22016/50176 [02:33<03:05, 151.43it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.83it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.68it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.32it/s]
 46%|████▌     | 23040/50176 [02:40<03:08, 144.32it/s]
 46%|████▋     | 23296/50176 [02:42<03:11, 140.03it/s]
 47%|████▋     | 23552/50176 [02:44<03:14, 136.95it/s]
 47%|████▋     | 23808/50176 [02:46<03:07, 140.82it/s]
 48%|████▊     | 24064/50176 [02:48<03:03, 142.02it/s]
 48%|████▊     | 24320/50176 [02:49<03:01, 142.48it/s]
 49%|████▉     | 24576/50176 [02:51<02:56, 145.15it/s]
 49%|████▉     | 24832/50176 [02:53<02:50, 148.36it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 148.85it/s]
 51%|█████     | 25344/50176 [02:56<02:46, 149.52it/s]
 51%|█████     | 25600/50176 [02:58<02:48, 145.62it/s]
 52%|█████▏    | 25856/50176 [02:59<02:41, 150.62it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.50it/s]
 53%|█████▎    | 26368/50176 [03:03<02:45, 143.57it/s]
 53%|█████▎    | 26624/50176 [03:05<02:41, 145.71it/s]
 54%|█████▎    | 26880/50176 [03:07<02:40, 145.28it/s]
 54%|█████▍    | 27136/50176 [03:08<02:37, 146.48it/s]
 55%|█████▍    | 27392/50176 [03:10<02:35, 146.68it/s]
 55%|█████▌    | 27648/50176 [03:12<02:26, 153.43it/s]
 56%|█████▌    | 27904/50176 [03:14<02:36, 142.21it/s]
 56%|█████▌    | 28160/50176 [03:15<02:32, 144.08it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.41it/s]
 57%|█████▋    | 28672/50176 [03:19<02:32, 140.63it/s]
 58%|█████▊    | 28928/50176 [03:21<02:28, 143.24it/s]
 58%|█████▊    | 29184/50176 [03:22<02:23, 146.38it/s]
 59%|█████▊    | 29440/50176 [03:25<02:30, 137.94it/s]
 59%|█████▉    | 29696/50176 [03:27<02:30, 136.30it/s]
 60%|█████▉    | 29952/50176 [03:28<02:29, 135.67it/s]
 60%|██████    | 30208/50176 [03:30<02:23, 138.81it/s]
 61%|██████    | 30464/50176 [03:32<02:15, 145.20it/s]
 61%|██████    | 30720/50176 [03:34<02:14, 144.23it/s]
 62%|██████▏   | 30976/50176 [03:35<02:10, 146.81it/s]
 62%|██████▏   | 31232/50176 [03:37<02:06, 149.17it/s]
 63%|██████▎   | 31488/50176 [03:39<02:03, 150.85it/s]
 63%|██████▎   | 31744/50176 [03:40<02:01, 151.27it/s]
 64%|██████▍   | 32000/50176 [03:42<02:01, 149.58it/s]
 64%|██████▍   | 32256/50176 [03:44<01:58, 150.66it/s]
 65%|██████▍   | 32512/50176 [03:46<02:01, 145.27it/s]
 65%|██████▌   | 32768/50176 [03:47<01:56, 149.53it/s]
 66%|██████▌   | 33024/50176 [03:49<01:56, 147.56it/s]
 66%|██████▋   | 33280/50176 [03:51<01:53, 149.03it/s]
 67%|██████▋   | 33536/50176 [03:52<01:54, 144.96it/s]
 67%|██████▋   | 33792/50176 [03:54<01:50, 148.23it/s]
 68%|██████▊   | 34048/50176 [03:56<01:45, 152.29it/s]
 68%|██████▊   | 34304/50176 [03:57<01:42, 154.56it/s]
 69%|██████▉   | 34560/50176 [03:59<01:45, 148.30it/s]
 69%|██████▉   | 34816/50176 [04:01<01:43, 148.75it/s]
 70%|██████▉   | 35072/50176 [04:03<01:42, 146.98it/s]
 70%|███████   | 35328/50176 [04:04<01:38, 150.08it/s]
 71%|███████   | 35584/50176 [04:06<01:38, 147.58it/s]
 71%|███████▏  | 35840/50176 [04:08<01:40, 143.19it/s]
 72%|███████▏  | 36096/50176 [04:10<01:36, 145.28it/s]
 72%|███████▏  | 36352/50176 [04:12<01:36, 143.47it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 145.71it/s]
 73%|███████▎  | 36864/50176 [04:15<01:32, 143.92it/s]
 74%|███████▍  | 37120/50176 [04:17<01:29, 145.96it/s]
 74%|███████▍  | 37376/50176 [04:18<01:25, 150.31it/s]
 75%|███████▌  | 37632/50176 [04:20<01:22, 151.24it/s]
 76%|███████▌  | 37888/50176 [04:22<01:22, 149.26it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 150.01it/s]
 77%|███████▋  | 38400/50176 [04:25<01:18, 149.17it/s]
 77%|███████▋  | 38656/50176 [04:27<01:17, 147.71it/s]
 78%|███████▊  | 38912/50176 [04:29<01:17, 145.87it/s]
 78%|███████▊  | 39168/50176 [04:31<01:17, 141.29it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 144.07it/s]
 79%|███████▉  | 39680/50176 [04:34<01:12, 145.35it/s]
 80%|███████▉  | 39936/50176 [04:36<01:09, 147.38it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.43it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.38it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 138.24it/s]
 82%|████████▏ | 40960/50176 [04:43<01:04, 143.71it/s]
 82%|████████▏ | 41216/50176 [04:45<01:01, 146.50it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.32it/s]
 83%|████████▎ | 41728/50176 [04:48<00:55, 150.98it/s]
 84%|████████▎ | 41984/50176 [04:50<00:54, 150.95it/s]
 84%|████████▍ | 42240/50176 [04:52<00:53, 147.90it/s]
 85%|████████▍ | 42496/50176 [04:53<00:51, 149.97it/s]
 85%|████████▌ | 42752/50176 [04:55<00:49, 148.85it/s]
 86%|████████▌ | 43008/50176 [04:57<00:47, 150.53it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.44it/s]
 87%|████████▋ | 43520/50176 [05:00<00:44, 148.91it/s]
 87%|████████▋ | 43776/50176 [05:02<00:43, 146.58it/s]
 88%|████████▊ | 44032/50176 [05:04<00:41, 148.35it/s]
 88%|████████▊ | 44288/50176 [05:05<00:39, 149.93it/s]
 89%|████████▉ | 44544/50176 [05:07<00:38, 147.54it/s]
 89%|████████▉ | 44800/50176 [05:09<00:36, 145.69it/s]
 90%|████████▉ | 45056/50176 [05:11<00:34, 147.58it/s]
 90%|█████████ | 45312/50176 [05:13<00:34, 140.93it/s]
 91%|█████████ | 45568/50176 [05:14<00:31, 146.10it/s]
 91%|█████████▏| 45824/50176 [05:16<00:30, 144.84it/s]
 92%|█████████▏| 46080/50176 [05:18<00:28, 145.05it/s]
 92%|█████████▏| 46336/50176 [05:19<00:26, 145.57it/s]
 93%|█████████▎| 46592/50176 [05:21<00:24, 147.12it/s]
 93%|█████████▎| 46848/50176 [05:23<00:22, 145.96it/s]
 94%|█████████▍| 47104/50176 [05:25<00:21, 142.38it/s]
 94%|█████████▍| 47360/50176 [05:27<00:19, 143.98it/s]
 95%|█████████▍| 47616/50176 [05:28<00:17, 142.45it/s]
 95%|█████████▌| 47872/50176 [05:30<00:15, 145.01it/s]
 96%|█████████▌| 48128/50176 [05:32<00:13, 147.18it/s]
 96%|█████████▋| 48384/50176 [05:33<00:12, 148.60it/s]
 97%|█████████▋| 48640/50176 [05:35<00:10, 150.27it/s]
 97%|█████████▋| 48896/50176 [05:37<00:08, 150.44it/s]
 98%|█████████▊| 49152/50176 [05:39<00:06, 147.19it/s]
 98%|█████████▊| 49408/50176 [05:40<00:05, 145.56it/s]
 99%|█████████▉| 49664/50176 [05:42<00:03, 145.32it/s]
 99%|█████████▉| 49920/50176 [05:44<00:01, 137.68it/s]
100%|██████████| 50176/50176 [05:45<00:00, 180.03it/s]
100%|██████████| 50176/50176 [05:45<00:00, 145.35it/s]
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5796	imagenet-zeroshot-val-top5: 0.8498
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1bce8b203ff1ec96265b8b31718eece48941405e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eba4cb72f108fe3a0e728c9fbd309a2f85479ffd2d25292d7a31dd9715094a20
+size 1795822369
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..42ca5b6ce382cfb3f13c181b1c3594f50a33a010
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt).
+2022-12-21,10:13:41 | INFO | Model:
+2022-12-21,10:13:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:42 | INFO | Params:
+2022-12-21,10:13:42 | INFO |   batch_size: 256
+2022-12-21,10:13:42 | INFO |   beta1: 0.9
+2022-12-21,10:13:42 | INFO |   beta2: 0.98
+2022-12-21,10:13:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/checkpoints
+2022-12-21,10:13:42 | INFO |   copy_codebase: False
+2022-12-21,10:13:42 | INFO |   csv_caption_key: title
+2022-12-21,10:13:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:42 | INFO |   csv_separator: 	
+2022-12-21,10:13:42 | INFO |   dataset_resampled: False
+2022-12-21,10:13:42 | INFO |   dataset_type: auto
+2022-12-21,10:13:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:42 | INFO |   debug: False
+2022-12-21,10:13:42 | INFO |   device: cuda:0
+2022-12-21,10:13:42 | INFO |   dist_backend: nccl
+2022-12-21,10:13:42 | INFO |   dist_url: env://
+2022-12-21,10:13:42 | INFO |   distributed: False
+2022-12-21,10:13:42 | INFO |   epochs: 32
+2022-12-21,10:13:42 | INFO |   eps: 1e-06
+2022-12-21,10:13:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:42 | INFO |   gather_with_grad: False
+2022-12-21,10:13:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:42 | INFO |   horovod: False
+2022-12-21,10:13:42 | INFO |   image_mean: None
+2022-12-21,10:13:42 | INFO |   image_std: None
+2022-12-21,10:13:42 | INFO |   imagenet_v2: None
+2022-12-21,10:13:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:42 | INFO |   local_loss: False
+2022-12-21,10:13:42 | INFO |   local_rank: 0
+2022-12-21,10:13:42 | INFO |   lock_image: False
+2022-12-21,10:13:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:42 | INFO |   log_level: 20
+2022-12-21,10:13:42 | INFO |   log_local: False
+2022-12-21,10:13:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/out.log
+2022-12-21,10:13:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:42 | INFO |   lr: 0.0005
+2022-12-21,10:13:42 | INFO |   model: ViT-B-16
+2022-12-21,10:13:42 | INFO |   name: epoch_3.pt
+2022-12-21,10:13:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt
+2022-12-21,10:13:42 | INFO |   pretrained_image: False
+2022-12-21,10:13:42 | INFO |   rank: 0
+2022-12-21,10:13:42 | INFO |   report_to: 
+2022-12-21,10:13:42 | INFO |   resume: None
+2022-12-21,10:13:42 | INFO |   save_frequency: 1
+2022-12-21,10:13:42 | INFO |   save_most_recent: False
+2022-12-21,10:13:42 | INFO |   seed: 0
+2022-12-21,10:13:42 | INFO |   skip_scheduler: False
+2022-12-21,10:13:42 | INFO |   tensorboard: False
+2022-12-21,10:13:42 | INFO |   tensorboard_path: 
+2022-12-21,10:13:42 | INFO |   torchscript: False
+2022-12-21,10:13:42 | INFO |   trace: False
+2022-12-21,10:13:42 | INFO |   train_data: None
+2022-12-21,10:13:42 | INFO |   train_num_samples: None
+2022-12-21,10:13:42 | INFO |   use_bn_sync: False
+2022-12-21,10:13:42 | INFO |   val_data: None
+2022-12-21,10:13:42 | INFO |   val_frequency: 1
+2022-12-21,10:13:42 | INFO |   val_num_samples: None
+2022-12-21,10:13:42 | INFO |   wandb: False
+2022-12-21,10:13:42 | INFO |   wandb_notes: 
+2022-12-21,10:13:42 | INFO |   warmup: 10000
+2022-12-21,10:13:42 | INFO |   wd: 0.2
+2022-12-21,10:13:42 | INFO |   workers: 1
+2022-12-21,10:13:42 | INFO |   world_size: 1
+2022-12-21,10:13:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:42 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:42 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:24,  1.60it/s]
  1%|          | 6/1000 [00:00<01:34, 10.53it/s]
  1%|          | 12/1000 [00:00<00:48, 20.51it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.19it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.27it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.87it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.05it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.14it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.45it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.09it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.30it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.79it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.50it/s]
  8%|▊         | 78/1000 [00:01<00:16, 55.99it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.52it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.49it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.96it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.19it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.50it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.55it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.56it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.53it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.59it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.55it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.60it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.67it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.72it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.59it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.47it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.47it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.44it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.28it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.25it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.21it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.03it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.91it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.03it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.00it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.22it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.08it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.05it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.14it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.28it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.41it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.14it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.06it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.14it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.37it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.51it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.74it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.74it/s]
 31%|███       | 306/1000 [00:05<00:12, 57.73it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.88it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.86it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.76it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.59it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.66it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.75it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.83it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 53.98it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 55.03it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 55.87it/s]
 37%|███▋      | 372/1000 [00:07<00:11, 56.48it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 56.75it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 56.73it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 56.99it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.23it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.47it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.61it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.65it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.69it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.72it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.86it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.86it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.66it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.73it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.78it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.77it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.75it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.89it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.80it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.77it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.77it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.69it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.58it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.65it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.70it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.79it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.73it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.85it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.86it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.75it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.47it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.61it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.55it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.53it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.70it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.71it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.65it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.76it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.68it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.69it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.80it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.80it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.89it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.71it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.64it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.70it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.67it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.77it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.82it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.76it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.82it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.83it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.86it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.75it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.83it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.69it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.69it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.73it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.71it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.72it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.72it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.82it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.91it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.75it/s]
 77%|███████▋  | 768/1000 [00:13<00:04, 57.69it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.69it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.75it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.81it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.63it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.64it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.71it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.69it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.66it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.69it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.63it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.80it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.86it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.79it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.86it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.52it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.61it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.62it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.74it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 57.76it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.65it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.77it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.78it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.75it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.77it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.78it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.75it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.46it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.60it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.66it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.74it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.82it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.77it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.86it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.94it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.98it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.73it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.83it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.71it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.53it/s]
+2022-12-21,10:14:00 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:58, 139.42it/s]
  1%|          | 512/50176 [00:03<05:10, 159.99it/s]
  2%|▏         | 768/50176 [00:04<04:44, 173.40it/s]
  2%|▏         | 1024/50176 [00:06<04:59, 164.33it/s]
  3%|▎         | 1280/50176 [00:08<05:11, 157.09it/s]
  3%|▎         | 1536/50176 [00:10<05:36, 144.56it/s]
  4%|▎         | 1792/50176 [00:11<05:38, 143.13it/s]
  4%|▍         | 2048/50176 [00:13<05:35, 143.62it/s]
  5%|▍         | 2304/50176 [00:15<05:37, 141.84it/s]
  5%|▌         | 2560/50176 [00:17<05:34, 142.48it/s]
  6%|▌         | 2816/50176 [00:19<05:38, 139.95it/s]
  6%|▌         | 3072/50176 [00:20<05:31, 142.18it/s]
  7%|▋         | 3328/50176 [00:22<05:28, 142.52it/s]
  7%|▋         | 3584/50176 [00:24<05:21, 144.89it/s]
  8%|▊         | 3840/50176 [00:26<05:32, 139.47it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.36it/s]
  9%|▊         | 4352/50176 [00:29<05:22, 142.04it/s]
  9%|▉         | 4608/50176 [00:31<05:17, 143.43it/s]
 10%|▉         | 4864/50176 [00:33<05:11, 145.68it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.29it/s]
 11%|█         | 5376/50176 [00:36<05:04, 147.16it/s]
 11%|█         | 5632/50176 [00:38<05:00, 148.16it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.18it/s]
 12%|█▏        | 6144/50176 [00:42<05:24, 135.62it/s]
 13%|█▎        | 6400/50176 [00:44<05:14, 139.40it/s]
 13%|█▎        | 6656/50176 [00:46<05:07, 141.61it/s]
 14%|█▍        | 6912/50176 [00:47<05:05, 141.58it/s]
 14%|█▍        | 7168/50176 [00:49<05:00, 142.94it/s]
 15%|█▍        | 7424/50176 [00:51<04:53, 145.79it/s]
 15%|█▌        | 7680/50176 [00:53<04:54, 144.44it/s]
 16%|█▌        | 7936/50176 [00:54<04:45, 148.01it/s]
 16%|█▋        | 8192/50176 [00:56<04:45, 147.11it/s]
 17%|█▋        | 8448/50176 [00:58<04:41, 148.41it/s]
 17%|█▋        | 8704/50176 [00:59<04:38, 149.11it/s]
 18%|█▊        | 8960/50176 [01:01<04:46, 143.67it/s]
 18%|█▊        | 9216/50176 [01:03<04:37, 147.72it/s]
 19%|█▉        | 9472/50176 [01:05<04:36, 147.29it/s]
 19%|█▉        | 9728/50176 [01:06<04:39, 144.52it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 147.95it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.64it/s]
 21%|██        | 10496/50176 [01:11<04:23, 150.34it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.30it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 147.97it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.40it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.71it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.73it/s]
 24%|██▍       | 12032/50176 [01:22<04:13, 150.41it/s]
 24%|██▍       | 12288/50176 [01:23<04:11, 150.73it/s]
 25%|██▌       | 12544/50176 [01:25<04:08, 151.40it/s]
 26%|██▌       | 12800/50176 [01:27<04:06, 151.56it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.76it/s]
 27%|██▋       | 13312/50176 [01:30<04:05, 150.03it/s]
 27%|██▋       | 13568/50176 [01:32<04:12, 144.84it/s]
 28%|██▊       | 13824/50176 [01:34<04:16, 141.53it/s]
 28%|██▊       | 14080/50176 [01:36<04:13, 142.23it/s]
 29%|██▊       | 14336/50176 [01:38<04:18, 138.91it/s]
 29%|██▉       | 14592/50176 [01:39<04:06, 144.44it/s]
 30%|██▉       | 14848/50176 [01:42<04:18, 136.68it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.13it/s]
 31%|███       | 15360/50176 [01:45<04:10, 139.09it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.21it/s]
 32%|███▏      | 15872/50176 [01:49<04:10, 137.20it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.89it/s]
 33%|███▎      | 16384/50176 [01:52<04:01, 139.87it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 135.03it/s]
 34%|███▎      | 16896/50176 [01:56<03:57, 140.04it/s]
 34%|███▍      | 17152/50176 [01:58<03:50, 143.09it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.03it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.92it/s]
 36%|███▌      | 17920/50176 [02:03<03:36, 149.08it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.72it/s]
 37%|███▋      | 18432/50176 [02:06<03:34, 147.91it/s]
 37%|███▋      | 18688/50176 [02:08<03:33, 147.29it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.26it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.25it/s]
 39%|███▉      | 19456/50176 [02:14<03:53, 131.48it/s]
 39%|███▉      | 19712/50176 [02:16<03:51, 131.70it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.71it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.31it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.90it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 144.07it/s]
 42%|████▏     | 20992/50176 [02:25<03:18, 147.29it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 142.08it/s]
 43%|████▎     | 21504/50176 [02:28<03:15, 146.85it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.43it/s]
 44%|████▍     | 22016/50176 [02:32<03:05, 151.66it/s]
 44%|████▍     | 22272/50176 [02:33<03:04, 150.92it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.61it/s]
 45%|████▌     | 22784/50176 [02:37<03:07, 145.85it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 143.90it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.52it/s]
 47%|████▋     | 23552/50176 [02:43<03:14, 136.92it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.02it/s]
 48%|████▊     | 24064/50176 [02:46<03:04, 141.87it/s]
 48%|████▊     | 24320/50176 [02:48<03:01, 142.44it/s]
 49%|████▉     | 24576/50176 [02:50<02:56, 145.21it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.70it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.94it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.73it/s]
 51%|█████     | 25600/50176 [02:57<02:48, 146.27it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.54it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.35it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.07it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 145.41it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.23it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 146.12it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.62it/s]
 55%|█████▌    | 27648/50176 [03:10<02:26, 153.27it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 142.67it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.59it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.41it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.69it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 143.23it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 145.98it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.01it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.10it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.36it/s]
 60%|██████    | 30208/50176 [03:29<02:23, 138.98it/s]
 61%|██████    | 30464/50176 [03:30<02:15, 145.40it/s]
 61%|██████    | 30720/50176 [03:32<02:14, 144.19it/s]
 62%|██████▏   | 30976/50176 [03:34<02:10, 146.99it/s]
 62%|██████▏   | 31232/50176 [03:36<02:06, 149.17it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.41it/s]
 63%|██████▎   | 31744/50176 [03:39<02:01, 151.32it/s]
 64%|██████▍   | 32000/50176 [03:41<02:01, 149.74it/s]
 64%|██████▍   | 32256/50176 [03:42<01:58, 150.67it/s]
 65%|██████▍   | 32512/50176 [03:44<02:00, 146.01it/s]
 65%|██████▌   | 32768/50176 [03:46<01:56, 149.31it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 147.66it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 149.01it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 145.23it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 148.12it/s]
 68%|██████▊   | 34048/50176 [03:54<01:45, 152.25it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 154.48it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.58it/s]
 69%|██████▉   | 34816/50176 [04:00<01:43, 148.61it/s]
 70%|██████▉   | 35072/50176 [04:01<01:42, 146.80it/s]
 70%|███████   | 35328/50176 [04:03<01:38, 150.05it/s]
 71%|███████   | 35584/50176 [04:05<01:38, 147.85it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.46it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.70it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 143.47it/s]
 73%|███████▎  | 36608/50176 [04:12<01:32, 146.03it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.82it/s]
 74%|███████▍  | 37120/50176 [04:15<01:29, 146.06it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 150.23it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.54it/s]
 76%|███████▌  | 37888/50176 [04:20<01:22, 149.12it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.89it/s]
 77%|███████▋  | 38400/50176 [04:24<01:19, 149.04it/s]
 77%|███████▋  | 38656/50176 [04:26<01:17, 147.80it/s]
 78%|███████▊  | 38912/50176 [04:27<01:17, 145.79it/s]
 78%|███████▊  | 39168/50176 [04:29<01:17, 141.92it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.81it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.32it/s]
 80%|███████▉  | 39936/50176 [04:34<01:09, 146.60it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.64it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.38it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.74it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.58it/s]
 82%|████████▏ | 41216/50176 [04:43<01:01, 146.52it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.06it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.02it/s]
 84%|████████▎ | 41984/50176 [04:48<00:54, 150.91it/s]
 84%|████████▍ | 42240/50176 [04:50<00:53, 147.97it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.88it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 148.92it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 150.44it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.54it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.85it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.65it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.30it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 149.98it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.51it/s]
 89%|████████▉ | 44800/50176 [05:07<00:36, 145.77it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.53it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 141.22it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.97it/s]
 91%|█████████▏| 45824/50176 [05:15<00:30, 144.69it/s]
 92%|█████████▏| 46080/50176 [05:16<00:28, 144.97it/s]
 92%|█████████▏| 46336/50176 [05:18<00:26, 145.62it/s]
 93%|█████████▎| 46592/50176 [05:20<00:24, 147.03it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 146.04it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 141.85it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 144.03it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.29it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 144.49it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 147.74it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.82it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.35it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 150.62it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 147.29it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.63it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.28it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 137.62it/s]
100%|██████████| 50176/50176 [05:43<00:00, 179.94it/s]
100%|██████████| 50176/50176 [05:43<00:00, 145.92it/s]
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.3597	imagenet-zeroshot-val-top5: 0.6317
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f5f5f35fb6a8a0de1dee223dd6cac2691f219db3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a93291f813f8f08018d385a88564d3f4b1099224d90d676eb709d98e5c40820
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9e2757c413e55d8f762dd023dc7e2cfe0afd4c9d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt).
+2022-12-21,10:13:41 | INFO | Model:
+2022-12-21,10:13:41 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:41 | INFO | Params:
+2022-12-21,10:13:41 | INFO |   batch_size: 256
+2022-12-21,10:13:41 | INFO |   beta1: 0.9
+2022-12-21,10:13:41 | INFO |   beta2: 0.98
+2022-12-21,10:13:41 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/checkpoints
+2022-12-21,10:13:41 | INFO |   copy_codebase: False
+2022-12-21,10:13:41 | INFO |   csv_caption_key: title
+2022-12-21,10:13:41 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:41 | INFO |   csv_separator: 	
+2022-12-21,10:13:41 | INFO |   dataset_resampled: False
+2022-12-21,10:13:41 | INFO |   dataset_type: auto
+2022-12-21,10:13:41 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:41 | INFO |   debug: False
+2022-12-21,10:13:41 | INFO |   device: cuda:0
+2022-12-21,10:13:41 | INFO |   dist_backend: nccl
+2022-12-21,10:13:41 | INFO |   dist_url: env://
+2022-12-21,10:13:41 | INFO |   distributed: False
+2022-12-21,10:13:41 | INFO |   epochs: 32
+2022-12-21,10:13:41 | INFO |   eps: 1e-06
+2022-12-21,10:13:41 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:41 | INFO |   gather_with_grad: False
+2022-12-21,10:13:41 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:41 | INFO |   horovod: False
+2022-12-21,10:13:41 | INFO |   image_mean: None
+2022-12-21,10:13:41 | INFO |   image_std: None
+2022-12-21,10:13:41 | INFO |   imagenet_v2: None
+2022-12-21,10:13:41 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:41 | INFO |   local_loss: False
+2022-12-21,10:13:41 | INFO |   local_rank: 0
+2022-12-21,10:13:41 | INFO |   lock_image: False
+2022-12-21,10:13:41 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:41 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:41 | INFO |   log_level: 20
+2022-12-21,10:13:41 | INFO |   log_local: False
+2022-12-21,10:13:41 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/out.log
+2022-12-21,10:13:41 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:41 | INFO |   lr: 0.0005
+2022-12-21,10:13:41 | INFO |   model: ViT-B-16
+2022-12-21,10:13:41 | INFO |   name: epoch_30.pt
+2022-12-21,10:13:41 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:41 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:41 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:41 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt
+2022-12-21,10:13:41 | INFO |   pretrained_image: False
+2022-12-21,10:13:41 | INFO |   rank: 0
+2022-12-21,10:13:41 | INFO |   report_to: 
+2022-12-21,10:13:41 | INFO |   resume: None
+2022-12-21,10:13:41 | INFO |   save_frequency: 1
+2022-12-21,10:13:41 | INFO |   save_most_recent: False
+2022-12-21,10:13:41 | INFO |   seed: 0
+2022-12-21,10:13:41 | INFO |   skip_scheduler: False
+2022-12-21,10:13:41 | INFO |   tensorboard: False
+2022-12-21,10:13:41 | INFO |   tensorboard_path: 
+2022-12-21,10:13:41 | INFO |   torchscript: False
+2022-12-21,10:13:41 | INFO |   trace: False
+2022-12-21,10:13:41 | INFO |   train_data: None
+2022-12-21,10:13:41 | INFO |   train_num_samples: None
+2022-12-21,10:13:41 | INFO |   use_bn_sync: False
+2022-12-21,10:13:41 | INFO |   val_data: None
+2022-12-21,10:13:41 | INFO |   val_frequency: 1
+2022-12-21,10:13:41 | INFO |   val_num_samples: None
+2022-12-21,10:13:41 | INFO |   wandb: False
+2022-12-21,10:13:41 | INFO |   wandb_notes: 
+2022-12-21,10:13:41 | INFO |   warmup: 10000
+2022-12-21,10:13:41 | INFO |   wd: 0.2
+2022-12-21,10:13:41 | INFO |   workers: 1
+2022-12-21,10:13:41 | INFO |   world_size: 1
+2022-12-21,10:13:41 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:41 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:41 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:24,  1.60it/s]
  1%|          | 6/1000 [00:00<01:33, 10.58it/s]
  1%|          | 12/1000 [00:00<00:48, 20.21it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.41it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.59it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.30it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.65it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.91it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.36it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.01it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.31it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.25it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.91it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.83it/s]
  9%|▉         | 90/1000 [00:02<00:17, 50.60it/s]
 10%|▉         | 96/1000 [00:02<00:17, 52.71it/s]
 10%|█         | 102/1000 [00:02<00:16, 54.22it/s]
 11%|█         | 108/1000 [00:02<00:16, 55.42it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 56.26it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 56.73it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 56.99it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.00it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.22it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.44it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.65it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.77it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.61it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.66it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.65it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.59it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.51it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.45it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.28it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.18it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.27it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.48it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.47it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.39it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.47it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.58it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.70it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.74it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.57it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.60it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.80it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.82it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.03it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.00it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.98it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.11it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.11it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.00it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.81it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.90it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.11it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.87it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.86it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.92it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.52it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.59it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.73it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.87it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.96it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.95it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.97it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.98it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.11it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.11it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.89it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.97it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.01it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.99it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.95it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.07it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.97it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.96it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.96it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.95it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.87it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.99it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.08it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.00it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.07it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.10it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.01it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.07it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.90it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.03it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.00it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.91it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.05it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.98it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.98it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.10it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.07it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.16it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.98it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.92it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.07it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.01it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.07it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.13it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.08it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.12it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.11it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.12it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.00it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.09it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.04it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.71it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.83it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.83it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.90it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.94it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.95it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.04it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.06it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.88it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.00it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.96it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.02it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.07it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.89it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.89it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.95it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.92it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.91it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.93it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.04it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.18it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.20it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.12it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.18it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.12it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.11it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.10it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.07it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.94it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.05it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.05it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.00it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.03it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.01it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.10it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.15it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.12it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.84it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.87it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.90it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.02it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.16it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.21it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.21it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.18it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.01it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.68it/s]
+2022-12-21,10:13:59 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:03, 137.32it/s]
  1%|          | 512/50176 [00:03<05:12, 158.76it/s]
  2%|▏         | 768/50176 [00:04<05:08, 159.96it/s]
  2%|▏         | 1024/50176 [00:06<05:17, 154.72it/s]
  3%|▎         | 1280/50176 [00:08<05:25, 150.30it/s]
  3%|▎         | 1536/50176 [00:10<05:45, 140.87it/s]
  4%|▎         | 1792/50176 [00:12<05:45, 139.84it/s]
  4%|▍         | 2048/50176 [00:14<05:39, 141.73it/s]
  5%|▍         | 2304/50176 [00:15<05:39, 140.94it/s]
  5%|▌         | 2560/50176 [00:17<05:36, 141.49it/s]
  6%|▌         | 2816/50176 [00:19<05:39, 139.53it/s]
  6%|▌         | 3072/50176 [00:21<05:32, 141.73it/s]
  7%|▋         | 3328/50176 [00:23<05:28, 142.48it/s]
  7%|▋         | 3584/50176 [00:24<05:22, 144.44it/s]
  8%|▊         | 3840/50176 [00:26<05:31, 139.67it/s]
  8%|▊         | 4096/50176 [00:28<05:29, 139.90it/s]
  9%|▊         | 4352/50176 [00:30<05:24, 141.32it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.42it/s]
 10%|▉         | 4864/50176 [00:33<05:11, 145.51it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.39it/s]
 11%|█         | 5376/50176 [00:37<05:04, 147.29it/s]
 11%|█         | 5632/50176 [00:38<05:00, 148.36it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.13it/s]
 12%|█▏        | 6144/50176 [00:42<05:23, 135.98it/s]
 13%|█▎        | 6400/50176 [00:44<05:14, 139.18it/s]
 13%|█▎        | 6656/50176 [00:46<05:08, 141.13it/s]
 14%|█▍        | 6912/50176 [00:48<05:05, 141.67it/s]
 14%|█▍        | 7168/50176 [00:49<05:00, 143.29it/s]
 15%|█▍        | 7424/50176 [00:51<04:53, 145.67it/s]
 15%|█▌        | 7680/50176 [00:53<04:53, 144.62it/s]
 16%|█▌        | 7936/50176 [00:55<04:44, 148.29it/s]
 16%|█▋        | 8192/50176 [00:56<04:45, 147.01it/s]
 17%|█▋        | 8448/50176 [00:58<04:41, 148.46it/s]
 17%|█▋        | 8704/50176 [01:00<04:38, 148.78it/s]
 18%|█▊        | 8960/50176 [01:02<04:46, 143.87it/s]
 18%|█▊        | 9216/50176 [01:03<04:37, 147.51it/s]
 19%|█▉        | 9472/50176 [01:05<04:36, 147.35it/s]
 19%|█▉        | 9728/50176 [01:07<04:41, 143.82it/s]
 20%|█▉        | 9984/50176 [01:08<04:30, 148.36it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.78it/s]
 21%|██        | 10496/50176 [01:12<04:23, 150.39it/s]
 21%|██▏       | 10752/50176 [01:14<04:30, 145.69it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 147.93it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.38it/s]
 23%|██▎       | 11520/50176 [01:19<04:16, 150.54it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.87it/s]
 24%|██▍       | 12032/50176 [01:22<04:13, 150.43it/s]
 24%|██▍       | 12288/50176 [01:24<04:12, 149.86it/s]
 25%|██▌       | 12544/50176 [01:26<04:08, 151.60it/s]
 26%|██▌       | 12800/50176 [01:27<04:06, 151.55it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.76it/s]
 27%|██▋       | 13312/50176 [01:31<04:05, 150.05it/s]
 27%|██▋       | 13568/50176 [01:33<04:13, 144.49it/s]
 28%|██▊       | 13824/50176 [01:34<04:16, 141.94it/s]
 28%|██▊       | 14080/50176 [01:36<04:13, 142.34it/s]
 29%|██▊       | 14336/50176 [01:38<04:17, 139.18it/s]
 29%|██▉       | 14592/50176 [01:40<04:06, 144.65it/s]
 30%|██▉       | 14848/50176 [01:42<04:18, 136.66it/s]
 30%|███       | 15104/50176 [01:44<04:19, 135.35it/s]
 31%|███       | 15360/50176 [01:46<04:10, 138.91it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.33it/s]
 32%|███▏      | 15872/50176 [01:49<04:10, 137.03it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.70it/s]
 33%|███▎      | 16384/50176 [01:53<04:01, 139.72it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 135.02it/s]
 34%|███▎      | 16896/50176 [01:57<03:57, 139.95it/s]
 34%|███▍      | 17152/50176 [01:58<03:50, 143.12it/s]
 35%|███▍      | 17408/50176 [02:00<03:46, 144.96it/s]
 35%|███▌      | 17664/50176 [02:02<03:41, 146.90it/s]
 36%|███▌      | 17920/50176 [02:03<03:36, 149.13it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.99it/s]
 37%|███▋      | 18432/50176 [02:07<03:34, 148.06it/s]
 37%|███▋      | 18688/50176 [02:09<03:33, 147.41it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.24it/s]
 38%|███▊      | 19200/50176 [02:13<03:50, 134.25it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.43it/s]
 39%|███▉      | 19712/50176 [02:17<03:50, 132.01it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.79it/s]
 40%|████      | 20224/50176 [02:20<03:32, 141.25it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.73it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 143.89it/s]
 42%|████▏     | 20992/50176 [02:25<03:18, 147.21it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 142.27it/s]
 43%|████▎     | 21504/50176 [02:29<03:15, 146.73it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.48it/s]
 44%|████▍     | 22016/50176 [02:32<03:05, 151.52it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.75it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.55it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.62it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 144.00it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.44it/s]
 47%|████▋     | 23552/50176 [02:43<03:13, 137.42it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.72it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 141.99it/s]
 48%|████▊     | 24320/50176 [02:48<03:01, 142.55it/s]
 49%|████▉     | 24576/50176 [02:50<02:56, 145.15it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.42it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.81it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.53it/s]
 51%|█████     | 25600/50176 [02:57<02:48, 146.19it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.50it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.41it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.00it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 145.42it/s]
 54%|█████▎    | 26880/50176 [03:06<02:40, 145.16it/s]
 54%|█████▍    | 27136/50176 [03:07<02:38, 145.78it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.85it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.45it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.51it/s]
 56%|█████▌    | 28160/50176 [03:14<02:32, 144.06it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.41it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.96it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.16it/s]
 58%|█████▊    | 29184/50176 [03:22<02:23, 146.18it/s]
 59%|█████▊    | 29440/50176 [03:24<02:30, 138.09it/s]
 59%|█████▉    | 29696/50176 [03:26<02:31, 135.56it/s]
 60%|█████▉    | 29952/50176 [03:27<02:28, 135.87it/s]
 60%|██████    | 30208/50176 [03:29<02:23, 138.83it/s]
 61%|██████    | 30464/50176 [03:31<02:15, 145.34it/s]
 61%|██████    | 30720/50176 [03:33<02:14, 144.23it/s]
 62%|██████▏   | 30976/50176 [03:34<02:10, 146.90it/s]
 62%|██████▏   | 31232/50176 [03:36<02:07, 149.07it/s]
 63%|██████▎   | 31488/50176 [03:38<02:03, 150.74it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.71it/s]
 64%|██████▍   | 32000/50176 [03:41<02:01, 149.81it/s]
 64%|██████▍   | 32256/50176 [03:43<01:59, 150.05it/s]
 65%|██████▍   | 32512/50176 [03:45<02:00, 146.14it/s]
 65%|██████▌   | 32768/50176 [03:46<01:56, 149.46it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 147.67it/s]
 66%|██████▋   | 33280/50176 [03:50<01:53, 148.99it/s]
 67%|██████▋   | 33536/50176 [03:52<01:55, 144.53it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 148.38it/s]
 68%|██████▊   | 34048/50176 [03:55<01:45, 152.49it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 154.56it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 147.71it/s]
 69%|██████▉   | 34816/50176 [04:00<01:43, 148.98it/s]
 70%|██████▉   | 35072/50176 [04:02<01:42, 147.13it/s]
 70%|███████   | 35328/50176 [04:03<01:38, 150.20it/s]
 71%|███████   | 35584/50176 [04:05<01:38, 147.65it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.83it/s]
 72%|███████▏  | 36096/50176 [04:09<01:36, 145.47it/s]
 72%|███████▏  | 36352/50176 [04:11<01:36, 143.59it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.78it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.98it/s]
 74%|███████▍  | 37120/50176 [04:16<01:29, 146.09it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 150.32it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.24it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 149.27it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 150.06it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.14it/s]
 77%|███████▋  | 38656/50176 [04:26<01:17, 147.77it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.84it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.86it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.84it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.28it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.17it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.46it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 144.00it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.32it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.36it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 145.76it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.48it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.06it/s]
 84%|████████▎ | 41984/50176 [04:49<00:54, 150.95it/s]
 84%|████████▍ | 42240/50176 [04:51<00:53, 147.43it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.87it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.09it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.65it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.55it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.92it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.61it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 148.34it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.03it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.53it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 145.72it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 147.58it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 141.15it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.05it/s]
 91%|█████████▏| 45824/50176 [05:15<00:30, 144.99it/s]
 92%|█████████▏| 46080/50176 [05:17<00:28, 144.91it/s]
 92%|█████████▏| 46336/50176 [05:18<00:26, 145.55it/s]
 93%|█████████▎| 46592/50176 [05:20<00:24, 147.10it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 145.95it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 142.33it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 144.12it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.46it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.20it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 146.96it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.57it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.13it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 150.44it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.13it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.51it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.31it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 137.56it/s]
100%|██████████| 50176/50176 [05:44<00:00, 180.14it/s]
100%|██████████| 50176/50176 [05:44<00:00, 145.76it/s]
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5460	imagenet-zeroshot-val-top5: 0.8225
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt
new file mode 100644
index 0000000000000000000000000000000000000000..54e9060e68cf7c3a2d43acffd1b141e11911cb93
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:03a35e10031a0f941d69633e872a951a274491f46a6d2eabfb76340f7826f6f1
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a9842e23fa7298bf642ac7be56880faae21468e4
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt).
+2022-12-21,10:19:58 | INFO | Model:
+2022-12-21,10:19:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:58 | INFO | Params:
+2022-12-21,10:19:58 | INFO |   batch_size: 256
+2022-12-21,10:19:58 | INFO |   beta1: 0.9
+2022-12-21,10:19:58 | INFO |   beta2: 0.98
+2022-12-21,10:19:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/checkpoints
+2022-12-21,10:19:58 | INFO |   copy_codebase: False
+2022-12-21,10:19:58 | INFO |   csv_caption_key: title
+2022-12-21,10:19:58 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:58 | INFO |   csv_separator: 	
+2022-12-21,10:19:58 | INFO |   dataset_resampled: False
+2022-12-21,10:19:58 | INFO |   dataset_type: auto
+2022-12-21,10:19:58 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:58 | INFO |   debug: False
+2022-12-21,10:19:58 | INFO |   device: cuda:0
+2022-12-21,10:19:58 | INFO |   dist_backend: nccl
+2022-12-21,10:19:58 | INFO |   dist_url: env://
+2022-12-21,10:19:58 | INFO |   distributed: False
+2022-12-21,10:19:58 | INFO |   epochs: 32
+2022-12-21,10:19:58 | INFO |   eps: 1e-06
+2022-12-21,10:19:58 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:58 | INFO |   gather_with_grad: False
+2022-12-21,10:19:58 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:58 | INFO |   horovod: False
+2022-12-21,10:19:58 | INFO |   image_mean: None
+2022-12-21,10:19:58 | INFO |   image_std: None
+2022-12-21,10:19:58 | INFO |   imagenet_v2: None
+2022-12-21,10:19:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:58 | INFO |   local_loss: False
+2022-12-21,10:19:58 | INFO |   local_rank: 0
+2022-12-21,10:19:58 | INFO |   lock_image: False
+2022-12-21,10:19:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:58 | INFO |   log_level: 20
+2022-12-21,10:19:58 | INFO |   log_local: False
+2022-12-21,10:19:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/out.log
+2022-12-21,10:19:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:58 | INFO |   lr: 0.0005
+2022-12-21,10:19:58 | INFO |   model: ViT-B-16
+2022-12-21,10:19:58 | INFO |   name: epoch_31.pt
+2022-12-21,10:19:58 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:58 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:58 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt
+2022-12-21,10:19:58 | INFO |   pretrained_image: False
+2022-12-21,10:19:58 | INFO |   rank: 0
+2022-12-21,10:19:58 | INFO |   report_to: 
+2022-12-21,10:19:58 | INFO |   resume: None
+2022-12-21,10:19:58 | INFO |   save_frequency: 1
+2022-12-21,10:19:58 | INFO |   save_most_recent: False
+2022-12-21,10:19:58 | INFO |   seed: 0
+2022-12-21,10:19:58 | INFO |   skip_scheduler: False
+2022-12-21,10:19:58 | INFO |   tensorboard: False
+2022-12-21,10:19:58 | INFO |   tensorboard_path: 
+2022-12-21,10:19:58 | INFO |   torchscript: False
+2022-12-21,10:19:58 | INFO |   trace: False
+2022-12-21,10:19:58 | INFO |   train_data: None
+2022-12-21,10:19:58 | INFO |   train_num_samples: None
+2022-12-21,10:19:58 | INFO |   use_bn_sync: False
+2022-12-21,10:19:58 | INFO |   val_data: None
+2022-12-21,10:19:58 | INFO |   val_frequency: 1
+2022-12-21,10:19:58 | INFO |   val_num_samples: None
+2022-12-21,10:19:58 | INFO |   wandb: False
+2022-12-21,10:19:58 | INFO |   wandb_notes: 
+2022-12-21,10:19:58 | INFO |   warmup: 10000
+2022-12-21,10:19:58 | INFO |   wd: 0.2
+2022-12-21,10:19:58 | INFO |   workers: 1
+2022-12-21,10:19:58 | INFO |   world_size: 1
+2022-12-21,10:19:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:58 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:14,  1.36it/s]
  1%|          | 6/1000 [00:00<01:48,  9.18it/s]
  1%|          | 12/1000 [00:00<00:54, 18.11it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.25it/s]
  2%|▏         | 24/1000 [00:01<00:29, 33.16it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.73it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.41it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.09it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.89it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.95it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.44it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.57it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.32it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.87it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.38it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.70it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.06it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.22it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.44it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.56it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.56it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.49it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.54it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.51it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.59it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.72it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.55it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.43it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.45it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.41it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.32it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.29it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.22it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.00it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.88it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.00it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.95it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.16it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.16it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.79it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 56.95it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.13it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.28it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.35it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.21it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.27it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.46it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.57it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.77it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.74it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.71it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.83it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.79it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.71it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.52it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.61it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.70it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.78it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.58it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.61it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.64it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.72it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.62it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.31it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.38it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.49it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.64it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.71it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.72it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.69it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.72it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.84it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.82it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.62it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.70it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.75it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.74it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.71it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.85it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.78it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.75it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.74it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.65it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.70it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.61it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.70it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.70it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.79it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.70it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.81it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.84it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.75it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.76it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.81it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.68it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.61it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.75it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.75it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.67it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.79it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.71it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.71it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.83it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.81it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.88it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.70it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.65it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.80it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.77it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.85it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.87it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.80it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.84it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.84it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.87it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.76it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.84it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.82it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.81it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.81it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.73it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.75it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.75it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.85it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.94it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.79it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.88it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.85it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.88it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.91it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.71it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.72it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.76it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.74it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.72it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.73it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.84it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.97it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.00it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.90it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.93it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.88it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.90it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.82it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.89it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.85it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.73it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.84it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.83it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.80it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.84it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.84it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.83it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.92it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.88it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.87it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.90it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.95it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.89it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.96it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.03it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.05it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.05it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.00it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.83it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.22it/s]
+2022-12-21,10:20:16 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:26, 111.68it/s]
  1%|          | 512/50176 [00:04<06:37, 125.05it/s]
  2%|▏         | 768/50176 [00:05<06:16, 131.39it/s]
  2%|▏         | 1024/50176 [00:07<06:03, 135.32it/s]
  3%|▎         | 1280/50176 [00:09<05:58, 136.47it/s]
  3%|▎         | 1536/50176 [00:11<06:09, 131.48it/s]
  4%|▎         | 1792/50176 [00:13<06:07, 131.79it/s]
  4%|▍         | 2048/50176 [00:15<05:50, 137.15it/s]
  5%|▍         | 2304/50176 [00:17<05:46, 138.20it/s]
  5%|▌         | 2560/50176 [00:18<05:40, 139.74it/s]
  6%|▌         | 2816/50176 [00:20<05:40, 139.05it/s]
  6%|▌         | 3072/50176 [00:22<05:29, 142.97it/s]
  7%|▋         | 3328/50176 [00:24<05:25, 143.78it/s]
  7%|▋         | 3584/50176 [00:26<05:22, 144.38it/s]
  8%|▊         | 3840/50176 [00:28<05:33, 138.75it/s]
  8%|▊         | 4096/50176 [00:29<05:31, 138.89it/s]
  9%|▊         | 4352/50176 [00:31<05:25, 140.75it/s]
  9%|▉         | 4608/50176 [00:33<05:21, 141.58it/s]
 10%|▉         | 4864/50176 [00:35<05:16, 143.24it/s]
 10%|█         | 5120/50176 [00:36<05:12, 143.98it/s]
 11%|█         | 5376/50176 [00:38<05:09, 144.77it/s]
 11%|█         | 5632/50176 [00:40<05:05, 145.58it/s]
 12%|█▏        | 5888/50176 [00:42<05:02, 146.52it/s]
 12%|█▏        | 6144/50176 [00:44<05:27, 134.30it/s]
 13%|█▎        | 6400/50176 [00:46<05:16, 138.29it/s]
 13%|█▎        | 6656/50176 [00:47<05:09, 140.40it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.49it/s]
 14%|█▍        | 7168/50176 [00:51<05:02, 142.28it/s]
 15%|█▍        | 7424/50176 [00:53<04:55, 144.76it/s]
 15%|█▌        | 7680/50176 [00:54<04:54, 144.33it/s]
 16%|█▌        | 7936/50176 [00:56<04:45, 147.83it/s]
 16%|█▋        | 8192/50176 [00:58<04:57, 140.99it/s]
 17%|█▋        | 8448/50176 [01:00<04:49, 144.30it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.23it/s]
 18%|█▊        | 8960/50176 [01:03<04:51, 141.62it/s]
 18%|█▊        | 9216/50176 [01:05<04:41, 145.60it/s]
 19%|█▉        | 9472/50176 [01:07<04:38, 145.90it/s]
 19%|█▉        | 9728/50176 [01:09<04:42, 143.33it/s]
 20%|█▉        | 9984/50176 [01:10<04:35, 146.03it/s]
 20%|██        | 10240/50176 [01:12<04:29, 148.21it/s]
 21%|██        | 10496/50176 [01:14<04:25, 149.22it/s]
 21%|██▏       | 10752/50176 [01:16<04:32, 144.59it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 146.97it/s]
 22%|██▏       | 11264/50176 [01:19<04:22, 148.19it/s]
 23%|██▎       | 11520/50176 [01:21<04:16, 150.60it/s]
 23%|██▎       | 11776/50176 [01:22<04:16, 149.61it/s]
 24%|██▍       | 12032/50176 [01:24<04:13, 150.46it/s]
 24%|██▍       | 12288/50176 [01:26<04:10, 151.49it/s]
 25%|██▌       | 12544/50176 [01:27<04:10, 150.06it/s]
 26%|██▌       | 12800/50176 [01:29<04:08, 150.71it/s]
 26%|██▌       | 13056/50176 [01:31<04:03, 152.22it/s]
 27%|██▋       | 13312/50176 [01:33<04:06, 149.63it/s]
 27%|██▋       | 13568/50176 [01:35<04:18, 141.70it/s]
 28%|██▊       | 13824/50176 [01:36<04:21, 138.91it/s]
 28%|██▊       | 14080/50176 [01:38<04:14, 141.67it/s]
 29%|██▊       | 14336/50176 [01:40<04:19, 138.24it/s]
 29%|██▉       | 14592/50176 [01:42<04:07, 143.84it/s]
 30%|██▉       | 14848/50176 [01:44<04:20, 135.83it/s]
 30%|███       | 15104/50176 [01:46<04:19, 135.40it/s]
 31%|███       | 15360/50176 [01:48<04:11, 138.18it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.75it/s]
 32%|███▏      | 15872/50176 [01:51<04:10, 136.97it/s]
 32%|███▏      | 16128/50176 [01:53<04:04, 139.35it/s]
 33%|███▎      | 16384/50176 [01:55<04:01, 139.75it/s]
 33%|███▎      | 16640/50176 [01:57<04:08, 135.02it/s]
 34%|███▎      | 16896/50176 [01:59<03:57, 140.12it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.90it/s]
 35%|███▍      | 17408/50176 [02:02<03:48, 143.64it/s]
 35%|███▌      | 17664/50176 [02:04<03:44, 144.70it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.29it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.65it/s]
 37%|███▋      | 18432/50176 [02:09<03:35, 147.31it/s]
 37%|███▋      | 18688/50176 [02:11<03:34, 147.02it/s]
 38%|███▊      | 18944/50176 [02:13<03:39, 142.21it/s]
 38%|███▊      | 19200/50176 [02:15<03:52, 133.47it/s]
 39%|███▉      | 19456/50176 [02:17<03:53, 131.36it/s]
 39%|███▉      | 19712/50176 [02:19<03:49, 132.86it/s]
 40%|███▉      | 19968/50176 [02:21<03:44, 134.84it/s]
 40%|████      | 20224/50176 [02:22<03:33, 140.32it/s]
 41%|████      | 20480/50176 [02:24<03:25, 144.75it/s]
 41%|████▏     | 20736/50176 [02:26<03:23, 144.59it/s]
 42%|████▏     | 20992/50176 [02:27<03:18, 147.39it/s]
 42%|████▏     | 21248/50176 [02:29<03:23, 142.19it/s]
 43%|████▎     | 21504/50176 [02:31<03:16, 145.62it/s]
 43%|████▎     | 21760/50176 [02:33<03:12, 147.65it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 151.24it/s]
 44%|████▍     | 22272/50176 [02:36<03:05, 150.53it/s]
 45%|████▍     | 22528/50176 [02:38<03:16, 140.91it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.96it/s]
 46%|████▌     | 23040/50176 [02:43<03:45, 120.38it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.46it/s]
 47%|████▋     | 23552/50176 [02:48<03:58, 111.59it/s]
 47%|████▋     | 23808/50176 [02:50<03:52, 113.63it/s]
 48%|████▊     | 24064/50176 [02:52<03:49, 113.74it/s]
 48%|████▊     | 24320/50176 [02:55<03:52, 111.18it/s]
 49%|████▉     | 24576/50176 [02:57<03:44, 114.21it/s]
 49%|████▉     | 24832/50176 [02:59<03:33, 118.82it/s]
 50%|█████     | 25088/50176 [03:01<03:30, 118.90it/s]
 51%|█████     | 25344/50176 [03:03<03:26, 120.48it/s]
 51%|█████     | 25600/50176 [03:05<03:29, 117.18it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.35it/s]
 52%|█████▏    | 26112/50176 [03:10<03:27, 115.98it/s]
 53%|█████▎    | 26368/50176 [03:12<03:29, 113.37it/s]
 53%|█████▎    | 26624/50176 [03:14<03:24, 115.27it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.33it/s]
 54%|█████▍    | 27136/50176 [03:19<03:23, 113.16it/s]
 55%|█████▍    | 27392/50176 [03:21<03:18, 114.91it/s]
 55%|█████▌    | 27648/50176 [03:23<03:03, 122.56it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.53it/s]
 56%|█████▌    | 28160/50176 [03:28<03:16, 111.77it/s]
 57%|█████▋    | 28416/50176 [03:30<03:14, 112.13it/s]
 57%|█████▋    | 28672/50176 [03:32<03:14, 110.66it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.80it/s]
 58%|█████▊    | 29184/50176 [03:37<03:06, 112.85it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.41it/s]
 59%|█████▉    | 29696/50176 [03:42<03:17, 103.83it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.50it/s]
 60%|██████    | 30208/50176 [03:47<03:05, 107.58it/s]
 61%|██████    | 30464/50176 [03:49<02:53, 113.62it/s]
 61%|██████    | 30720/50176 [03:51<02:51, 113.68it/s]
 62%|██████▏   | 30976/50176 [03:53<02:46, 115.38it/s]
 62%|██████▏   | 31232/50176 [03:55<02:44, 115.48it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.88it/s]
 63%|██████▎   | 31744/50176 [03:59<02:34, 119.05it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.31it/s]
 64%|██████▍   | 32256/50176 [04:04<02:31, 118.31it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.57it/s]
 65%|██████▌   | 32768/50176 [04:08<02:30, 115.54it/s]
 66%|██████▌   | 33024/50176 [04:11<02:29, 114.62it/s]
 66%|██████▋   | 33280/50176 [04:13<02:26, 115.32it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.78it/s]
 67%|██████▋   | 33792/50176 [04:18<02:26, 111.82it/s]
 68%|██████▊   | 34048/50176 [04:20<02:18, 116.12it/s]
 68%|██████▊   | 34304/50176 [04:22<02:14, 117.65it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.68it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.97it/s]
 70%|██████▉   | 35072/50176 [04:29<02:13, 112.89it/s]
 70%|███████   | 35328/50176 [04:31<02:11, 113.17it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.90it/s]
 71%|███████▏  | 35840/50176 [04:36<02:08, 111.41it/s]
 72%|███████▏  | 36096/50176 [04:38<02:03, 113.58it/s]
 72%|███████▏  | 36352/50176 [04:40<02:03, 112.09it/s]
 73%|███████▎  | 36608/50176 [04:43<02:03, 109.90it/s]
 73%|███████▎  | 36864/50176 [04:45<02:04, 106.93it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 109.05it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 115.03it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.65it/s]
 76%|███████▌  | 37888/50176 [04:54<01:46, 115.80it/s]
 76%|███████▌  | 38144/50176 [04:56<01:42, 117.32it/s]
 77%|███████▋  | 38400/50176 [04:58<01:44, 113.22it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 113.80it/s]
 78%|███████▊  | 38912/50176 [05:03<01:39, 112.93it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.56it/s]
 79%|███████▊  | 39424/50176 [05:07<01:34, 113.23it/s]
 79%|███████▉  | 39680/50176 [05:10<01:33, 112.55it/s]
 80%|███████▉  | 39936/50176 [05:12<01:28, 115.91it/s]
 80%|████████  | 40192/50176 [05:14<01:26, 114.89it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.52it/s]
 81%|████████  | 40704/50176 [05:19<01:28, 106.79it/s]
 82%|████████▏ | 40960/50176 [05:21<01:22, 111.46it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 110.77it/s]
 83%|████████▎ | 41472/50176 [05:26<01:20, 108.13it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 109.05it/s]
 84%|████████▎ | 41984/50176 [05:31<01:14, 109.71it/s]
 84%|████████▍ | 42240/50176 [05:33<01:11, 110.90it/s]
 85%|████████▍ | 42496/50176 [05:35<01:08, 111.72it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 112.97it/s]
 86%|████████▌ | 43008/50176 [05:40<01:03, 112.66it/s]
 86%|████████▌ | 43264/50176 [05:42<01:01, 113.01it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.72it/s]
 87%|████████▋ | 43776/50176 [05:47<00:58, 110.31it/s]
 88%|████████▊ | 44032/50176 [05:49<00:56, 109.00it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.18it/s]
 89%|████████▉ | 44544/50176 [05:54<00:52, 106.58it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.14it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.36it/s]
 90%|█████████ | 45312/50176 [06:01<00:45, 106.88it/s]
 91%|█████████ | 45568/50176 [06:03<00:41, 111.18it/s]
 91%|█████████▏| 45824/50176 [06:05<00:39, 108.84it/s]
 92%|█████████▏| 46080/50176 [06:08<00:38, 105.96it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.28it/s]
 93%|█████████▎| 46592/50176 [06:13<00:33, 108.52it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 106.04it/s]
 94%|█████████▍| 47104/50176 [06:18<00:30, 101.90it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.85it/s]
 95%|█████████▍| 47616/50176 [06:23<00:24, 102.71it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 103.06it/s]
 96%|█████████▌| 48128/50176 [06:28<00:19, 104.54it/s]
 96%|█████████▋| 48384/50176 [06:30<00:16, 106.89it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.22it/s]
 97%|█████████▋| 48896/50176 [06:35<00:11, 106.82it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 104.04it/s]
 98%|█████████▊| 49408/50176 [06:40<00:07, 103.74it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.74it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.54it/s] 
100%|██████████| 50176/50176 [06:46<00:00, 126.63it/s]
100%|██████████| 50176/50176 [06:46<00:00, 123.44it/s]
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5883	imagenet-zeroshot-val-top5: 0.8555
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cef1847d3cd470d62ad2905b1884fa799d1b4fa8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddcfa9a95d108850d965a0bd2115bf21b4b75eb39c4f25eee94b85335d4760c4
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4ea3c01b72eb021da4adc7855b2442a0f35fec64
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt).
+2022-12-21,10:19:58 | INFO | Model:
+2022-12-21,10:19:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:58 | INFO | Params:
+2022-12-21,10:19:58 | INFO |   batch_size: 256
+2022-12-21,10:19:58 | INFO |   beta1: 0.9
+2022-12-21,10:19:58 | INFO |   beta2: 0.98
+2022-12-21,10:19:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/checkpoints
+2022-12-21,10:19:58 | INFO |   copy_codebase: False
+2022-12-21,10:19:58 | INFO |   csv_caption_key: title
+2022-12-21,10:19:58 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:58 | INFO |   csv_separator: 	
+2022-12-21,10:19:58 | INFO |   dataset_resampled: False
+2022-12-21,10:19:58 | INFO |   dataset_type: auto
+2022-12-21,10:19:58 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:58 | INFO |   debug: False
+2022-12-21,10:19:58 | INFO |   device: cuda:0
+2022-12-21,10:19:58 | INFO |   dist_backend: nccl
+2022-12-21,10:19:58 | INFO |   dist_url: env://
+2022-12-21,10:19:58 | INFO |   distributed: False
+2022-12-21,10:19:58 | INFO |   epochs: 32
+2022-12-21,10:19:58 | INFO |   eps: 1e-06
+2022-12-21,10:19:58 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:58 | INFO |   gather_with_grad: False
+2022-12-21,10:19:58 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:58 | INFO |   horovod: False
+2022-12-21,10:19:58 | INFO |   image_mean: None
+2022-12-21,10:19:58 | INFO |   image_std: None
+2022-12-21,10:19:58 | INFO |   imagenet_v2: None
+2022-12-21,10:19:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:58 | INFO |   local_loss: False
+2022-12-21,10:19:58 | INFO |   local_rank: 0
+2022-12-21,10:19:58 | INFO |   lock_image: False
+2022-12-21,10:19:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:58 | INFO |   log_level: 20
+2022-12-21,10:19:58 | INFO |   log_local: False
+2022-12-21,10:19:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/out.log
+2022-12-21,10:19:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:58 | INFO |   lr: 0.0005
+2022-12-21,10:19:58 | INFO |   model: ViT-B-16
+2022-12-21,10:19:58 | INFO |   name: epoch_32.pt
+2022-12-21,10:19:58 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:58 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:58 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt
+2022-12-21,10:19:58 | INFO |   pretrained_image: False
+2022-12-21,10:19:58 | INFO |   rank: 0
+2022-12-21,10:19:58 | INFO |   report_to: 
+2022-12-21,10:19:58 | INFO |   resume: None
+2022-12-21,10:19:58 | INFO |   save_frequency: 1
+2022-12-21,10:19:58 | INFO |   save_most_recent: False
+2022-12-21,10:19:58 | INFO |   seed: 0
+2022-12-21,10:19:58 | INFO |   skip_scheduler: False
+2022-12-21,10:19:58 | INFO |   tensorboard: False
+2022-12-21,10:19:58 | INFO |   tensorboard_path: 
+2022-12-21,10:19:58 | INFO |   torchscript: False
+2022-12-21,10:19:58 | INFO |   trace: False
+2022-12-21,10:19:58 | INFO |   train_data: None
+2022-12-21,10:19:58 | INFO |   train_num_samples: None
+2022-12-21,10:19:58 | INFO |   use_bn_sync: False
+2022-12-21,10:19:58 | INFO |   val_data: None
+2022-12-21,10:19:58 | INFO |   val_frequency: 1
+2022-12-21,10:19:58 | INFO |   val_num_samples: None
+2022-12-21,10:19:58 | INFO |   wandb: False
+2022-12-21,10:19:58 | INFO |   wandb_notes: 
+2022-12-21,10:19:58 | INFO |   warmup: 10000
+2022-12-21,10:19:58 | INFO |   wd: 0.2
+2022-12-21,10:19:58 | INFO |   workers: 1
+2022-12-21,10:19:58 | INFO |   world_size: 1
+2022-12-21,10:19:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:58 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:20,  1.25it/s]
  1%|          | 6/1000 [00:00<01:55,  8.61it/s]
  1%|          | 12/1000 [00:01<00:57, 17.15it/s]
  2%|▏         | 18/1000 [00:01<00:39, 24.83it/s]
  2%|▏         | 24/1000 [00:01<00:30, 31.64it/s]
  3%|▎         | 30/1000 [00:01<00:25, 37.33it/s]
  4%|▎         | 36/1000 [00:01<00:23, 41.91it/s]
  4%|▍         | 42/1000 [00:01<00:20, 45.88it/s]
  5%|▍         | 48/1000 [00:01<00:19, 48.94it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.21it/s]
  6%|▌         | 60/1000 [00:01<00:17, 52.88it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.13it/s]
  7%|▋         | 72/1000 [00:02<00:16, 54.98it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.58it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.19it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.55it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.97it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.13it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.43it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.56it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.54it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.44it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.43it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.40it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.29it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.43it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.50it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.39it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.25it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.27it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.24it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.16it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.15it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.08it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.86it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.76it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.83it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 56.85it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 56.90it/s]
 24%|██▍       | 240/1000 [00:05<00:15, 48.16it/s]
 25%|██▍       | 246/1000 [00:05<00:14, 50.52it/s]
 25%|██▌       | 252/1000 [00:05<00:14, 52.32it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 53.78it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 54.80it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 55.31it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 55.87it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 56.42it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 56.79it/s]
 29%|██▉       | 294/1000 [00:06<00:12, 57.14it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.24it/s]
 31%|███       | 306/1000 [00:06<00:12, 56.97it/s]
 31%|███       | 312/1000 [00:06<00:12, 57.28it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.39it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.39it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.24it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.37it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.50it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.61it/s]
 35%|███▌      | 354/1000 [00:07<00:11, 57.43it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.44it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.55it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.65it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.54it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.21it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.25it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.36it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.52it/s]
 41%|████      | 408/1000 [00:08<00:10, 57.59it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.54it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.57it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.59it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.70it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.62it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.40it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.48it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.54it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.53it/s]
 47%|████▋     | 468/1000 [00:09<00:09, 57.49it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.66it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.58it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.56it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.54it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.43it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.48it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.40it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.46it/s]
 52%|█████▏    | 522/1000 [00:10<00:08, 57.50it/s]
 53%|█████▎    | 528/1000 [00:10<00:08, 57.58it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.52it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.64it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.65it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.56it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.57it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.63it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.51it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.45it/s]
 58%|█████▊    | 582/1000 [00:11<00:07, 57.59it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.57it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.49it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.60it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.50it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.50it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.60it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.60it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.69it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.50it/s]
 64%|██████▍   | 642/1000 [00:12<00:06, 57.47it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.62it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.59it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.68it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.70it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.64it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.69it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.70it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.73it/s]
 70%|██████▉   | 696/1000 [00:13<00:05, 57.61it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.70it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.66it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.64it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.65it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.59it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.60it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.62it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.61it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.68it/s]
 76%|███████▌  | 756/1000 [00:14<00:04, 57.76it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.60it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.72it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.67it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.71it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.74it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.52it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.51it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.54it/s]
 81%|████████  | 810/1000 [00:15<00:03, 57.51it/s]
 82%|████████▏ | 816/1000 [00:15<00:03, 57.50it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.53it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.62it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.76it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.79it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.69it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.73it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.67it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.68it/s]
 87%|████████▋ | 870/1000 [00:16<00:02, 57.64it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.70it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.34it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.32it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.50it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.52it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.54it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.60it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.62it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.61it/s]
 93%|█████████▎| 930/1000 [00:17<00:01, 57.72it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.77it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.76it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.77it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.80it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.73it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.77it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.85it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.87it/s]
 98%|█████████▊| 984/1000 [00:18<00:00, 57.85it/s]
 99%|█████████▉| 990/1000 [00:18<00:00, 57.76it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.47it/s]
100%|██████████| 1000/1000 [00:18<00:00, 54.64it/s]
+2022-12-21,10:20:16 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:01, 118.48it/s]
  1%|          | 512/50176 [00:04<06:33, 126.27it/s]
  2%|▏         | 768/50176 [00:05<06:08, 133.90it/s]
  2%|▏         | 1024/50176 [00:07<06:01, 136.09it/s]
  3%|▎         | 1280/50176 [00:09<05:56, 137.34it/s]
  3%|▎         | 1536/50176 [00:11<06:07, 132.48it/s]
  4%|▎         | 1792/50176 [00:13<06:01, 133.98it/s]
  4%|▍         | 2048/50176 [00:15<05:51, 136.92it/s]
  5%|▍         | 2304/50176 [00:17<05:47, 137.95it/s]
  5%|▌         | 2560/50176 [00:18<05:40, 139.75it/s]
  6%|▌         | 2816/50176 [00:20<05:39, 139.60it/s]
  6%|▌         | 3072/50176 [00:22<05:30, 142.69it/s]
  7%|▋         | 3328/50176 [00:24<05:26, 143.63it/s]
  7%|▋         | 3584/50176 [00:25<05:23, 144.03it/s]
  8%|▊         | 3840/50176 [00:27<05:33, 138.79it/s]
  8%|▊         | 4096/50176 [00:29<05:30, 139.24it/s]
  9%|▊         | 4352/50176 [00:31<05:25, 140.91it/s]
  9%|▉         | 4608/50176 [00:33<05:22, 141.12it/s]
 10%|▉         | 4864/50176 [00:35<05:21, 141.11it/s]
 10%|█         | 5120/50176 [00:36<05:10, 144.88it/s]
 11%|█         | 5376/50176 [00:38<05:07, 145.49it/s]
 11%|█         | 5632/50176 [00:40<05:04, 146.22it/s]
 12%|█▏        | 5888/50176 [00:41<05:01, 146.80it/s]
 12%|█▏        | 6144/50176 [00:44<05:27, 134.27it/s]
 13%|█▎        | 6400/50176 [00:45<05:16, 138.35it/s]
 13%|█▎        | 6656/50176 [00:47<05:08, 140.84it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.54it/s]
 14%|█▍        | 7168/50176 [00:51<05:02, 142.39it/s]
 15%|█▍        | 7424/50176 [00:53<04:56, 144.28it/s]
 15%|█▌        | 7680/50176 [00:54<04:53, 144.56it/s]
 16%|█▌        | 7936/50176 [00:56<04:45, 147.85it/s]
 16%|█▋        | 8192/50176 [00:58<04:57, 140.90it/s]
 17%|█▋        | 8448/50176 [01:00<04:49, 144.31it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.39it/s]
 18%|█▊        | 8960/50176 [01:03<04:51, 141.60it/s]
 18%|█▊        | 9216/50176 [01:05<04:41, 145.73it/s]
 19%|█▉        | 9472/50176 [01:07<04:39, 145.68it/s]
 19%|█▉        | 9728/50176 [01:09<04:41, 143.48it/s]
 20%|█▉        | 9984/50176 [01:10<04:35, 145.95it/s]
 20%|██        | 10240/50176 [01:12<04:29, 148.30it/s]
 21%|██        | 10496/50176 [01:14<04:26, 149.11it/s]
 21%|██▏       | 10752/50176 [01:15<04:33, 144.14it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 147.05it/s]
 22%|██▏       | 11264/50176 [01:19<04:22, 148.44it/s]
 23%|██▎       | 11520/50176 [01:20<04:17, 149.93it/s]
 23%|██▎       | 11776/50176 [01:22<04:15, 150.29it/s]
 24%|██▍       | 12032/50176 [01:24<04:13, 150.42it/s]
 24%|██▍       | 12288/50176 [01:26<04:09, 151.63it/s]
 25%|██▌       | 12544/50176 [01:27<04:11, 149.90it/s]
 26%|██▌       | 12800/50176 [01:29<04:09, 149.85it/s]
 26%|██▌       | 13056/50176 [01:31<04:03, 152.54it/s]
 27%|██▋       | 13312/50176 [01:32<04:05, 150.45it/s]
 27%|██▋       | 13568/50176 [01:34<04:18, 141.66it/s]
 28%|██▊       | 13824/50176 [01:36<04:20, 139.56it/s]
 28%|██▊       | 14080/50176 [01:38<04:15, 141.47it/s]
 29%|██▊       | 14336/50176 [01:40<04:18, 138.76it/s]
 29%|██▉       | 14592/50176 [01:42<04:07, 143.53it/s]
 30%|██▉       | 14848/50176 [01:44<04:19, 136.08it/s]
 30%|███       | 15104/50176 [01:46<04:19, 135.12it/s]
 31%|███       | 15360/50176 [01:47<04:12, 137.97it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.56it/s]
 32%|███▏      | 15872/50176 [01:51<04:11, 136.62it/s]
 32%|███▏      | 16128/50176 [01:53<04:04, 139.53it/s]
 33%|███▎      | 16384/50176 [01:55<04:01, 139.81it/s]
 33%|███▎      | 16640/50176 [01:57<04:08, 134.93it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 140.19it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.73it/s]
 35%|███▍      | 17408/50176 [02:02<03:48, 143.24it/s]
 35%|███▌      | 17664/50176 [02:04<03:44, 144.71it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.42it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.64it/s]
 37%|███▋      | 18432/50176 [02:09<03:35, 147.27it/s]
 37%|███▋      | 18688/50176 [02:11<03:34, 146.94it/s]
 38%|███▊      | 18944/50176 [02:12<03:39, 142.28it/s]
 38%|███▊      | 19200/50176 [02:15<03:51, 133.89it/s]
 39%|███▉      | 19456/50176 [02:17<03:52, 131.94it/s]
 39%|███▉      | 19712/50176 [02:19<03:50, 131.98it/s]
 40%|███▉      | 19968/50176 [02:20<03:43, 135.43it/s]
 40%|████      | 20224/50176 [02:22<03:32, 140.99it/s]
 41%|████      | 20480/50176 [02:24<03:25, 144.30it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 144.04it/s]
 42%|████▏     | 20992/50176 [02:27<03:17, 147.47it/s]
 42%|████▏     | 21248/50176 [02:29<03:24, 141.57it/s]
 43%|████▎     | 21504/50176 [02:31<03:15, 146.98it/s]
 43%|████▎     | 21760/50176 [02:32<03:12, 147.34it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 151.24it/s]
 44%|████▍     | 22272/50176 [02:36<03:05, 150.27it/s]
 45%|████▍     | 22528/50176 [02:38<03:16, 141.05it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.88it/s]
 46%|████▌     | 23040/50176 [02:43<03:45, 120.52it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.41it/s]
 47%|████▋     | 23552/50176 [02:48<03:59, 111.07it/s]
 47%|████▋     | 23808/50176 [02:50<03:52, 113.31it/s]
 48%|████▊     | 24064/50176 [02:52<03:48, 114.03it/s]
 48%|████▊     | 24320/50176 [02:54<03:52, 111.35it/s]
 49%|████▉     | 24576/50176 [02:57<03:44, 114.10it/s]
 49%|████▉     | 24832/50176 [02:58<03:33, 118.98it/s]
 50%|█████     | 25088/50176 [03:01<03:31, 118.75it/s]
 51%|█████     | 25344/50176 [03:03<03:26, 120.30it/s]
 51%|█████     | 25600/50176 [03:05<03:30, 117.00it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.23it/s]
 52%|█████▏    | 26112/50176 [03:09<03:27, 116.16it/s]
 53%|█████▎    | 26368/50176 [03:12<03:29, 113.49it/s]
 53%|█████▎    | 26624/50176 [03:14<03:24, 115.38it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.25it/s]
 54%|█████▍    | 27136/50176 [03:19<03:23, 113.33it/s]
 55%|█████▍    | 27392/50176 [03:21<03:18, 114.80it/s]
 55%|█████▌    | 27648/50176 [03:22<03:03, 122.57it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.55it/s]
 56%|█████▌    | 28160/50176 [03:27<03:17, 111.75it/s]
 57%|█████▋    | 28416/50176 [03:30<03:14, 112.02it/s]
 57%|█████▋    | 28672/50176 [03:32<03:13, 110.85it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.74it/s]
 58%|█████▊    | 29184/50176 [03:37<03:05, 112.90it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.34it/s]
 59%|█████▉    | 29696/50176 [03:42<03:16, 104.00it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.34it/s]
 60%|██████    | 30208/50176 [03:46<03:05, 107.85it/s]
 61%|██████    | 30464/50176 [03:48<02:53, 113.36it/s]
 61%|██████    | 30720/50176 [03:51<02:50, 113.82it/s]
 62%|██████▏   | 30976/50176 [03:53<02:46, 115.14it/s]
 62%|██████▏   | 31232/50176 [03:55<02:44, 115.25it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.96it/s]
 63%|██████▎   | 31744/50176 [03:59<02:34, 119.29it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.30it/s]
 64%|██████▍   | 32256/50176 [04:04<02:31, 118.42it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.53it/s]
 65%|██████▌   | 32768/50176 [04:08<02:31, 115.18it/s]
 66%|██████▌   | 33024/50176 [04:10<02:29, 114.67it/s]
 66%|██████▋   | 33280/50176 [04:13<02:26, 115.44it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.66it/s]
 67%|██████▋   | 33792/50176 [04:17<02:26, 111.96it/s]
 68%|██████▊   | 34048/50176 [04:19<02:18, 116.14it/s]
 68%|██████▊   | 34304/50176 [04:22<02:14, 117.87it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.49it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.97it/s]
 70%|██████▉   | 35072/50176 [04:29<02:13, 112.85it/s]
 70%|███████   | 35328/50176 [04:31<02:11, 113.25it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.86it/s]
 71%|███████▏  | 35840/50176 [04:35<02:08, 111.51it/s]
 72%|███████▏  | 36096/50176 [04:38<02:04, 113.42it/s]
 72%|███████▏  | 36352/50176 [04:40<02:03, 112.34it/s]
 73%|███████▎  | 36608/50176 [04:42<02:03, 109.98it/s]
 73%|███████▎  | 36864/50176 [04:45<02:04, 106.99it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 108.98it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 114.69it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.46it/s]
 76%|███████▌  | 37888/50176 [04:53<01:45, 116.19it/s]
 76%|███████▌  | 38144/50176 [04:56<01:42, 117.26it/s]
 77%|███████▋  | 38400/50176 [04:58<01:44, 113.20it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 113.87it/s]
 78%|███████▊  | 38912/50176 [05:03<01:39, 112.94it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.56it/s]
 79%|███████▊  | 39424/50176 [05:07<01:34, 113.38it/s]
 79%|███████▉  | 39680/50176 [05:10<01:33, 112.38it/s]
 80%|███████▉  | 39936/50176 [05:12<01:28, 116.18it/s]
 80%|████████  | 40192/50176 [05:14<01:26, 115.18it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.50it/s]
 81%|████████  | 40704/50176 [05:19<01:28, 106.75it/s]
 82%|████████▏ | 40960/50176 [05:21<01:22, 111.19it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 110.96it/s]
 83%|████████▎ | 41472/50176 [05:26<01:20, 108.36it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 108.85it/s]
 84%|████████▎ | 41984/50176 [05:30<01:14, 109.93it/s]
 84%|████████▍ | 42240/50176 [05:33<01:11, 110.51it/s]
 85%|████████▍ | 42496/50176 [05:35<01:08, 111.79it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 112.90it/s]
 86%|████████▌ | 43008/50176 [05:39<01:03, 112.77it/s]
 86%|████████▌ | 43264/50176 [05:42<01:01, 112.97it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.59it/s]
 87%|████████▋ | 43776/50176 [05:46<00:58, 110.23it/s]
 88%|████████▊ | 44032/50176 [05:49<00:56, 108.98it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.04it/s]
 89%|████████▉ | 44544/50176 [05:54<00:52, 106.62it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.16it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.60it/s]
 90%|█████████ | 45312/50176 [06:01<00:45, 106.53it/s]
 91%|█████████ | 45568/50176 [06:03<00:41, 111.44it/s]
 91%|█████████▏| 45824/50176 [06:05<00:40, 108.78it/s]
 92%|█████████▏| 46080/50176 [06:08<00:38, 106.08it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.14it/s]
 93%|█████████▎| 46592/50176 [06:13<00:33, 108.57it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 105.91it/s]
 94%|█████████▍| 47104/50176 [06:18<00:30, 102.14it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.75it/s]
 95%|█████████▍| 47616/50176 [06:23<00:24, 102.77it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 102.98it/s]
 96%|█████████▌| 48128/50176 [06:28<00:19, 104.45it/s]
 96%|█████████▋| 48384/50176 [06:30<00:16, 106.75it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.41it/s]
 97%|█████████▋| 48896/50176 [06:35<00:12, 106.62it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 104.03it/s]
 98%|█████████▊| 49408/50176 [06:40<00:07, 103.79it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.87it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.55it/s] 
100%|██████████| 50176/50176 [06:46<00:00, 126.68it/s]
100%|██████████| 50176/50176 [06:46<00:00, 123.48it/s]
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5907	imagenet-zeroshot-val-top5: 0.8584
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt
new file mode 100644
index 0000000000000000000000000000000000000000..50e00bcd316d520f8838488ad0d85e740d71ede1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db02311c98c4328442b9114de2fa1995f70b1fcd58c6cd786fa7371b714a38fd
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a137909de8f9246bd1c012987c29913cc14bb1a8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt).
+2022-12-21,10:19:59 | INFO | Model:
+2022-12-21,10:19:59 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:59 | INFO | Params:
+2022-12-21,10:19:59 | INFO |   batch_size: 256
+2022-12-21,10:19:59 | INFO |   beta1: 0.9
+2022-12-21,10:19:59 | INFO |   beta2: 0.98
+2022-12-21,10:19:59 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/checkpoints
+2022-12-21,10:19:59 | INFO |   copy_codebase: False
+2022-12-21,10:19:59 | INFO |   csv_caption_key: title
+2022-12-21,10:19:59 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:59 | INFO |   csv_separator: 	
+2022-12-21,10:19:59 | INFO |   dataset_resampled: False
+2022-12-21,10:19:59 | INFO |   dataset_type: auto
+2022-12-21,10:19:59 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:59 | INFO |   debug: False
+2022-12-21,10:19:59 | INFO |   device: cuda:0
+2022-12-21,10:19:59 | INFO |   dist_backend: nccl
+2022-12-21,10:19:59 | INFO |   dist_url: env://
+2022-12-21,10:19:59 | INFO |   distributed: False
+2022-12-21,10:19:59 | INFO |   epochs: 32
+2022-12-21,10:19:59 | INFO |   eps: 1e-06
+2022-12-21,10:19:59 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:59 | INFO |   gather_with_grad: False
+2022-12-21,10:19:59 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:59 | INFO |   horovod: False
+2022-12-21,10:19:59 | INFO |   image_mean: None
+2022-12-21,10:19:59 | INFO |   image_std: None
+2022-12-21,10:19:59 | INFO |   imagenet_v2: None
+2022-12-21,10:19:59 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:59 | INFO |   local_loss: False
+2022-12-21,10:19:59 | INFO |   local_rank: 0
+2022-12-21,10:19:59 | INFO |   lock_image: False
+2022-12-21,10:19:59 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:59 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:59 | INFO |   log_level: 20
+2022-12-21,10:19:59 | INFO |   log_local: False
+2022-12-21,10:19:59 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/out.log
+2022-12-21,10:19:59 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:59 | INFO |   lr: 0.0005
+2022-12-21,10:19:59 | INFO |   model: ViT-B-16
+2022-12-21,10:19:59 | INFO |   name: epoch_33.pt
+2022-12-21,10:19:59 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:59 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:59 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:59 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt
+2022-12-21,10:19:59 | INFO |   pretrained_image: False
+2022-12-21,10:19:59 | INFO |   rank: 0
+2022-12-21,10:19:59 | INFO |   report_to: 
+2022-12-21,10:19:59 | INFO |   resume: None
+2022-12-21,10:19:59 | INFO |   save_frequency: 1
+2022-12-21,10:19:59 | INFO |   save_most_recent: False
+2022-12-21,10:19:59 | INFO |   seed: 0
+2022-12-21,10:19:59 | INFO |   skip_scheduler: False
+2022-12-21,10:19:59 | INFO |   tensorboard: False
+2022-12-21,10:19:59 | INFO |   tensorboard_path: 
+2022-12-21,10:19:59 | INFO |   torchscript: False
+2022-12-21,10:19:59 | INFO |   trace: False
+2022-12-21,10:19:59 | INFO |   train_data: None
+2022-12-21,10:19:59 | INFO |   train_num_samples: None
+2022-12-21,10:19:59 | INFO |   use_bn_sync: False
+2022-12-21,10:19:59 | INFO |   val_data: None
+2022-12-21,10:19:59 | INFO |   val_frequency: 1
+2022-12-21,10:19:59 | INFO |   val_num_samples: None
+2022-12-21,10:19:59 | INFO |   wandb: False
+2022-12-21,10:19:59 | INFO |   wandb_notes: 
+2022-12-21,10:19:59 | INFO |   warmup: 10000
+2022-12-21,10:19:59 | INFO |   wd: 0.2
+2022-12-21,10:19:59 | INFO |   workers: 1
+2022-12-21,10:19:59 | INFO |   world_size: 1
+2022-12-21,10:19:59 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:59 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:59 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:33,  1.58it/s]
  1%|          | 6/1000 [00:00<01:35, 10.43it/s]
  1%|          | 12/1000 [00:00<00:48, 20.25it/s]
  2%|▏         | 18/1000 [00:00<00:33, 28.95it/s]
  2%|▏         | 24/1000 [00:01<00:27, 36.03it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.65it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.90it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.04it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.42it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.92it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.21it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.02it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.74it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.20it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.71it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.88it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.31it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.50it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.90it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.79it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.83it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.76it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.82it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.81it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.85it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.76it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.64it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.64it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.61it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.55it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.49it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.45it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.26it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.07it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.18it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.18it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.39it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.40it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.30it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.37it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.48it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.59it/s]
 26%|██▋       | 264/1000 [00:05<00:14, 49.84it/s]
 27%|██▋       | 270/1000 [00:05<00:14, 51.81it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 53.42it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 54.74it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 55.67it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.48it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.88it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.18it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.50it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.64it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.67it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.57it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.70it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.84it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.93it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.75it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.76it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.86it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.94it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.62it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.34it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.45it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.60it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.77it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.87it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.88it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.90it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.92it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.04it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.04it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.83it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.94it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.94it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.90it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.01it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.92it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.89it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.89it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.82it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.86it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.79it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.86it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.88it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.98it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.97it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.88it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.91it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.98it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.87it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.82it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.93it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.84it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.96it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.89it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.85it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.98it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.97it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.04it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.85it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.80it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.94it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.90it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.96it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.00it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.99it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.99it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.01it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.90it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.01it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.00it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.97it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.98it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.91it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.91it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.94it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.93it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.02it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.05it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.91it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.03it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.99it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.04it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.08it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.88it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.78it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.84it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.83it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.84it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.86it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.95it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.11it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.15it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.06it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.08it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.01it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.02it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.99it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.04it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.04it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.91it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.00it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.01it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.99it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.96it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.10it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.08it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.10it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.13it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.08it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.17it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.01it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.86it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.87it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.95it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.86it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.59it/s]
+2022-12-21,10:20:17 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:20, 131.33it/s]
  1%|          | 512/50176 [00:03<05:47, 143.05it/s]
  2%|▏         | 768/50176 [00:05<05:45, 143.02it/s]
  2%|▏         | 1024/50176 [00:07<05:46, 141.77it/s]
  3%|▎         | 1280/50176 [00:09<05:47, 140.59it/s]
  3%|▎         | 1536/50176 [00:11<06:03, 133.89it/s]
  4%|▎         | 1792/50176 [00:13<05:57, 135.53it/s]
  4%|▍         | 2048/50176 [00:14<05:48, 138.01it/s]
  5%|▍         | 2304/50176 [00:16<05:46, 138.14it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.11it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 140.04it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 142.94it/s]
  7%|▋         | 3328/50176 [00:23<05:25, 143.80it/s]
  7%|▋         | 3584/50176 [00:25<05:23, 144.19it/s]
  8%|▊         | 3840/50176 [00:27<05:33, 138.95it/s]
  8%|▊         | 4096/50176 [00:29<05:31, 138.91it/s]
  9%|▊         | 4352/50176 [00:31<05:25, 140.96it/s]
  9%|▉         | 4608/50176 [00:32<05:24, 140.63it/s]
 10%|▉         | 4864/50176 [00:34<05:16, 143.08it/s]
 10%|█         | 5120/50176 [00:36<05:11, 144.65it/s]
 11%|█         | 5376/50176 [00:38<05:08, 145.20it/s]
 11%|█         | 5632/50176 [00:39<05:05, 145.93it/s]
 12%|█▏        | 5888/50176 [00:41<05:01, 146.72it/s]
 12%|█▏        | 6144/50176 [00:43<05:27, 134.25it/s]
 13%|█▎        | 6400/50176 [00:45<05:16, 138.33it/s]
 13%|█▎        | 6656/50176 [00:47<05:09, 140.70it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.57it/s]
 14%|█▍        | 7168/50176 [00:50<05:02, 142.19it/s]
 15%|█▍        | 7424/50176 [00:52<04:55, 144.72it/s]
 15%|█▌        | 7680/50176 [00:54<04:54, 144.36it/s]
 16%|█▌        | 7936/50176 [00:55<04:45, 147.90it/s]
 16%|█▋        | 8192/50176 [00:58<04:58, 140.77it/s]
 17%|█▋        | 8448/50176 [00:59<04:48, 144.40it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.23it/s]
 18%|█▊        | 8960/50176 [01:03<04:50, 141.71it/s]
 18%|█▊        | 9216/50176 [01:04<04:41, 145.61it/s]
 19%|█▉        | 9472/50176 [01:06<04:40, 145.36it/s]
 19%|█▉        | 9728/50176 [01:08<04:41, 143.52it/s]
 20%|█▉        | 9984/50176 [01:10<04:34, 146.16it/s]
 20%|██        | 10240/50176 [01:11<04:29, 148.24it/s]
 21%|██        | 10496/50176 [01:13<04:25, 149.23it/s]
 21%|██▏       | 10752/50176 [01:15<04:33, 144.14it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 147.17it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.32it/s]
 23%|██▎       | 11520/50176 [01:20<04:16, 150.76it/s]
 23%|██▎       | 11776/50176 [01:22<04:16, 149.99it/s]
 24%|██▍       | 12032/50176 [01:23<04:13, 150.41it/s]
 24%|██▍       | 12288/50176 [01:25<04:11, 150.73it/s]
 25%|██▌       | 12544/50176 [01:27<04:10, 150.30it/s]
 26%|██▌       | 12800/50176 [01:29<04:07, 150.74it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.35it/s]
 27%|██▋       | 13312/50176 [01:32<04:06, 149.84it/s]
 27%|██▋       | 13568/50176 [01:34<04:18, 141.78it/s]
 28%|██▊       | 13824/50176 [01:36<04:20, 139.40it/s]
 28%|██▊       | 14080/50176 [01:38<04:14, 141.61it/s]
 29%|██▊       | 14336/50176 [01:40<04:18, 138.66it/s]
 29%|██▉       | 14592/50176 [01:41<04:08, 143.48it/s]
 30%|██▉       | 14848/50176 [01:43<04:20, 135.69it/s]
 30%|███       | 15104/50176 [01:45<04:19, 135.23it/s]
 31%|███       | 15360/50176 [01:47<04:12, 137.97it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.76it/s]
 32%|███▏      | 15872/50176 [01:51<04:11, 136.25it/s]
 32%|███▏      | 16128/50176 [01:52<04:03, 139.65it/s]
 33%|███▎      | 16384/50176 [01:54<04:02, 139.27it/s]
 33%|███▎      | 16640/50176 [01:56<04:07, 135.40it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 140.23it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.81it/s]
 35%|███▍      | 17408/50176 [02:01<03:48, 143.70it/s]
 35%|███▌      | 17664/50176 [02:03<03:44, 144.78it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.41it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.75it/s]
 37%|███▋      | 18432/50176 [02:08<03:35, 147.46it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.03it/s]
 38%|███▊      | 18944/50176 [02:12<03:40, 141.66it/s]
 38%|███▊      | 19200/50176 [02:14<03:51, 133.69it/s]
 39%|███▉      | 19456/50176 [02:16<03:52, 131.95it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.20it/s]
 40%|███▉      | 19968/50176 [02:20<03:43, 135.24it/s]
 40%|████      | 20224/50176 [02:22<03:32, 141.07it/s]
 41%|████      | 20480/50176 [02:23<03:25, 144.22it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 144.31it/s]
 42%|████▏     | 20992/50176 [02:27<03:18, 147.38it/s]
 42%|████▏     | 21248/50176 [02:29<03:24, 141.62it/s]
 43%|████▎     | 21504/50176 [02:30<03:15, 146.84it/s]
 43%|████▎     | 21760/50176 [02:32<03:12, 147.39it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 151.09it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.43it/s]
 45%|████▍     | 22528/50176 [02:37<03:16, 140.95it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.74it/s]
 46%|████▌     | 23040/50176 [02:42<03:45, 120.38it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.50it/s]
 47%|████▋     | 23552/50176 [02:47<03:59, 111.23it/s]
 47%|████▋     | 23808/50176 [02:49<03:52, 113.53it/s]
 48%|████▊     | 24064/50176 [02:52<03:49, 113.89it/s]
 48%|████▊     | 24320/50176 [02:54<03:52, 111.36it/s]
 49%|████▉     | 24576/50176 [02:56<03:44, 114.25it/s]
 49%|████▉     | 24832/50176 [02:58<03:32, 119.23it/s]
 50%|█████     | 25088/50176 [03:00<03:31, 118.81it/s]
 51%|█████     | 25344/50176 [03:02<03:26, 120.43it/s]
 51%|█████     | 25600/50176 [03:05<03:30, 116.78it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.48it/s]
 52%|█████▏    | 26112/50176 [03:09<03:27, 115.94it/s]
 53%|█████▎    | 26368/50176 [03:11<03:29, 113.41it/s]
 53%|█████▎    | 26624/50176 [03:13<03:24, 115.23it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.32it/s]
 54%|█████▍    | 27136/50176 [03:18<03:23, 113.12it/s]
 55%|█████▍    | 27392/50176 [03:20<03:18, 114.81it/s]
 55%|█████▌    | 27648/50176 [03:22<03:03, 122.73it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.48it/s]
 56%|█████▌    | 28160/50176 [03:27<03:16, 111.86it/s]
 57%|█████▋    | 28416/50176 [03:29<03:14, 112.14it/s]
 57%|█████▋    | 28672/50176 [03:32<03:14, 110.70it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.81it/s]
 58%|█████▊    | 29184/50176 [03:36<03:06, 112.84it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.36it/s]
 59%|█████▉    | 29696/50176 [03:41<03:17, 103.86it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.52it/s]
 60%|██████    | 30208/50176 [03:46<03:05, 107.73it/s]
 61%|██████    | 30464/50176 [03:48<02:53, 113.47it/s]
 61%|██████    | 30720/50176 [03:50<02:51, 113.73it/s]
 62%|██████▏   | 30976/50176 [03:52<02:46, 115.22it/s]
 62%|██████▏   | 31232/50176 [03:55<02:44, 115.35it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.70it/s]
 63%|██████▎   | 31744/50176 [03:59<02:35, 118.88it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.33it/s]
 64%|██████▍   | 32256/50176 [04:03<02:31, 118.33it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.69it/s]
 65%|██████▌   | 32768/50176 [04:08<02:30, 115.43it/s]
 66%|██████▌   | 33024/50176 [04:10<02:29, 114.79it/s]
 66%|██████▋   | 33280/50176 [04:12<02:26, 115.38it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.74it/s]
 67%|██████▋   | 33792/50176 [04:17<02:26, 111.80it/s]
 68%|██████▊   | 34048/50176 [04:19<02:18, 116.05it/s]
 68%|██████▊   | 34304/50176 [04:21<02:14, 117.84it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.81it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.94it/s]
 70%|██████▉   | 35072/50176 [04:28<02:13, 112.90it/s]
 70%|███████   | 35328/50176 [04:30<02:11, 113.21it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.88it/s]
 71%|███████▏  | 35840/50176 [04:35<02:08, 111.38it/s]
 72%|███████▏  | 36096/50176 [04:37<02:04, 113.39it/s]
 72%|███████▏  | 36352/50176 [04:40<02:03, 111.93it/s]
 73%|███████▎  | 36608/50176 [04:42<02:03, 109.74it/s]
 73%|███████▎  | 36864/50176 [04:45<02:04, 106.88it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 109.10it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 114.65it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.67it/s]
 76%|███████▌  | 37888/50176 [04:53<01:46, 115.88it/s]
 76%|███████▌  | 38144/50176 [04:55<01:42, 117.52it/s]
 77%|███████▋  | 38400/50176 [04:58<01:43, 113.43it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 114.01it/s]
 78%|███████▊  | 38912/50176 [05:02<01:39, 112.64it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.71it/s]
 79%|███████▊  | 39424/50176 [05:07<01:35, 113.17it/s]
 79%|███████▉  | 39680/50176 [05:09<01:33, 112.36it/s]
 80%|███████▉  | 39936/50176 [05:11<01:28, 116.10it/s]
 80%|████████  | 40192/50176 [05:13<01:26, 115.47it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.42it/s]
 81%|████████  | 40704/50176 [05:18<01:28, 106.78it/s]
 82%|████████▏ | 40960/50176 [05:21<01:22, 111.33it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 111.09it/s]
 83%|████████▎ | 41472/50176 [05:25<01:20, 108.18it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 109.02it/s]
 84%|████████▎ | 41984/50176 [05:30<01:14, 109.76it/s]
 84%|████████▍ | 42240/50176 [05:32<01:11, 110.69it/s]
 85%|████████▍ | 42496/50176 [05:34<01:08, 111.71it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 113.00it/s]
 86%|████████▌ | 43008/50176 [05:39<01:03, 112.73it/s]
 86%|████████▌ | 43264/50176 [05:41<01:01, 113.04it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.53it/s]
 87%|████████▋ | 43776/50176 [05:46<00:58, 110.31it/s]
 88%|████████▊ | 44032/50176 [05:48<00:56, 108.85it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.17it/s]
 89%|████████▉ | 44544/50176 [05:53<00:52, 106.54it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.17it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.48it/s]
 90%|█████████ | 45312/50176 [06:00<00:45, 106.74it/s]
 91%|█████████ | 45568/50176 [06:02<00:41, 111.37it/s]
 91%|█████████▏| 45824/50176 [06:05<00:39, 108.84it/s]
 92%|█████████▏| 46080/50176 [06:07<00:38, 106.09it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.19it/s]
 93%|█████████▎| 46592/50176 [06:12<00:33, 108.60it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 105.93it/s]
 94%|█████████▍| 47104/50176 [06:17<00:30, 102.03it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.79it/s]
 95%|█████████▍| 47616/50176 [06:22<00:24, 102.69it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 103.10it/s]
 96%|█████████▌| 48128/50176 [06:27<00:19, 104.39it/s]
 96%|█████████▋| 48384/50176 [06:29<00:16, 106.83it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.40it/s]
 97%|█████████▋| 48896/50176 [06:34<00:12, 106.64it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 103.95it/s]
 98%|█████████▊| 49408/50176 [06:39<00:07, 103.88it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.77it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.62it/s] 
100%|██████████| 50176/50176 [06:45<00:00, 126.58it/s]
100%|██████████| 50176/50176 [06:45<00:00, 123.61it/s]
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5897	imagenet-zeroshot-val-top5: 0.8586
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt
new file mode 100644
index 0000000000000000000000000000000000000000..190380b3750152adf002bbbf15cdc632de173a4c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7822fa8f631d6449f00680d2aaff93501fcd7ace33bd14deb16054a60b300550
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..112e9b153c36d0cdba70bf4bab89ae3a919bfbae
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt).
+2022-12-21,10:19:59 | INFO | Model:
+2022-12-21,10:19:59 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:59 | INFO | Params:
+2022-12-21,10:19:59 | INFO |   batch_size: 256
+2022-12-21,10:19:59 | INFO |   beta1: 0.9
+2022-12-21,10:19:59 | INFO |   beta2: 0.98
+2022-12-21,10:19:59 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/checkpoints
+2022-12-21,10:19:59 | INFO |   copy_codebase: False
+2022-12-21,10:19:59 | INFO |   csv_caption_key: title
+2022-12-21,10:19:59 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:59 | INFO |   csv_separator: 	
+2022-12-21,10:19:59 | INFO |   dataset_resampled: False
+2022-12-21,10:19:59 | INFO |   dataset_type: auto
+2022-12-21,10:19:59 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:59 | INFO |   debug: False
+2022-12-21,10:19:59 | INFO |   device: cuda:0
+2022-12-21,10:19:59 | INFO |   dist_backend: nccl
+2022-12-21,10:19:59 | INFO |   dist_url: env://
+2022-12-21,10:19:59 | INFO |   distributed: False
+2022-12-21,10:19:59 | INFO |   epochs: 32
+2022-12-21,10:19:59 | INFO |   eps: 1e-06
+2022-12-21,10:19:59 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:59 | INFO |   gather_with_grad: False
+2022-12-21,10:19:59 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:59 | INFO |   horovod: False
+2022-12-21,10:19:59 | INFO |   image_mean: None
+2022-12-21,10:19:59 | INFO |   image_std: None
+2022-12-21,10:19:59 | INFO |   imagenet_v2: None
+2022-12-21,10:19:59 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:59 | INFO |   local_loss: False
+2022-12-21,10:19:59 | INFO |   local_rank: 0
+2022-12-21,10:19:59 | INFO |   lock_image: False
+2022-12-21,10:19:59 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:59 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:59 | INFO |   log_level: 20
+2022-12-21,10:19:59 | INFO |   log_local: False
+2022-12-21,10:19:59 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/out.log
+2022-12-21,10:19:59 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:59 | INFO |   lr: 0.0005
+2022-12-21,10:19:59 | INFO |   model: ViT-B-16
+2022-12-21,10:19:59 | INFO |   name: epoch_34.pt
+2022-12-21,10:19:59 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:59 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:59 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:59 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt
+2022-12-21,10:19:59 | INFO |   pretrained_image: False
+2022-12-21,10:19:59 | INFO |   rank: 0
+2022-12-21,10:19:59 | INFO |   report_to: 
+2022-12-21,10:19:59 | INFO |   resume: None
+2022-12-21,10:19:59 | INFO |   save_frequency: 1
+2022-12-21,10:19:59 | INFO |   save_most_recent: False
+2022-12-21,10:19:59 | INFO |   seed: 0
+2022-12-21,10:19:59 | INFO |   skip_scheduler: False
+2022-12-21,10:19:59 | INFO |   tensorboard: False
+2022-12-21,10:19:59 | INFO |   tensorboard_path: 
+2022-12-21,10:19:59 | INFO |   torchscript: False
+2022-12-21,10:19:59 | INFO |   trace: False
+2022-12-21,10:19:59 | INFO |   train_data: None
+2022-12-21,10:19:59 | INFO |   train_num_samples: None
+2022-12-21,10:19:59 | INFO |   use_bn_sync: False
+2022-12-21,10:19:59 | INFO |   val_data: None
+2022-12-21,10:19:59 | INFO |   val_frequency: 1
+2022-12-21,10:19:59 | INFO |   val_num_samples: None
+2022-12-21,10:19:59 | INFO |   wandb: False
+2022-12-21,10:19:59 | INFO |   wandb_notes: 
+2022-12-21,10:19:59 | INFO |   warmup: 10000
+2022-12-21,10:19:59 | INFO |   wd: 0.2
+2022-12-21,10:19:59 | INFO |   workers: 1
+2022-12-21,10:19:59 | INFO |   world_size: 1
+2022-12-21,10:19:59 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:59 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:59 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:18,  1.47it/s]
  1%|          | 6/1000 [00:00<01:40,  9.90it/s]
  1%|          | 12/1000 [00:00<00:51, 19.30it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.89it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.01it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.84it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.26it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.61it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.92it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.76it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.11it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.07it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.75it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.23it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.74it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.04it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.38it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.56it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.80it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.93it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.89it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.81it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.86it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.82it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.65it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.78it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.83it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.73it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.60it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.60it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.55it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.32it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.37it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.37it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.18it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.21it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.95it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.01it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.13it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.12it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.25it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.39it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.54it/s]
 26%|██▋       | 264/1000 [00:05<00:14, 51.13it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 52.78it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 54.15it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 55.29it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 56.11it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.82it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.16it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.39it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.69it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.79it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.68it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.34it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.53it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.73it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.59it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.68it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.81it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.92it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.84it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 56.47it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 56.84it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.17it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.17it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.40it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.54it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.68it/s]
 43%|████▎     | 426/1000 [00:08<00:10, 57.20it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.51it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.67it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.57it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.46it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.64it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.73it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.75it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.64it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.67it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.74it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.80it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.76it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.85it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.78it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.86it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.89it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.98it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.93it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.05it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.05it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.94it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.97it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.03it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.67it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.65it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.81it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.81it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.75it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.86it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.78it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.79it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.93it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.02it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.85it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.79it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.93it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.87it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.93it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.00it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.94it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.98it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.00it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.02it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.92it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.00it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.99it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.98it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.95it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.98it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.97it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.05it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.13it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.97it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.07it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.02it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.06it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.09it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.89it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.86it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.92it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.88it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.90it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.98it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.10it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.14it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.05it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.09it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.03it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.03it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.96it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.01it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.01it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.89it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.99it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.00it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.96it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.09it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.06it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.07it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.11it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.06it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.11it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.88it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.68it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.44it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.56it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.61it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.41it/s]
+2022-12-21,10:20:17 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:04, 137.11it/s]
  1%|          | 512/50176 [00:03<05:43, 144.42it/s]
  2%|▏         | 768/50176 [00:05<05:44, 143.41it/s]
  2%|▏         | 1024/50176 [00:07<05:42, 143.34it/s]
  3%|▎         | 1280/50176 [00:08<05:43, 142.15it/s]
  3%|▎         | 1536/50176 [00:11<06:00, 135.06it/s]
  4%|▎         | 1792/50176 [00:12<05:56, 135.61it/s]
  4%|▍         | 2048/50176 [00:14<05:50, 137.23it/s]
  5%|▍         | 2304/50176 [00:16<05:46, 138.19it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.31it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 139.74it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 143.12it/s]
  7%|▋         | 3328/50176 [00:23<05:25, 143.91it/s]
  7%|▋         | 3584/50176 [00:25<05:22, 144.34it/s]
  8%|▊         | 3840/50176 [00:27<05:34, 138.67it/s]
  8%|▊         | 4096/50176 [00:29<05:31, 138.98it/s]
  9%|▊         | 4352/50176 [00:30<05:24, 141.15it/s]
  9%|▉         | 4608/50176 [00:32<05:22, 141.32it/s]
 10%|▉         | 4864/50176 [00:34<05:16, 143.03it/s]
 10%|█         | 5120/50176 [00:36<05:13, 143.85it/s]
 11%|█         | 5376/50176 [00:38<05:09, 144.68it/s]
 11%|█         | 5632/50176 [00:39<05:06, 145.48it/s]
 12%|█▏        | 5888/50176 [00:41<05:02, 146.44it/s]
 12%|█▏        | 6144/50176 [00:43<05:28, 134.13it/s]
 13%|█▎        | 6400/50176 [00:45<05:16, 138.19it/s]
 13%|█▎        | 6656/50176 [00:47<05:09, 140.49it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.52it/s]
 14%|█▍        | 7168/50176 [00:50<05:03, 141.69it/s]
 15%|█▍        | 7424/50176 [00:52<04:54, 145.02it/s]
 15%|█▌        | 7680/50176 [00:54<04:55, 144.03it/s]
 16%|█▌        | 7936/50176 [00:55<04:45, 148.00it/s]
 16%|█▋        | 8192/50176 [00:57<04:57, 141.10it/s]
 17%|█▋        | 8448/50176 [00:59<04:49, 144.39it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.29it/s]
 18%|█▊        | 8960/50176 [01:03<04:52, 141.11it/s]
 18%|█▊        | 9216/50176 [01:04<04:40, 145.82it/s]
 19%|█▉        | 9472/50176 [01:06<04:38, 146.10it/s]
 19%|█▉        | 9728/50176 [01:08<04:42, 143.42it/s]
 20%|█▉        | 9984/50176 [01:10<04:35, 146.12it/s]
 20%|██        | 10240/50176 [01:11<04:29, 148.22it/s]
 21%|██        | 10496/50176 [01:13<04:27, 148.59it/s]
 21%|██▏       | 10752/50176 [01:15<04:32, 144.80it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 147.15it/s]
 22%|██▏       | 11264/50176 [01:18<04:23, 147.60it/s]
 23%|██▎       | 11520/50176 [01:20<04:16, 150.92it/s]
 23%|██▎       | 11776/50176 [01:22<04:15, 150.27it/s]
 24%|██▍       | 12032/50176 [01:23<04:13, 150.48it/s]
 24%|██▍       | 12288/50176 [01:25<04:10, 151.46it/s]
 25%|██▌       | 12544/50176 [01:27<04:10, 150.10it/s]
 26%|██▌       | 12800/50176 [01:28<04:07, 150.76it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.20it/s]
 27%|██▋       | 13312/50176 [01:32<04:05, 150.06it/s]
 27%|██▋       | 13568/50176 [01:34<04:18, 141.45it/s]
 28%|██▊       | 13824/50176 [01:36<04:21, 138.98it/s]
 28%|██▊       | 14080/50176 [01:38<04:14, 141.61it/s]
 29%|██▊       | 14336/50176 [01:40<04:19, 138.15it/s]
 29%|██▉       | 14592/50176 [01:41<04:07, 143.83it/s]
 30%|██▉       | 14848/50176 [01:43<04:19, 135.99it/s]
 30%|███       | 15104/50176 [01:45<04:19, 135.10it/s]
 31%|███       | 15360/50176 [01:47<04:11, 138.18it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.79it/s]
 32%|███▏      | 15872/50176 [01:51<04:10, 136.99it/s]
 32%|███▏      | 16128/50176 [01:52<04:04, 139.40it/s]
 33%|███▎      | 16384/50176 [01:54<04:01, 139.72it/s]
 33%|███▎      | 16640/50176 [01:56<04:08, 134.95it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 140.11it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.88it/s]
 35%|███▍      | 17408/50176 [02:01<03:48, 143.63it/s]
 35%|███▌      | 17664/50176 [02:03<03:44, 144.69it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.33it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.25it/s]
 37%|███▋      | 18432/50176 [02:08<03:35, 147.50it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.13it/s]
 38%|███▊      | 18944/50176 [02:12<03:39, 142.17it/s]
 38%|███▊      | 19200/50176 [02:14<03:50, 134.16it/s]
 39%|███▉      | 19456/50176 [02:16<03:52, 131.97it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.23it/s]
 40%|███▉      | 19968/50176 [02:20<03:44, 134.79it/s]
 40%|████      | 20224/50176 [02:22<03:32, 140.84it/s]
 41%|████      | 20480/50176 [02:23<03:25, 144.48it/s]
 41%|████▏     | 20736/50176 [02:25<03:23, 144.41it/s]
 42%|████▏     | 20992/50176 [02:27<03:19, 146.58it/s]
 42%|████▏     | 21248/50176 [02:29<03:23, 142.30it/s]
 43%|████▎     | 21504/50176 [02:30<03:15, 146.91it/s]
 43%|████▎     | 21760/50176 [02:32<03:12, 147.28it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 150.89it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.30it/s]
 45%|████▍     | 22528/50176 [02:37<03:16, 140.77it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.92it/s]
 46%|████▌     | 23040/50176 [02:42<03:45, 120.35it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.49it/s]
 47%|████▋     | 23552/50176 [02:47<03:58, 111.58it/s]
 47%|████▋     | 23808/50176 [02:49<03:51, 113.67it/s]
 48%|████▊     | 24064/50176 [02:51<03:49, 113.78it/s]
 48%|████▊     | 24320/50176 [02:54<03:52, 111.42it/s]
 49%|████▉     | 24576/50176 [02:56<03:44, 114.05it/s]
 49%|████▉     | 24832/50176 [02:58<03:33, 118.66it/s]
 50%|█████     | 25088/50176 [03:00<03:31, 118.84it/s]
 51%|█████     | 25344/50176 [03:02<03:26, 120.48it/s]
 51%|█████     | 25600/50176 [03:05<03:29, 117.03it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.22it/s]
 52%|█████▏    | 26112/50176 [03:09<03:27, 116.00it/s]
 53%|█████▎    | 26368/50176 [03:11<03:30, 113.30it/s]
 53%|█████▎    | 26624/50176 [03:13<03:24, 115.28it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.40it/s]
 54%|█████▍    | 27136/50176 [03:18<03:23, 113.23it/s]
 55%|█████▍    | 27392/50176 [03:20<03:18, 114.78it/s]
 55%|█████▌    | 27648/50176 [03:22<03:03, 122.69it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.53it/s]
 56%|█████▌    | 28160/50176 [03:27<03:16, 111.77it/s]
 57%|█████▋    | 28416/50176 [03:29<03:14, 112.04it/s]
 57%|█████▋    | 28672/50176 [03:32<03:14, 110.69it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.80it/s]
 58%|█████▊    | 29184/50176 [03:36<03:06, 112.74it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.47it/s]
 59%|█████▉    | 29696/50176 [03:41<03:17, 103.89it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.49it/s]
 60%|██████    | 30208/50176 [03:46<03:05, 107.61it/s]
 61%|██████    | 30464/50176 [03:48<02:53, 113.51it/s]
 61%|██████    | 30720/50176 [03:50<02:51, 113.70it/s]
 62%|██████▏   | 30976/50176 [03:52<02:46, 115.38it/s]
 62%|██████▏   | 31232/50176 [03:55<02:43, 115.52it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.80it/s]
 63%|██████▎   | 31744/50176 [03:59<02:34, 119.09it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.34it/s]
 64%|██████▍   | 32256/50176 [04:03<02:31, 118.30it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.51it/s]
 65%|██████▌   | 32768/50176 [04:08<02:30, 115.50it/s]
 66%|██████▌   | 33024/50176 [04:10<02:29, 114.70it/s]
 66%|██████▋   | 33280/50176 [04:12<02:26, 115.35it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.83it/s]
 67%|██████▋   | 33792/50176 [04:17<02:26, 111.73it/s]
 68%|██████▊   | 34048/50176 [04:19<02:18, 116.12it/s]
 68%|██████▊   | 34304/50176 [04:21<02:14, 117.66it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.66it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.92it/s]
 70%|██████▉   | 35072/50176 [04:28<02:13, 112.95it/s]
 70%|███████   | 35328/50176 [04:30<02:11, 113.12it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.87it/s]
 71%|███████▏  | 35840/50176 [04:35<02:08, 111.46it/s]
 72%|███████▏  | 36096/50176 [04:37<02:03, 113.57it/s]
 72%|███████▏  | 36352/50176 [04:39<02:03, 112.11it/s]
 73%|███████▎  | 36608/50176 [04:42<02:03, 109.88it/s]
 73%|███████▎  | 36864/50176 [04:44<02:04, 106.92it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 109.02it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 114.97it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.69it/s]
 76%|███████▌  | 37888/50176 [04:53<01:46, 115.87it/s]
 76%|███████▌  | 38144/50176 [04:55<01:42, 117.35it/s]
 77%|███████▋  | 38400/50176 [04:58<01:43, 113.26it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 113.85it/s]
 78%|███████▊  | 38912/50176 [05:02<01:39, 112.91it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.59it/s]
 79%|███████▊  | 39424/50176 [05:07<01:34, 113.22it/s]
 79%|███████▉  | 39680/50176 [05:09<01:33, 112.58it/s]
 80%|███████▉  | 39936/50176 [05:11<01:28, 115.88it/s]
 80%|████████  | 40192/50176 [05:13<01:26, 114.97it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.44it/s]
 81%|████████  | 40704/50176 [05:18<01:28, 106.80it/s]
 82%|████████▏ | 40960/50176 [05:20<01:22, 111.37it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 111.06it/s]
 83%|████████▎ | 41472/50176 [05:25<01:20, 108.37it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 108.88it/s]
 84%|████████▎ | 41984/50176 [05:30<01:14, 109.68it/s]
 84%|████████▍ | 42240/50176 [05:32<01:11, 110.81it/s]
 85%|████████▍ | 42496/50176 [05:34<01:08, 111.74it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 113.01it/s]
 86%|████████▌ | 43008/50176 [05:39<01:03, 112.60it/s]
 86%|████████▌ | 43264/50176 [05:41<01:01, 112.96it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.59it/s]
 87%|████████▋ | 43776/50176 [05:46<00:58, 110.34it/s]
 88%|████████▊ | 44032/50176 [05:48<00:56, 109.06it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.04it/s]
 89%|████████▉ | 44544/50176 [05:53<00:52, 106.61it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.14it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.34it/s]
 90%|█████████ | 45312/50176 [06:00<00:45, 106.88it/s]
 91%|█████████ | 45568/50176 [06:02<00:41, 111.26it/s]
 91%|█████████▏| 45824/50176 [06:05<00:40, 108.80it/s]
 92%|█████████▏| 46080/50176 [06:07<00:38, 105.99it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.27it/s]
 93%|█████████▎| 46592/50176 [06:12<00:33, 108.53it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 106.11it/s]
 94%|█████████▍| 47104/50176 [06:17<00:30, 101.94it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.80it/s]
 95%|█████████▍| 47616/50176 [06:22<00:24, 102.74it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 103.01it/s]
 96%|█████████▌| 48128/50176 [06:27<00:19, 104.53it/s]
 96%|█████████▋| 48384/50176 [06:29<00:16, 106.84it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.26it/s]
 97%|█████████▋| 48896/50176 [06:34<00:11, 106.77it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 103.94it/s]
 98%|█████████▊| 49408/50176 [06:39<00:07, 103.83it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.78it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.54it/s] 
100%|██████████| 50176/50176 [06:45<00:00, 126.75it/s]
100%|██████████| 50176/50176 [06:45<00:00, 123.63it/s]
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5929	imagenet-zeroshot-val-top5: 0.8605
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4b81bb2e0480ad5d27e4dd7e03beb86ad573b0b7
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d88351b592f77743effe4509915108c527cf273dc1ddcc2c21a3fe98ab365694
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6ea40ccb4a4c194c05f3eb65b76069b56048afde
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt).
+2022-12-21,10:27:26 | INFO | Model:
+2022-12-21,10:27:26 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:26 | INFO | Params:
+2022-12-21,10:27:26 | INFO |   batch_size: 256
+2022-12-21,10:27:26 | INFO |   beta1: 0.9
+2022-12-21,10:27:26 | INFO |   beta2: 0.98
+2022-12-21,10:27:26 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/checkpoints
+2022-12-21,10:27:26 | INFO |   copy_codebase: False
+2022-12-21,10:27:26 | INFO |   csv_caption_key: title
+2022-12-21,10:27:26 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:26 | INFO |   csv_separator: 	
+2022-12-21,10:27:26 | INFO |   dataset_resampled: False
+2022-12-21,10:27:26 | INFO |   dataset_type: auto
+2022-12-21,10:27:26 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:26 | INFO |   debug: False
+2022-12-21,10:27:26 | INFO |   device: cuda:0
+2022-12-21,10:27:26 | INFO |   dist_backend: nccl
+2022-12-21,10:27:26 | INFO |   dist_url: env://
+2022-12-21,10:27:26 | INFO |   distributed: False
+2022-12-21,10:27:26 | INFO |   epochs: 32
+2022-12-21,10:27:26 | INFO |   eps: 1e-06
+2022-12-21,10:27:26 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:26 | INFO |   gather_with_grad: False
+2022-12-21,10:27:26 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:26 | INFO |   horovod: False
+2022-12-21,10:27:26 | INFO |   image_mean: None
+2022-12-21,10:27:26 | INFO |   image_std: None
+2022-12-21,10:27:26 | INFO |   imagenet_v2: None
+2022-12-21,10:27:26 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:26 | INFO |   local_loss: False
+2022-12-21,10:27:26 | INFO |   local_rank: 0
+2022-12-21,10:27:26 | INFO |   lock_image: False
+2022-12-21,10:27:26 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:26 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:26 | INFO |   log_level: 20
+2022-12-21,10:27:26 | INFO |   log_local: False
+2022-12-21,10:27:26 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/out.log
+2022-12-21,10:27:26 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:26 | INFO |   lr: 0.0005
+2022-12-21,10:27:26 | INFO |   model: ViT-B-16
+2022-12-21,10:27:26 | INFO |   name: epoch_35.pt
+2022-12-21,10:27:26 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:26 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:26 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:26 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt
+2022-12-21,10:27:26 | INFO |   pretrained_image: False
+2022-12-21,10:27:26 | INFO |   rank: 0
+2022-12-21,10:27:26 | INFO |   report_to: 
+2022-12-21,10:27:26 | INFO |   resume: None
+2022-12-21,10:27:26 | INFO |   save_frequency: 1
+2022-12-21,10:27:26 | INFO |   save_most_recent: False
+2022-12-21,10:27:26 | INFO |   seed: 0
+2022-12-21,10:27:26 | INFO |   skip_scheduler: False
+2022-12-21,10:27:26 | INFO |   tensorboard: False
+2022-12-21,10:27:26 | INFO |   tensorboard_path: 
+2022-12-21,10:27:26 | INFO |   torchscript: False
+2022-12-21,10:27:26 | INFO |   trace: False
+2022-12-21,10:27:26 | INFO |   train_data: None
+2022-12-21,10:27:26 | INFO |   train_num_samples: None
+2022-12-21,10:27:26 | INFO |   use_bn_sync: False
+2022-12-21,10:27:26 | INFO |   val_data: None
+2022-12-21,10:27:26 | INFO |   val_frequency: 1
+2022-12-21,10:27:26 | INFO |   val_num_samples: None
+2022-12-21,10:27:26 | INFO |   wandb: False
+2022-12-21,10:27:26 | INFO |   wandb_notes: 
+2022-12-21,10:27:26 | INFO |   warmup: 10000
+2022-12-21,10:27:26 | INFO |   wd: 0.2
+2022-12-21,10:27:26 | INFO |   workers: 1
+2022-12-21,10:27:26 | INFO |   world_size: 1
+2022-12-21,10:27:26 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:26 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:26 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<09:35,  1.73it/s]
  1%|          | 6/1000 [00:00<01:28, 11.30it/s]
  1%|          | 12/1000 [00:00<00:45, 21.61it/s]
  2%|▏         | 18/1000 [00:00<00:32, 30.07it/s]
  2%|▏         | 24/1000 [00:00<00:26, 37.16it/s]
  3%|▎         | 30/1000 [00:01<00:22, 42.68it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.62it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.59it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.86it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.50it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.60it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.49it/s]
  7%|▋         | 72/1000 [00:01<00:16, 56.00it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.39it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.85it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.93it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.33it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.54it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.84it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.98it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.94it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.68it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.78it/s]
 14%|█▍        | 138/1000 [00:02<00:14, 57.79it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.87it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.97it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.97it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.85it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.71it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.72it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.67it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.60it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.58it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.52it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.35it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.36it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.33it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.55it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.04it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.09it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.25it/s]
 25%|██▌       | 252/1000 [00:04<00:13, 57.43it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.59it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.58it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.43it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.49it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.72it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.84it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.03it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.00it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.98it/s]
 31%|███       | 312/1000 [00:05<00:11, 58.15it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.15it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.06it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.88it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.97it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.07it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.18it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.95it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.95it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.03it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.13it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.67it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.86it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.01it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.08it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.08it/s]
 43%|████▎     | 426/1000 [00:07<00:09, 58.12it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.20it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.97it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.05it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.18it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.09it/s]
 49%|████▊     | 486/1000 [00:08<00:08, 58.07it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.02it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.95it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.02it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.91it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.03it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.01it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.14it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.16it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.07it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.95it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.02it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.15it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.06it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.07it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.19it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.18it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.27it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.08it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.03it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.15it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.10it/s]
 66%|██████▌   | 660/1000 [00:11<00:05, 58.20it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.25it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.17it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.22it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.23it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.24it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.15it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.23it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.23it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.20it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.22it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.11it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.20it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.29it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.13it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.22it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.17it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.21it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.24it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.02it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.08it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.03it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.07it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.17it/s]
 83%|████████▎ | 834/1000 [00:14<00:02, 58.31it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.36it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.27it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.30it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.24it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.25it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.24it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.22it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.10it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.22it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.24it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.18it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.18it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.18it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.16it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.24it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.28it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.27it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.28it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.31it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.25it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.27it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.35it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.33it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.14it/s]
100%|██████████| 1000/1000 [00:17<00:00, 56.08it/s]
+2022-12-21,10:27:44 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:35, 78.55it/s]
  1%|          | 512/50176 [00:05<09:13, 89.65it/s]
  2%|▏         | 768/50176 [00:07<08:09, 100.87it/s]
  2%|▏         | 1024/50176 [00:10<07:57, 102.85it/s]
  3%|▎         | 1280/50176 [00:12<07:52, 103.49it/s]
  3%|▎         | 1536/50176 [00:15<08:18, 97.49it/s] 
  4%|▎         | 1792/50176 [00:18<08:05, 99.76it/s]
  4%|▍         | 2048/50176 [00:20<07:50, 102.20it/s]
  5%|▍         | 2304/50176 [00:23<07:50, 101.67it/s]
  5%|▌         | 2560/50176 [00:25<07:41, 103.09it/s]
  6%|▌         | 2816/50176 [00:28<07:47, 101.33it/s]
  6%|▌         | 3072/50176 [00:30<07:32, 104.18it/s]
  7%|▋         | 3328/50176 [00:32<07:19, 106.55it/s]
  7%|▋         | 3584/50176 [00:35<07:11, 108.02it/s]
  8%|▊         | 3840/50176 [00:37<07:27, 103.48it/s]
  8%|▊         | 4096/50176 [00:40<07:31, 102.13it/s]
  9%|▊         | 4352/50176 [00:42<07:25, 102.89it/s]
  9%|▉         | 4608/50176 [00:44<07:08, 106.44it/s]
 10%|▉         | 4864/50176 [00:47<07:03, 107.12it/s]
 10%|█         | 5120/50176 [00:49<06:52, 109.23it/s]
 11%|█         | 5376/50176 [00:51<06:53, 108.25it/s]
 11%|█         | 5632/50176 [00:54<06:53, 107.79it/s]
 12%|█▏        | 5888/50176 [00:56<06:54, 106.75it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.68it/s]
 13%|█▎        | 6400/50176 [01:01<06:51, 106.46it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.15it/s]
 14%|█▍        | 6912/50176 [01:06<06:40, 107.93it/s]
 14%|█▍        | 7168/50176 [01:08<06:21, 112.69it/s]
 15%|█▍        | 7424/50176 [01:10<06:14, 114.13it/s]
 15%|█▌        | 7680/50176 [01:12<06:05, 116.13it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.50it/s]
 16%|█▋        | 8192/50176 [01:17<06:06, 114.69it/s]
 17%|█▋        | 8448/50176 [01:19<05:58, 116.32it/s]
 17%|█▋        | 8704/50176 [01:21<06:04, 113.75it/s]
 18%|█▊        | 8960/50176 [01:24<06:04, 112.99it/s]
 18%|█▊        | 9216/50176 [01:26<05:57, 114.70it/s]
 19%|█▉        | 9472/50176 [01:28<05:56, 114.27it/s]
 19%|█▉        | 9728/50176 [01:30<06:01, 111.98it/s]
 20%|█▉        | 9984/50176 [01:33<05:58, 112.26it/s]
 20%|██        | 10240/50176 [01:35<05:42, 116.54it/s]
 21%|██        | 10496/50176 [01:37<05:41, 116.25it/s]
 21%|██▏       | 10752/50176 [01:39<05:54, 111.29it/s]
 22%|██▏       | 11008/50176 [01:42<05:53, 110.71it/s]
 22%|██▏       | 11264/50176 [01:44<05:40, 114.22it/s]
 23%|██▎       | 11520/50176 [01:46<05:34, 115.58it/s]
 23%|██▎       | 11776/50176 [01:48<05:28, 117.07it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.46it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.51it/s]
 25%|██▌       | 12544/50176 [01:55<05:22, 116.78it/s]
 26%|██▌       | 12800/50176 [01:57<05:13, 119.16it/s]
 26%|██▌       | 13056/50176 [01:59<05:10, 119.58it/s]
 27%|██▋       | 13312/50176 [02:01<05:10, 118.68it/s]
 27%|██▋       | 13568/50176 [02:04<05:28, 111.59it/s]
 28%|██▊       | 13824/50176 [02:06<05:29, 110.19it/s]
 28%|██▊       | 14080/50176 [02:08<05:29, 109.69it/s]
 29%|██▊       | 14336/50176 [02:11<05:32, 107.70it/s]
 29%|██▉       | 14592/50176 [02:13<05:18, 111.77it/s]
 30%|██▉       | 14848/50176 [02:15<05:29, 107.33it/s]
 30%|███       | 15104/50176 [02:18<05:28, 106.83it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.79it/s]
 31%|███       | 15616/50176 [02:22<05:12, 110.45it/s]
 32%|███▏      | 15872/50176 [02:25<05:17, 108.02it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.55it/s]
 33%|███▎      | 16384/50176 [02:29<05:06, 110.09it/s]
 33%|███▎      | 16640/50176 [02:32<05:20, 104.60it/s]
 34%|███▎      | 16896/50176 [02:34<05:10, 107.05it/s]
 34%|███▍      | 17152/50176 [02:37<05:04, 108.57it/s]
 35%|███▍      | 17408/50176 [02:39<04:56, 110.70it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.42it/s]
 36%|███▌      | 17920/50176 [02:43<04:39, 115.36it/s]
 36%|███▌      | 18176/50176 [02:45<04:38, 114.92it/s]
 37%|███▋      | 18432/50176 [02:48<04:37, 114.38it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.00it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.63it/s]
 38%|███▊      | 19200/50176 [02:55<04:43, 109.32it/s]
 39%|███▉      | 19456/50176 [02:57<04:49, 106.26it/s]
 39%|███▉      | 19712/50176 [03:00<04:51, 104.48it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.10it/s]
 40%|████      | 20224/50176 [03:04<04:32, 109.97it/s]
 41%|████      | 20480/50176 [03:06<04:20, 114.18it/s]
 41%|████▏     | 20736/50176 [03:09<04:17, 114.46it/s]
 42%|████▏     | 20992/50176 [03:11<04:04, 119.44it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.87it/s]
 43%|████▎     | 21504/50176 [03:15<04:10, 114.61it/s]
 43%|████▎     | 21760/50176 [03:18<04:08, 114.57it/s]
 44%|████▍     | 22016/50176 [03:19<03:55, 119.44it/s]
 44%|████▍     | 22272/50176 [03:22<03:55, 118.58it/s]
 45%|████▍     | 22528/50176 [03:24<03:54, 118.11it/s]
 45%|████▌     | 22784/50176 [03:26<04:02, 112.92it/s]
 46%|████▌     | 23040/50176 [03:29<04:02, 111.80it/s]
 46%|████▋     | 23296/50176 [03:31<04:07, 108.39it/s]
 47%|████▋     | 23552/50176 [03:34<04:09, 106.76it/s]
 47%|████▋     | 23808/50176 [03:36<03:55, 111.75it/s]
 48%|████▊     | 24064/50176 [03:38<03:49, 113.56it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.68it/s]
 49%|████▉     | 24576/50176 [03:43<03:47, 112.77it/s]
 49%|████▉     | 24832/50176 [03:45<03:42, 113.98it/s]
 50%|█████     | 25088/50176 [03:47<03:35, 116.45it/s]
 51%|█████     | 25344/50176 [03:49<03:30, 118.21it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 114.22it/s]
 52%|█████▏    | 25856/50176 [03:53<03:29, 116.22it/s]
 52%|█████▏    | 26112/50176 [03:56<03:28, 115.59it/s]
 53%|█████▎    | 26368/50176 [03:58<03:31, 112.30it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 115.13it/s]
 54%|█████▎    | 26880/50176 [04:03<03:24, 113.95it/s]
 54%|█████▍    | 27136/50176 [04:05<03:21, 114.55it/s]
 55%|█████▍    | 27392/50176 [04:07<03:15, 116.31it/s]
 55%|█████▌    | 27648/50176 [04:09<03:03, 122.66it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.30it/s]
 56%|█████▌    | 28160/50176 [04:14<03:20, 110.02it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.46it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.77it/s]
 58%|█████▊    | 28928/50176 [04:21<03:08, 112.52it/s]
 58%|█████▊    | 29184/50176 [04:23<03:09, 110.77it/s]
 59%|█████▊    | 29440/50176 [04:26<03:15, 105.88it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.22it/s]
 60%|█████▉    | 29952/50176 [04:31<03:11, 105.47it/s]
 60%|██████    | 30208/50176 [04:33<03:06, 107.24it/s]
 61%|██████    | 30464/50176 [04:35<02:56, 111.95it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.40it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.38it/s]
 62%|██████▏   | 31232/50176 [04:42<02:45, 114.57it/s]
 63%|██████▎   | 31488/50176 [04:44<02:41, 116.03it/s]
 63%|██████▎   | 31744/50176 [04:46<02:37, 117.26it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.73it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 117.00it/s]
 65%|██████▍   | 32512/50176 [04:53<02:38, 111.22it/s]
 65%|██████▌   | 32768/50176 [04:55<02:32, 114.52it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.73it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.89it/s]
 67%|██████▋   | 33536/50176 [05:02<02:30, 110.91it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.36it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.55it/s]
 68%|██████▊   | 34304/50176 [05:08<02:16, 116.58it/s]
 69%|██████▉   | 34560/50176 [05:11<02:19, 111.74it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.77it/s]
 70%|██████▉   | 35072/50176 [05:16<02:18, 108.82it/s]
 70%|███████   | 35328/50176 [05:18<02:13, 110.93it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 109.77it/s]
 71%|███████▏  | 35840/50176 [05:23<02:12, 108.25it/s]
 72%|███████▏  | 36096/50176 [05:25<02:07, 110.12it/s]
 72%|███████▏  | 36352/50176 [05:28<02:09, 107.10it/s]
 73%|███████▎  | 36608/50176 [05:30<02:05, 108.20it/s]
 73%|███████▎  | 36864/50176 [05:32<02:03, 107.41it/s]
 74%|███████▍  | 37120/50176 [05:35<02:02, 106.28it/s]
 74%|███████▍  | 37376/50176 [05:37<01:53, 112.32it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.43it/s]
 76%|███████▌  | 37888/50176 [05:42<01:51, 110.14it/s]
 76%|███████▌  | 38144/50176 [05:44<01:46, 112.83it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 110.95it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.37it/s]
 78%|███████▊  | 38912/50176 [05:51<01:42, 110.09it/s]
 78%|███████▊  | 39168/50176 [05:53<01:41, 108.87it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.91it/s]
 79%|███████▉  | 39680/50176 [05:58<01:37, 107.87it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.36it/s]
 80%|████████  | 40192/50176 [06:03<01:34, 105.92it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.16it/s]
 81%|████████  | 40704/50176 [06:08<01:33, 101.81it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.60it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.45it/s]
 83%|████████▎ | 41472/50176 [06:15<01:22, 105.97it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.55it/s]
 84%|████████▎ | 41984/50176 [06:20<01:15, 108.51it/s]
 84%|████████▍ | 42240/50176 [06:22<01:13, 108.42it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 110.06it/s]
 85%|████████▌ | 42752/50176 [06:27<01:07, 109.47it/s]
 86%|████████▌ | 43008/50176 [06:29<01:05, 110.08it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.56it/s]
 87%|████████▋ | 43520/50176 [06:34<01:02, 105.98it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.07it/s]
 88%|████████▊ | 44032/50176 [06:39<00:58, 105.31it/s]
 88%|████████▊ | 44288/50176 [06:41<00:54, 107.69it/s]
 89%|████████▉ | 44544/50176 [06:44<00:54, 104.17it/s]
 89%|████████▉ | 44800/50176 [06:46<00:51, 104.89it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.03it/s]
 90%|█████████ | 45312/50176 [06:51<00:45, 106.30it/s]
 91%|█████████ | 45568/50176 [06:53<00:41, 110.14it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.33it/s]
 92%|█████████▏| 46080/50176 [06:58<00:39, 104.48it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.42it/s]
 93%|█████████▎| 46592/50176 [07:03<00:33, 108.34it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.16it/s]
 94%|█████████▍| 47104/50176 [07:08<00:29, 102.41it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.35it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.79it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.08it/s]
 96%|█████████▌| 48128/50176 [07:18<00:19, 105.02it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.21it/s]
 97%|█████████▋| 48640/50176 [07:23<00:14, 105.29it/s]
 97%|█████████▋| 48896/50176 [07:25<00:12, 106.23it/s]
 98%|█████████▊| 49152/50176 [07:28<00:09, 104.03it/s]
 98%|█████████▊| 49408/50176 [07:30<00:07, 103.43it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.77it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.98it/s] 
100%|██████████| 50176/50176 [07:36<00:00, 127.47it/s]
100%|██████████| 50176/50176 [07:36<00:00, 109.90it/s]
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5998	imagenet-zeroshot-val-top5: 0.8633
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9ad460aefc0d113f118f7290c8c2a5ba2a1a230
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:724c2b5395fc29d8f6e310992a5eb14cfaed06a5a9bdedf369ad26caca0b75e0
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d091bfba3ca6587ac8f7ea934d8c162c03552e92
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt).
+2022-12-21,10:27:27 | INFO | Model:
+2022-12-21,10:27:27 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:27 | INFO | Params:
+2022-12-21,10:27:27 | INFO |   batch_size: 256
+2022-12-21,10:27:27 | INFO |   beta1: 0.9
+2022-12-21,10:27:27 | INFO |   beta2: 0.98
+2022-12-21,10:27:27 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/checkpoints
+2022-12-21,10:27:27 | INFO |   copy_codebase: False
+2022-12-21,10:27:27 | INFO |   csv_caption_key: title
+2022-12-21,10:27:27 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:27 | INFO |   csv_separator: 	
+2022-12-21,10:27:27 | INFO |   dataset_resampled: False
+2022-12-21,10:27:27 | INFO |   dataset_type: auto
+2022-12-21,10:27:27 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:27 | INFO |   debug: False
+2022-12-21,10:27:27 | INFO |   device: cuda:0
+2022-12-21,10:27:27 | INFO |   dist_backend: nccl
+2022-12-21,10:27:27 | INFO |   dist_url: env://
+2022-12-21,10:27:27 | INFO |   distributed: False
+2022-12-21,10:27:27 | INFO |   epochs: 32
+2022-12-21,10:27:27 | INFO |   eps: 1e-06
+2022-12-21,10:27:27 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:27 | INFO |   gather_with_grad: False
+2022-12-21,10:27:27 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:27 | INFO |   horovod: False
+2022-12-21,10:27:27 | INFO |   image_mean: None
+2022-12-21,10:27:27 | INFO |   image_std: None
+2022-12-21,10:27:27 | INFO |   imagenet_v2: None
+2022-12-21,10:27:27 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:27 | INFO |   local_loss: False
+2022-12-21,10:27:27 | INFO |   local_rank: 0
+2022-12-21,10:27:27 | INFO |   lock_image: False
+2022-12-21,10:27:27 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:27 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:27 | INFO |   log_level: 20
+2022-12-21,10:27:27 | INFO |   log_local: False
+2022-12-21,10:27:27 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/out.log
+2022-12-21,10:27:27 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:27 | INFO |   lr: 0.0005
+2022-12-21,10:27:27 | INFO |   model: ViT-B-16
+2022-12-21,10:27:27 | INFO |   name: epoch_36.pt
+2022-12-21,10:27:27 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:27 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:27 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:27 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt
+2022-12-21,10:27:27 | INFO |   pretrained_image: False
+2022-12-21,10:27:27 | INFO |   rank: 0
+2022-12-21,10:27:27 | INFO |   report_to: 
+2022-12-21,10:27:27 | INFO |   resume: None
+2022-12-21,10:27:27 | INFO |   save_frequency: 1
+2022-12-21,10:27:27 | INFO |   save_most_recent: False
+2022-12-21,10:27:27 | INFO |   seed: 0
+2022-12-21,10:27:27 | INFO |   skip_scheduler: False
+2022-12-21,10:27:27 | INFO |   tensorboard: False
+2022-12-21,10:27:27 | INFO |   tensorboard_path: 
+2022-12-21,10:27:27 | INFO |   torchscript: False
+2022-12-21,10:27:27 | INFO |   trace: False
+2022-12-21,10:27:27 | INFO |   train_data: None
+2022-12-21,10:27:27 | INFO |   train_num_samples: None
+2022-12-21,10:27:27 | INFO |   use_bn_sync: False
+2022-12-21,10:27:27 | INFO |   val_data: None
+2022-12-21,10:27:27 | INFO |   val_frequency: 1
+2022-12-21,10:27:27 | INFO |   val_num_samples: None
+2022-12-21,10:27:27 | INFO |   wandb: False
+2022-12-21,10:27:27 | INFO |   wandb_notes: 
+2022-12-21,10:27:27 | INFO |   warmup: 10000
+2022-12-21,10:27:27 | INFO |   wd: 0.2
+2022-12-21,10:27:27 | INFO |   workers: 1
+2022-12-21,10:27:27 | INFO |   world_size: 1
+2022-12-21,10:27:27 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:27 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:27 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:30,  1.45it/s]
  1%|          | 6/1000 [00:00<01:42,  9.70it/s]
  1%|          | 12/1000 [00:00<00:51, 19.04it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.42it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.63it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.49it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.01it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.39it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.94it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.80it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.13it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.14it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.80it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.13it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.66it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.96it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.35it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.47it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.74it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.88it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.78it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.82it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.79it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.81it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.88it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.90it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.78it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.65it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.66it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.62it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.55it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.52it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.47it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.27it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.17it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.27it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.25it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.46it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.45it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.35it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.43it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.52it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.63it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 55.09it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 55.68it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 56.25it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 56.80it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.14it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.55it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.66it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.70it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.85it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.88it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.84it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.70it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.79it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.91it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.01it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.80it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.81it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.88it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.97it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.86it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.54it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.61it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.72it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.84it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.92it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.92it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.94it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.97it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.08it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.08it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.86it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.97it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.96it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.93it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.07it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.98it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.96it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.96it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.87it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.92it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.82it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.92it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.00it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.95it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.05it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.05it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.95it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.95it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.00it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.90it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.84it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.98it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.87it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.00it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.93it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.05it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.04it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.11it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.92it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.86it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.02it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.97it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.05it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.10it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.03it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.09it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.10it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.13it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.02it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.10it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.07it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.03it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.04it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.96it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.02it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.01it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.07it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.14it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.97it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.08it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.96it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.00it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.05it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.88it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.88it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.95it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.92it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.92it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.01it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.14it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.17it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.09it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.14it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.08it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.08it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.97it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.02it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.92it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.02it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.02it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.99it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.99it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.08it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.11it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.11it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.12it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.15it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.09it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.17it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.25it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.87it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.97it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.00it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.87it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.56it/s]
+2022-12-21,10:27:45 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<08:54, 93.33it/s]
  1%|          | 512/50176 [00:05<08:29, 97.48it/s]
  2%|▏         | 768/50176 [00:07<07:47, 105.61it/s]
  2%|▏         | 1024/50176 [00:09<07:43, 106.10it/s]
  3%|▎         | 1280/50176 [00:12<07:44, 105.19it/s]
  3%|▎         | 1536/50176 [00:15<08:12, 98.73it/s] 
  4%|▎         | 1792/50176 [00:17<08:00, 100.72it/s]
  4%|▍         | 2048/50176 [00:20<07:47, 103.00it/s]
  5%|▍         | 2304/50176 [00:22<07:48, 102.11it/s]
  5%|▌         | 2560/50176 [00:25<07:41, 103.28it/s]
  6%|▌         | 2816/50176 [00:27<07:46, 101.50it/s]
  6%|▌         | 3072/50176 [00:29<07:31, 104.39it/s]
  7%|▋         | 3328/50176 [00:32<07:19, 106.55it/s]
  7%|▋         | 3584/50176 [00:34<07:10, 108.18it/s]
  8%|▊         | 3840/50176 [00:37<07:26, 103.67it/s]
  8%|▊         | 4096/50176 [00:39<07:31, 102.01it/s]
  9%|▊         | 4352/50176 [00:42<07:24, 103.03it/s]
  9%|▉         | 4608/50176 [00:44<07:07, 106.54it/s]
 10%|▉         | 4864/50176 [00:46<07:03, 107.11it/s]
 10%|█         | 5120/50176 [00:49<06:52, 109.16it/s]
 11%|█         | 5376/50176 [00:51<06:54, 108.05it/s]
 11%|█         | 5632/50176 [00:53<06:52, 107.88it/s]
 12%|█▏        | 5888/50176 [00:56<06:54, 106.76it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.59it/s]
 13%|█▎        | 6400/50176 [01:01<06:51, 106.45it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.13it/s]
 14%|█▍        | 6912/50176 [01:05<06:40, 107.92it/s]
 14%|█▍        | 7168/50176 [01:07<06:21, 112.71it/s]
 15%|█▍        | 7424/50176 [01:10<06:13, 114.36it/s]
 15%|█▌        | 7680/50176 [01:12<06:06, 115.87it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.48it/s]
 16%|█▋        | 8192/50176 [01:16<06:05, 114.79it/s]
 17%|█▋        | 8448/50176 [01:18<05:58, 116.24it/s]
 17%|█▋        | 8704/50176 [01:21<06:04, 113.74it/s]
 18%|█▊        | 8960/50176 [01:23<06:04, 112.98it/s]
 18%|█▊        | 9216/50176 [01:25<05:56, 114.87it/s]
 19%|█▉        | 9472/50176 [01:27<05:57, 113.78it/s]
 19%|█▉        | 9728/50176 [01:30<06:00, 112.25it/s]
 20%|█▉        | 9984/50176 [01:32<05:58, 112.25it/s]
 20%|██        | 10240/50176 [01:34<05:42, 116.70it/s]
 21%|██        | 10496/50176 [01:36<05:41, 116.20it/s]
 21%|██▏       | 10752/50176 [01:39<05:54, 111.32it/s]
 22%|██▏       | 11008/50176 [01:41<05:53, 110.69it/s]
 22%|██▏       | 11264/50176 [01:43<05:40, 114.27it/s]
 23%|██▎       | 11520/50176 [01:45<05:34, 115.48it/s]
 23%|██▎       | 11776/50176 [01:47<05:27, 117.15it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.40it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.50it/s]
 25%|██▌       | 12544/50176 [01:54<05:22, 116.65it/s]
 26%|██▌       | 12800/50176 [01:56<05:13, 119.17it/s]
 26%|██▌       | 13056/50176 [01:58<05:10, 119.48it/s]
 27%|██▋       | 13312/50176 [02:00<05:10, 118.79it/s]
 27%|██▋       | 13568/50176 [02:03<05:27, 111.80it/s]
 28%|██▊       | 13824/50176 [02:05<05:29, 110.26it/s]
 28%|██▊       | 14080/50176 [02:08<05:28, 109.76it/s]
 29%|██▊       | 14336/50176 [02:10<05:32, 107.68it/s]
 29%|██▉       | 14592/50176 [02:12<05:19, 111.46it/s]
 30%|██▉       | 14848/50176 [02:15<05:28, 107.50it/s]
 30%|███       | 15104/50176 [02:17<05:27, 106.94it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.62it/s]
 31%|███       | 15616/50176 [02:22<05:13, 110.31it/s]
 32%|███▏      | 15872/50176 [02:24<05:17, 108.20it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.68it/s]
 33%|███▎      | 16384/50176 [02:29<05:07, 109.94it/s]
 33%|███▎      | 16640/50176 [02:32<05:20, 104.48it/s]
 34%|███▎      | 16896/50176 [02:34<05:10, 107.10it/s]
 34%|███▍      | 17152/50176 [02:36<05:04, 108.47it/s]
 35%|███▍      | 17408/50176 [02:38<04:56, 110.60it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.53it/s]
 36%|███▌      | 17920/50176 [02:43<04:40, 115.08it/s]
 36%|███▌      | 18176/50176 [02:45<04:38, 115.08it/s]
 37%|███▋      | 18432/50176 [02:47<04:37, 114.30it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.03it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.71it/s]
 38%|███▊      | 19200/50176 [02:54<04:42, 109.51it/s]
 39%|███▉      | 19456/50176 [02:57<04:48, 106.32it/s]
 39%|███▉      | 19712/50176 [02:59<04:51, 104.43it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.06it/s]
 40%|████      | 20224/50176 [03:04<04:31, 110.17it/s]
 41%|████      | 20480/50176 [03:06<04:20, 114.13it/s]
 41%|████▏     | 20736/50176 [03:08<04:17, 114.46it/s]
 42%|████▏     | 20992/50176 [03:10<04:04, 119.22it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.98it/s]
 43%|████▎     | 21504/50176 [03:15<04:09, 114.70it/s]
 43%|████▎     | 21760/50176 [03:17<04:07, 114.61it/s]
 44%|████▍     | 22016/50176 [03:19<03:56, 119.28it/s]
 44%|████▍     | 22272/50176 [03:21<03:55, 118.56it/s]
 45%|████▍     | 22528/50176 [03:23<03:54, 118.13it/s]
 45%|████▌     | 22784/50176 [03:26<04:02, 112.78it/s]
 46%|████▌     | 23040/50176 [03:28<04:02, 111.72it/s]
 46%|████▋     | 23296/50176 [03:31<04:07, 108.57it/s]
 47%|████▋     | 23552/50176 [03:33<04:09, 106.85it/s]
 47%|████▋     | 23808/50176 [03:35<03:55, 111.87it/s]
 48%|████▊     | 24064/50176 [03:37<03:49, 113.58it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.70it/s]
 49%|████▉     | 24576/50176 [03:42<03:47, 112.73it/s]
 49%|████▉     | 24832/50176 [03:44<03:42, 114.00it/s]
 50%|█████     | 25088/50176 [03:46<03:35, 116.42it/s]
 51%|█████     | 25344/50176 [03:48<03:30, 118.23it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 113.98it/s]
 52%|█████▏    | 25856/50176 [03:53<03:28, 116.46it/s]
 52%|█████▏    | 26112/50176 [03:55<03:27, 115.81it/s]
 53%|█████▎    | 26368/50176 [03:58<03:32, 112.27it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 115.10it/s]
 54%|█████▎    | 26880/50176 [04:02<03:24, 113.95it/s]
 54%|█████▍    | 27136/50176 [04:04<03:21, 114.56it/s]
 55%|█████▍    | 27392/50176 [04:06<03:15, 116.36it/s]
 55%|█████▌    | 27648/50176 [04:08<03:03, 122.62it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.12it/s]
 56%|█████▌    | 28160/50176 [04:13<03:20, 109.92it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.41it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.68it/s]
 58%|█████▊    | 28928/50176 [04:20<03:08, 112.66it/s]
 58%|█████▊    | 29184/50176 [04:22<03:09, 110.88it/s]
 59%|█████▊    | 29440/50176 [04:25<03:15, 106.00it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.16it/s]
 60%|█████▉    | 29952/50176 [04:30<03:11, 105.40it/s]
 60%|██████    | 30208/50176 [04:32<03:06, 107.24it/s]
 61%|██████    | 30464/50176 [04:34<02:56, 112.00it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.18it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.44it/s]
 62%|██████▏   | 31232/50176 [04:41<02:45, 114.71it/s]
 63%|██████▎   | 31488/50176 [04:43<02:41, 116.00it/s]
 63%|██████▎   | 31744/50176 [04:45<02:37, 117.07it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.80it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 116.95it/s]
 65%|██████▍   | 32512/50176 [04:52<02:38, 111.31it/s]
 65%|██████▌   | 32768/50176 [04:54<02:32, 114.36it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.83it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.95it/s]
 67%|██████▋   | 33536/50176 [05:01<02:29, 110.94it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.22it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.50it/s]
 68%|██████▊   | 34304/50176 [05:08<02:15, 116.94it/s]
 69%|██████▉   | 34560/50176 [05:10<02:19, 111.71it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.83it/s]
 70%|██████▉   | 35072/50176 [05:15<02:19, 108.59it/s]
 70%|███████   | 35328/50176 [05:17<02:13, 110.93it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 109.78it/s]
 71%|███████▏  | 35840/50176 [05:22<02:12, 108.28it/s]
 72%|███████▏  | 36096/50176 [05:25<02:08, 109.94it/s]
 72%|███████▏  | 36352/50176 [05:27<02:09, 107.08it/s]
 73%|███████▎  | 36608/50176 [05:29<02:05, 108.22it/s]
 73%|███████▎  | 36864/50176 [05:32<02:03, 107.41it/s]
 74%|███████▍  | 37120/50176 [05:34<02:02, 106.41it/s]
 74%|███████▍  | 37376/50176 [05:36<01:54, 112.20it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.48it/s]
 76%|███████▌  | 37888/50176 [05:41<01:51, 110.06it/s]
 76%|███████▌  | 38144/50176 [05:43<01:46, 112.90it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 111.01it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.28it/s]
 78%|███████▊  | 38912/50176 [05:50<01:42, 110.18it/s]
 78%|███████▊  | 39168/50176 [05:53<01:40, 109.06it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.86it/s]
 79%|███████▉  | 39680/50176 [05:57<01:37, 107.82it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.39it/s]
 80%|████████  | 40192/50176 [06:02<01:34, 105.85it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.16it/s]
 81%|████████  | 40704/50176 [06:07<01:33, 101.81it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.70it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.46it/s]
 83%|████████▎ | 41472/50176 [06:14<01:22, 106.05it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.52it/s]
 84%|████████▎ | 41984/50176 [06:19<01:15, 108.58it/s]
 84%|████████▍ | 42240/50176 [06:21<01:13, 108.48it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 109.94it/s]
 85%|████████▌ | 42752/50176 [06:26<01:07, 109.41it/s]
 86%|████████▌ | 43008/50176 [06:28<01:05, 110.04it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.61it/s]
 87%|████████▋ | 43520/50176 [06:33<01:02, 105.94it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.06it/s]
 88%|████████▊ | 44032/50176 [06:38<00:58, 105.30it/s]
 88%|████████▊ | 44288/50176 [06:40<00:54, 107.78it/s]
 89%|████████▉ | 44544/50176 [06:43<00:54, 104.14it/s]
 89%|████████▉ | 44800/50176 [06:45<00:51, 104.93it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.12it/s]
 90%|█████████ | 45312/50176 [06:50<00:45, 106.14it/s]
 91%|█████████ | 45568/50176 [06:52<00:41, 110.04it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.33it/s]
 92%|█████████▏| 46080/50176 [06:57<00:39, 104.38it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.42it/s]
 93%|█████████▎| 46592/50176 [07:02<00:33, 108.23it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.12it/s]
 94%|█████████▍| 47104/50176 [07:07<00:29, 102.51it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.38it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.86it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.10it/s]
 96%|█████████▌| 48128/50176 [07:17<00:19, 105.05it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.27it/s]
 97%|█████████▋| 48640/50176 [07:22<00:14, 105.17it/s]
 97%|█████████▋| 48896/50176 [07:24<00:12, 106.19it/s]
 98%|█████████▊| 49152/50176 [07:27<00:09, 103.82it/s]
 98%|█████████▊| 49408/50176 [07:29<00:07, 103.57it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.77it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.85it/s] 
100%|██████████| 50176/50176 [07:36<00:00, 127.22it/s]
100%|██████████| 50176/50176 [07:36<00:00, 110.02it/s]
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6048	imagenet-zeroshot-val-top5: 0.8640
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt
new file mode 100644
index 0000000000000000000000000000000000000000..80014255778ddd5af1083eda563d1a3a3f642539
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ea5c9b1f8e09e79c65d8128e73815bf3e047b91480c4950778499973ebc767f
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ce8e6378963465b02a41927961a83bb3fcee19a1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt).
+2022-12-21,10:27:26 | INFO | Model:
+2022-12-21,10:27:26 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:26 | INFO | Params:
+2022-12-21,10:27:26 | INFO |   batch_size: 256
+2022-12-21,10:27:26 | INFO |   beta1: 0.9
+2022-12-21,10:27:26 | INFO |   beta2: 0.98
+2022-12-21,10:27:26 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/checkpoints
+2022-12-21,10:27:26 | INFO |   copy_codebase: False
+2022-12-21,10:27:26 | INFO |   csv_caption_key: title
+2022-12-21,10:27:26 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:26 | INFO |   csv_separator: 	
+2022-12-21,10:27:26 | INFO |   dataset_resampled: False
+2022-12-21,10:27:26 | INFO |   dataset_type: auto
+2022-12-21,10:27:26 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:26 | INFO |   debug: False
+2022-12-21,10:27:26 | INFO |   device: cuda:0
+2022-12-21,10:27:26 | INFO |   dist_backend: nccl
+2022-12-21,10:27:26 | INFO |   dist_url: env://
+2022-12-21,10:27:26 | INFO |   distributed: False
+2022-12-21,10:27:26 | INFO |   epochs: 32
+2022-12-21,10:27:26 | INFO |   eps: 1e-06
+2022-12-21,10:27:26 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:26 | INFO |   gather_with_grad: False
+2022-12-21,10:27:26 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:26 | INFO |   horovod: False
+2022-12-21,10:27:26 | INFO |   image_mean: None
+2022-12-21,10:27:26 | INFO |   image_std: None
+2022-12-21,10:27:26 | INFO |   imagenet_v2: None
+2022-12-21,10:27:26 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:26 | INFO |   local_loss: False
+2022-12-21,10:27:26 | INFO |   local_rank: 0
+2022-12-21,10:27:26 | INFO |   lock_image: False
+2022-12-21,10:27:26 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:26 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:26 | INFO |   log_level: 20
+2022-12-21,10:27:26 | INFO |   log_local: False
+2022-12-21,10:27:26 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/out.log
+2022-12-21,10:27:26 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:26 | INFO |   lr: 0.0005
+2022-12-21,10:27:26 | INFO |   model: ViT-B-16
+2022-12-21,10:27:26 | INFO |   name: epoch_37.pt
+2022-12-21,10:27:26 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:26 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:26 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:26 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt
+2022-12-21,10:27:26 | INFO |   pretrained_image: False
+2022-12-21,10:27:26 | INFO |   rank: 0
+2022-12-21,10:27:26 | INFO |   report_to: 
+2022-12-21,10:27:26 | INFO |   resume: None
+2022-12-21,10:27:26 | INFO |   save_frequency: 1
+2022-12-21,10:27:26 | INFO |   save_most_recent: False
+2022-12-21,10:27:26 | INFO |   seed: 0
+2022-12-21,10:27:26 | INFO |   skip_scheduler: False
+2022-12-21,10:27:26 | INFO |   tensorboard: False
+2022-12-21,10:27:26 | INFO |   tensorboard_path: 
+2022-12-21,10:27:26 | INFO |   torchscript: False
+2022-12-21,10:27:26 | INFO |   trace: False
+2022-12-21,10:27:26 | INFO |   train_data: None
+2022-12-21,10:27:26 | INFO |   train_num_samples: None
+2022-12-21,10:27:26 | INFO |   use_bn_sync: False
+2022-12-21,10:27:26 | INFO |   val_data: None
+2022-12-21,10:27:26 | INFO |   val_frequency: 1
+2022-12-21,10:27:26 | INFO |   val_num_samples: None
+2022-12-21,10:27:26 | INFO |   wandb: False
+2022-12-21,10:27:26 | INFO |   wandb_notes: 
+2022-12-21,10:27:26 | INFO |   warmup: 10000
+2022-12-21,10:27:26 | INFO |   wd: 0.2
+2022-12-21,10:27:26 | INFO |   workers: 1
+2022-12-21,10:27:26 | INFO |   world_size: 1
+2022-12-21,10:27:26 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:26 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:26 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:01,  1.38it/s]
  1%|          | 6/1000 [00:00<01:46,  9.32it/s]
  1%|          | 11/1000 [00:00<00:58, 17.00it/s]
  2%|▏         | 17/1000 [00:01<00:38, 25.47it/s]
  2%|▏         | 23/1000 [00:01<00:29, 32.99it/s]
  3%|▎         | 29/1000 [00:01<00:24, 39.18it/s]
  4%|▎         | 35/1000 [00:01<00:21, 44.07it/s]
  4%|▍         | 41/1000 [00:01<00:20, 47.75it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.25it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.43it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.89it/s]
  6%|▋         | 65/1000 [00:01<00:16, 55.05it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.82it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.43it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.97it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.40it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.63it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.84it/s]
 11%|█         | 107/1000 [00:02<00:15, 58.13it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.26it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.27it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 58.23it/s]
 13%|█▎        | 131/1000 [00:03<00:14, 58.29it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 58.21it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 58.19it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.25it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.27it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 58.16it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.93it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.94it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.80it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.76it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.71it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.60it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.46it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.52it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.54it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.74it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.83it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.65it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 57.67it/s]
 25%|██▌       | 251/1000 [00:05<00:12, 57.74it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.91it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.95it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.72it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.82it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.95it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 58.03it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 58.26it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 58.28it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.29it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.41it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.39it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.35it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 58.14it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.22it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.26it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.38it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 58.15it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.27it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.21it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.34it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.14it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.91it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.88it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.99it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.15it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.24it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.29it/s]
 42%|████▏     | 419/1000 [00:07<00:09, 58.32it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.23it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.39it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.40it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.20it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.25it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.32it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.28it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.27it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.37it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.26it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.26it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.24it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 58.15it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.28it/s]
 51%|█████     | 509/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.20it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.23it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.35it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.25it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.32it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.38it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.27it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.26it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.31it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.19it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 58.09it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.27it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 593/1000 [00:10<00:06, 58.18it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.30it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.21it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.28it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.36it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.35it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.41it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.29it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 58.18it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.38it/s]
 65%|██████▌   | 653/1000 [00:11<00:05, 58.30it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.38it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.37it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.40it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.38it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.42it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.29it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.41it/s]
 71%|███████   | 707/1000 [00:12<00:05, 58.39it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.38it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.42it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.37it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.36it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.39it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.33it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.41it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.51it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.31it/s]
 77%|███████▋  | 767/1000 [00:13<00:03, 58.36it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.35it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.38it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.41it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.23it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.20it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.13it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.24it/s]
 83%|████████▎ | 827/1000 [00:14<00:02, 58.25it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.43it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.53it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.39it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.45it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.40it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.45it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.39it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.41it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.43it/s]
 89%|████████▊ | 887/1000 [00:15<00:01, 58.26it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.37it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.41it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.38it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.35it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.40it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.33it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.44it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.52it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.49it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.47it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.50it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.45it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.51it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.58it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.61it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.64it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.56it/s]
100%|█████████▉| 995/1000 [00:17<00:00, 58.43it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.77it/s]
+2022-12-21,10:27:44 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:31, 79.09it/s]
  1%|          | 512/50176 [00:05<09:10, 90.14it/s]
  2%|▏         | 768/50176 [00:07<08:08, 101.19it/s]
  2%|▏         | 1024/50176 [00:10<07:56, 103.23it/s]
  3%|▎         | 1280/50176 [00:12<07:53, 103.32it/s]
  3%|▎         | 1536/50176 [00:15<08:17, 97.72it/s] 
  4%|▎         | 1792/50176 [00:18<08:04, 99.78it/s]
  4%|▍         | 2048/50176 [00:20<07:50, 102.33it/s]
  5%|▍         | 2304/50176 [00:23<07:49, 101.90it/s]
  5%|▌         | 2560/50176 [00:25<07:40, 103.31it/s]
  6%|▌         | 2816/50176 [00:28<07:47, 101.31it/s]
  6%|▌         | 3072/50176 [00:30<07:32, 104.21it/s]
  7%|▋         | 3328/50176 [00:32<07:20, 106.47it/s]
  7%|▋         | 3584/50176 [00:35<07:11, 107.88it/s]
  8%|▊         | 3840/50176 [00:37<07:28, 103.37it/s]
  8%|▊         | 4096/50176 [00:40<07:32, 101.88it/s]
  9%|▊         | 4352/50176 [00:42<07:25, 102.89it/s]
  9%|▉         | 4608/50176 [00:44<07:07, 106.49it/s]
 10%|▉         | 4864/50176 [00:47<07:02, 107.13it/s]
 10%|█         | 5120/50176 [00:49<06:51, 109.49it/s]
 11%|█         | 5376/50176 [00:51<06:54, 107.99it/s]
 11%|█         | 5632/50176 [00:54<06:53, 107.83it/s]
 12%|█▏        | 5888/50176 [00:56<06:55, 106.69it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.59it/s]
 13%|█▎        | 6400/50176 [01:01<06:51, 106.45it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.07it/s]
 14%|█▍        | 6912/50176 [01:06<06:41, 107.83it/s]
 14%|█▍        | 7168/50176 [01:08<06:21, 112.72it/s]
 15%|█▍        | 7424/50176 [01:10<06:14, 114.23it/s]
 15%|█▌        | 7680/50176 [01:12<06:06, 116.01it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.42it/s]
 16%|█▋        | 8192/50176 [01:17<06:05, 115.00it/s]
 17%|█▋        | 8448/50176 [01:19<05:58, 116.44it/s]
 17%|█▋        | 8704/50176 [01:21<06:05, 113.60it/s]
 18%|█▊        | 8960/50176 [01:23<06:04, 113.16it/s]
 18%|█▊        | 9216/50176 [01:26<05:56, 114.86it/s]
 19%|█▉        | 9472/50176 [01:28<05:57, 113.90it/s]
 19%|█▉        | 9728/50176 [01:30<06:00, 112.15it/s]
 20%|█▉        | 9984/50176 [01:33<05:57, 112.47it/s]
 20%|██        | 10240/50176 [01:35<05:42, 116.51it/s]
 21%|██        | 10496/50176 [01:37<05:41, 116.23it/s]
 21%|██▏       | 10752/50176 [01:39<05:53, 111.39it/s]
 22%|██▏       | 11008/50176 [01:42<05:53, 110.76it/s]
 22%|██▏       | 11264/50176 [01:44<05:40, 114.27it/s]
 23%|██▎       | 11520/50176 [01:46<05:34, 115.57it/s]
 23%|██▎       | 11776/50176 [01:48<05:27, 117.08it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.38it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.45it/s]
 25%|██▌       | 12544/50176 [01:55<05:22, 116.61it/s]
 26%|██▌       | 12800/50176 [01:57<05:13, 119.32it/s]
 26%|██▌       | 13056/50176 [01:59<05:11, 119.31it/s]
 27%|██▋       | 13312/50176 [02:01<05:11, 118.53it/s]
 27%|██▋       | 13568/50176 [02:04<05:27, 111.80it/s]
 28%|██▊       | 13824/50176 [02:06<05:29, 110.33it/s]
 28%|██▊       | 14080/50176 [02:08<05:28, 109.96it/s]
 29%|██▊       | 14336/50176 [02:11<05:32, 107.70it/s]
 29%|██▉       | 14592/50176 [02:13<05:19, 111.43it/s]
 30%|██▉       | 14848/50176 [02:15<05:28, 107.41it/s]
 30%|███       | 15104/50176 [02:18<05:28, 106.87it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.80it/s]
 31%|███       | 15616/50176 [02:22<05:12, 110.46it/s]
 32%|███▏      | 15872/50176 [02:25<05:17, 108.08it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.63it/s]
 33%|███▎      | 16384/50176 [02:29<05:07, 110.05it/s]
 33%|███▎      | 16640/50176 [02:32<05:21, 104.38it/s]
 34%|███▎      | 16896/50176 [02:34<05:11, 106.90it/s]
 34%|███▍      | 17152/50176 [02:37<05:03, 108.66it/s]
 35%|███▍      | 17408/50176 [02:39<04:55, 110.76it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.51it/s]
 36%|███▌      | 17920/50176 [02:43<04:40, 115.12it/s]
 36%|███▌      | 18176/50176 [02:45<04:37, 115.16it/s]
 37%|███▋      | 18432/50176 [02:48<04:38, 114.12it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.11it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.72it/s]
 38%|███▊      | 19200/50176 [02:55<04:43, 109.40it/s]
 39%|███▉      | 19456/50176 [02:57<04:48, 106.31it/s]
 39%|███▉      | 19712/50176 [03:00<04:51, 104.52it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.22it/s]
 40%|████      | 20224/50176 [03:04<04:32, 109.98it/s]
 41%|████      | 20480/50176 [03:06<04:20, 114.18it/s]
 41%|████▏     | 20736/50176 [03:09<04:17, 114.46it/s]
 42%|████▏     | 20992/50176 [03:11<04:04, 119.46it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.78it/s]
 43%|████▎     | 21504/50176 [03:15<04:10, 114.66it/s]
 43%|████▎     | 21760/50176 [03:18<04:08, 114.55it/s]
 44%|████▍     | 22016/50176 [03:19<03:55, 119.48it/s]
 44%|████▍     | 22272/50176 [03:22<03:55, 118.45it/s]
 45%|████▍     | 22528/50176 [03:24<03:54, 117.84it/s]
 45%|████▌     | 22784/50176 [03:26<04:03, 112.70it/s]
 46%|████▌     | 23040/50176 [03:29<04:03, 111.58it/s]
 46%|████▋     | 23296/50176 [03:31<04:07, 108.65it/s]
 47%|████▋     | 23552/50176 [03:34<04:09, 106.89it/s]
 47%|████▋     | 23808/50176 [03:36<03:55, 111.82it/s]
 48%|████▊     | 24064/50176 [03:38<03:49, 113.65it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.72it/s]
 49%|████▉     | 24576/50176 [03:43<03:46, 112.81it/s]
 49%|████▉     | 24832/50176 [03:45<03:41, 114.17it/s]
 50%|█████     | 25088/50176 [03:47<03:35, 116.40it/s]
 51%|█████     | 25344/50176 [03:49<03:30, 118.13it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 113.94it/s]
 52%|█████▏    | 25856/50176 [03:53<03:28, 116.42it/s]
 52%|█████▏    | 26112/50176 [03:56<03:27, 115.83it/s]
 53%|█████▎    | 26368/50176 [03:58<03:32, 112.29it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 114.98it/s]
 54%|█████▎    | 26880/50176 [04:02<03:24, 113.88it/s]
 54%|█████▍    | 27136/50176 [04:05<03:20, 114.71it/s]
 55%|█████▍    | 27392/50176 [04:07<03:15, 116.29it/s]
 55%|█████▌    | 27648/50176 [04:09<03:03, 122.70it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.39it/s]
 56%|█████▌    | 28160/50176 [04:14<03:20, 109.73it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.31it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.80it/s]
 58%|█████▊    | 28928/50176 [04:21<03:08, 112.64it/s]
 58%|█████▊    | 29184/50176 [04:23<03:09, 110.70it/s]
 59%|█████▊    | 29440/50176 [04:26<03:15, 105.96it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.28it/s]
 60%|█████▉    | 29952/50176 [04:31<03:11, 105.40it/s]
 60%|██████    | 30208/50176 [04:33<03:06, 107.25it/s]
 61%|██████    | 30464/50176 [04:35<02:56, 111.93it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.40it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.34it/s]
 62%|██████▏   | 31232/50176 [04:42<02:45, 114.52it/s]
 63%|██████▎   | 31488/50176 [04:44<02:40, 116.14it/s]
 63%|██████▎   | 31744/50176 [04:46<02:37, 117.26it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.66it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 117.05it/s]
 65%|██████▍   | 32512/50176 [04:53<02:38, 111.24it/s]
 65%|██████▌   | 32768/50176 [04:55<02:32, 114.51it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.73it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.91it/s]
 67%|██████▋   | 33536/50176 [05:02<02:30, 110.91it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.38it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.54it/s]
 68%|██████▊   | 34304/50176 [05:08<02:16, 116.51it/s]
 69%|██████▉   | 34560/50176 [05:11<02:19, 111.66it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.84it/s]
 70%|██████▉   | 35072/50176 [05:16<02:19, 108.64it/s]
 70%|███████   | 35328/50176 [05:18<02:13, 110.86it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 109.78it/s]
 71%|███████▏  | 35840/50176 [05:23<02:12, 108.26it/s]
 72%|███████▏  | 36096/50176 [05:25<02:07, 110.15it/s]
 72%|███████▏  | 36352/50176 [05:28<02:09, 107.11it/s]
 73%|███████▎  | 36608/50176 [05:30<02:05, 108.26it/s]
 73%|███████▎  | 36864/50176 [05:32<02:03, 107.44it/s]
 74%|███████▍  | 37120/50176 [05:35<02:02, 106.27it/s]
 74%|███████▍  | 37376/50176 [05:37<01:53, 112.41it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.37it/s]
 76%|███████▌  | 37888/50176 [05:41<01:51, 110.17it/s]
 76%|███████▌  | 38144/50176 [05:44<01:46, 112.73it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 110.93it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.31it/s]
 78%|███████▊  | 38912/50176 [05:51<01:42, 110.16it/s]
 78%|███████▊  | 39168/50176 [05:53<01:41, 108.96it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.94it/s]
 79%|███████▉  | 39680/50176 [05:58<01:37, 107.79it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.40it/s]
 80%|████████  | 40192/50176 [06:03<01:34, 105.95it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.16it/s]
 81%|████████  | 40704/50176 [06:08<01:33, 101.81it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.65it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.48it/s]
 83%|████████▎ | 41472/50176 [06:15<01:22, 106.00it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.48it/s]
 84%|████████▎ | 41984/50176 [06:20<01:15, 108.53it/s]
 84%|████████▍ | 42240/50176 [06:22<01:13, 108.38it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 110.04it/s]
 85%|████████▌ | 42752/50176 [06:27<01:07, 109.40it/s]
 86%|████████▌ | 43008/50176 [06:29<01:05, 110.14it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.61it/s]
 87%|████████▋ | 43520/50176 [06:34<01:02, 105.87it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.10it/s]
 88%|████████▊ | 44032/50176 [06:39<00:58, 105.34it/s]
 88%|████████▊ | 44288/50176 [06:41<00:54, 107.73it/s]
 89%|████████▉ | 44544/50176 [06:44<00:54, 104.03it/s]
 89%|████████▉ | 44800/50176 [06:46<00:51, 104.96it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.11it/s]
 90%|█████████ | 45312/50176 [06:51<00:45, 106.35it/s]
 91%|█████████ | 45568/50176 [06:53<00:41, 110.10it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.33it/s]
 92%|█████████▏| 46080/50176 [06:58<00:39, 104.43it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.33it/s]
 93%|█████████▎| 46592/50176 [07:03<00:33, 108.39it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.00it/s]
 94%|█████████▍| 47104/50176 [07:08<00:29, 102.48it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.37it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.77it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.13it/s]
 96%|█████████▌| 48128/50176 [07:18<00:19, 104.99it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.23it/s]
 97%|█████████▋| 48640/50176 [07:23<00:14, 105.22it/s]
 97%|█████████▋| 48896/50176 [07:25<00:12, 106.28it/s]
 98%|█████████▊| 49152/50176 [07:28<00:09, 103.84it/s]
 98%|█████████▊| 49408/50176 [07:30<00:07, 103.53it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.73it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.87it/s] 
100%|██████████| 50176/50176 [07:36<00:00, 127.51it/s]
100%|██████████| 50176/50176 [07:36<00:00, 109.90it/s]
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6031	imagenet-zeroshot-val-top5: 0.8665
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt
new file mode 100644
index 0000000000000000000000000000000000000000..a84528aa6ccff024a2baefcbc0e7bce17a43c103
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ca5d53d959906eb10f91f85b138f22dade77b9211d9810df1c698d3b8a0523f5
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..a05645fc0d0c7be698b0fcc53b9fdc96614366a5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt).
+2022-12-21,10:27:27 | INFO | Model:
+2022-12-21,10:27:27 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:27 | INFO | Params:
+2022-12-21,10:27:27 | INFO |   batch_size: 256
+2022-12-21,10:27:27 | INFO |   beta1: 0.9
+2022-12-21,10:27:27 | INFO |   beta2: 0.98
+2022-12-21,10:27:27 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/checkpoints
+2022-12-21,10:27:27 | INFO |   copy_codebase: False
+2022-12-21,10:27:27 | INFO |   csv_caption_key: title
+2022-12-21,10:27:27 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:27 | INFO |   csv_separator: 	
+2022-12-21,10:27:27 | INFO |   dataset_resampled: False
+2022-12-21,10:27:27 | INFO |   dataset_type: auto
+2022-12-21,10:27:27 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:27 | INFO |   debug: False
+2022-12-21,10:27:27 | INFO |   device: cuda:0
+2022-12-21,10:27:27 | INFO |   dist_backend: nccl
+2022-12-21,10:27:27 | INFO |   dist_url: env://
+2022-12-21,10:27:27 | INFO |   distributed: False
+2022-12-21,10:27:27 | INFO |   epochs: 32
+2022-12-21,10:27:27 | INFO |   eps: 1e-06
+2022-12-21,10:27:27 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:27 | INFO |   gather_with_grad: False
+2022-12-21,10:27:27 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:27 | INFO |   horovod: False
+2022-12-21,10:27:27 | INFO |   image_mean: None
+2022-12-21,10:27:27 | INFO |   image_std: None
+2022-12-21,10:27:27 | INFO |   imagenet_v2: None
+2022-12-21,10:27:27 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:27 | INFO |   local_loss: False
+2022-12-21,10:27:27 | INFO |   local_rank: 0
+2022-12-21,10:27:27 | INFO |   lock_image: False
+2022-12-21,10:27:27 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:27 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:27 | INFO |   log_level: 20
+2022-12-21,10:27:27 | INFO |   log_local: False
+2022-12-21,10:27:27 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/out.log
+2022-12-21,10:27:27 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:27 | INFO |   lr: 0.0005
+2022-12-21,10:27:27 | INFO |   model: ViT-B-16
+2022-12-21,10:27:27 | INFO |   name: epoch_38.pt
+2022-12-21,10:27:27 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:27 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:27 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:27 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt
+2022-12-21,10:27:27 | INFO |   pretrained_image: False
+2022-12-21,10:27:27 | INFO |   rank: 0
+2022-12-21,10:27:27 | INFO |   report_to: 
+2022-12-21,10:27:27 | INFO |   resume: None
+2022-12-21,10:27:27 | INFO |   save_frequency: 1
+2022-12-21,10:27:27 | INFO |   save_most_recent: False
+2022-12-21,10:27:27 | INFO |   seed: 0
+2022-12-21,10:27:27 | INFO |   skip_scheduler: False
+2022-12-21,10:27:27 | INFO |   tensorboard: False
+2022-12-21,10:27:27 | INFO |   tensorboard_path: 
+2022-12-21,10:27:27 | INFO |   torchscript: False
+2022-12-21,10:27:27 | INFO |   trace: False
+2022-12-21,10:27:27 | INFO |   train_data: None
+2022-12-21,10:27:27 | INFO |   train_num_samples: None
+2022-12-21,10:27:27 | INFO |   use_bn_sync: False
+2022-12-21,10:27:27 | INFO |   val_data: None
+2022-12-21,10:27:27 | INFO |   val_frequency: 1
+2022-12-21,10:27:27 | INFO |   val_num_samples: None
+2022-12-21,10:27:27 | INFO |   wandb: False
+2022-12-21,10:27:27 | INFO |   wandb_notes: 
+2022-12-21,10:27:27 | INFO |   warmup: 10000
+2022-12-21,10:27:27 | INFO |   wd: 0.2
+2022-12-21,10:27:27 | INFO |   workers: 1
+2022-12-21,10:27:27 | INFO |   world_size: 1
+2022-12-21,10:27:27 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:27 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:27 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:03,  1.65it/s]
  1%|          | 6/1000 [00:00<01:31, 10.88it/s]
  1%|          | 12/1000 [00:00<00:47, 20.80it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.55it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.65it/s]
  3%|▎         | 30/1000 [00:01<00:22, 42.22it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.38it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.43it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.73it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.55it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.33it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.94it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.30it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.78it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.06it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.31it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.52it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.78it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.92it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.81it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.84it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.80it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.84it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.92it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.82it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.46it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.53it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.53it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.49it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.48it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.43it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.24it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.13it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.22it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.43it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.43it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.42it/s]
 25%|██▌       | 252/1000 [00:04<00:13, 57.52it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.46it/s]
 26%|██▋       | 264/1000 [00:05<00:15, 48.60it/s]
 27%|██▋       | 270/1000 [00:05<00:14, 50.87it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 52.72it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 53.84it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 55.02it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.01it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.57it/s]
 31%|███       | 306/1000 [00:05<00:12, 56.95it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.36it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.53it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.56it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.48it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.62it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.60it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.78it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.62it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.68it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.77it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.90it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.76it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.46it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.55it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.83it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.92it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.91it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.92it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.94it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.05it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.04it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.83it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.95it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.92it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.85it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.01it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.92it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.72it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.78it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.72it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.80it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.74it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.83it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.87it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.95it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.89it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.97it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.98it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.88it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.89it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.93it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.85it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.65it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.84it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.85it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.80it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.92it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.85it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.85it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.96it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.96it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.05it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.86it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.80it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.96it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.93it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.02it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.89it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.87it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.96it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.00it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.05it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.95it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.00it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.00it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.97it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.95it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.96it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.98it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.95it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.04it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.89it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.01it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.97it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.01it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.04it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.84it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.84it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.89it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.87it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.84it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.86it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.95it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.09it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.13it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.04it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.09it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.03it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.03it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.98it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.04it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.91it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.02it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.01it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.98it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.96it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.01it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.01it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.03it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.06it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.01it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.08it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.15it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.86it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.90it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.79it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.64it/s]
+2022-12-21,10:27:45 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:06, 91.28it/s]
  1%|          | 512/50176 [00:05<08:36, 96.22it/s]
  2%|▏         | 768/50176 [00:07<07:50, 104.92it/s]
  2%|▏         | 1024/50176 [00:09<07:44, 105.93it/s]
  3%|▎         | 1280/50176 [00:12<07:46, 104.71it/s]
  3%|▎         | 1536/50176 [00:15<08:14, 98.37it/s] 
  4%|▎         | 1792/50176 [00:17<08:01, 100.42it/s]
  4%|▍         | 2048/50176 [00:20<07:48, 102.71it/s]
  5%|▍         | 2304/50176 [00:22<07:49, 102.06it/s]
  5%|▌         | 2560/50176 [00:25<07:40, 103.34it/s]
  6%|▌         | 2816/50176 [00:27<07:45, 101.66it/s]
  6%|▌         | 3072/50176 [00:30<07:32, 104.19it/s]
  7%|▋         | 3328/50176 [00:32<07:19, 106.58it/s]
  7%|▋         | 3584/50176 [00:34<07:10, 108.21it/s]
  8%|▊         | 3840/50176 [00:37<07:27, 103.65it/s]
  8%|▊         | 4096/50176 [00:39<07:32, 101.91it/s]
  9%|▊         | 4352/50176 [00:42<07:24, 103.14it/s]
  9%|▉         | 4608/50176 [00:44<07:08, 106.39it/s]
 10%|▉         | 4864/50176 [00:46<07:02, 107.31it/s]
 10%|█         | 5120/50176 [00:49<06:52, 109.15it/s]
 11%|█         | 5376/50176 [00:51<06:54, 107.97it/s]
 11%|█         | 5632/50176 [00:53<06:52, 107.87it/s]
 12%|█▏        | 5888/50176 [00:56<06:55, 106.56it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.53it/s]
 13%|█▎        | 6400/50176 [01:01<06:50, 106.61it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.04it/s]
 14%|█▍        | 6912/50176 [01:05<06:41, 107.83it/s]
 14%|█▍        | 7168/50176 [01:07<06:20, 112.92it/s]
 15%|█▍        | 7424/50176 [01:10<06:13, 114.35it/s]
 15%|█▌        | 7680/50176 [01:12<06:07, 115.77it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.36it/s]
 16%|█▋        | 8192/50176 [01:16<06:05, 114.74it/s]
 17%|█▋        | 8448/50176 [01:18<05:58, 116.30it/s]
 17%|█▋        | 8704/50176 [01:21<06:05, 113.60it/s]
 18%|█▊        | 8960/50176 [01:23<06:04, 113.08it/s]
 18%|█▊        | 9216/50176 [01:25<05:57, 114.73it/s]
 19%|█▉        | 9472/50176 [01:27<05:57, 114.01it/s]
 19%|█▉        | 9728/50176 [01:30<06:00, 112.05it/s]
 20%|█▉        | 9984/50176 [01:32<05:57, 112.39it/s]
 20%|██        | 10240/50176 [01:34<05:42, 116.53it/s]
 21%|██        | 10496/50176 [01:36<05:41, 116.25it/s]
 21%|██▏       | 10752/50176 [01:39<05:54, 111.34it/s]
 22%|██▏       | 11008/50176 [01:41<05:53, 110.78it/s]
 22%|██▏       | 11264/50176 [01:43<05:40, 114.21it/s]
 23%|██▎       | 11520/50176 [01:45<05:34, 115.59it/s]
 23%|██▎       | 11776/50176 [01:48<05:28, 117.06it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.43it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.51it/s]
 25%|██▌       | 12544/50176 [01:54<05:22, 116.64it/s]
 26%|██▌       | 12800/50176 [01:56<05:13, 119.24it/s]
 26%|██▌       | 13056/50176 [01:58<05:10, 119.43it/s]
 27%|██▋       | 13312/50176 [02:01<05:10, 118.59it/s]
 27%|██▋       | 13568/50176 [02:03<05:27, 111.68it/s]
 28%|██▊       | 13824/50176 [02:06<05:29, 110.27it/s]
 28%|██▊       | 14080/50176 [02:08<05:28, 109.74it/s]
 29%|██▊       | 14336/50176 [02:10<05:32, 107.72it/s]
 29%|██▉       | 14592/50176 [02:12<05:18, 111.59it/s]
 30%|██▉       | 14848/50176 [02:15<05:29, 107.38it/s]
 30%|███       | 15104/50176 [02:17<05:28, 106.85it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.79it/s]
 31%|███       | 15616/50176 [02:22<05:12, 110.48it/s]
 32%|███▏      | 15872/50176 [02:24<05:17, 108.06it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.58it/s]
 33%|███▎      | 16384/50176 [02:29<05:06, 110.09it/s]
 33%|███▎      | 16640/50176 [02:32<05:20, 104.50it/s]
 34%|███▎      | 16896/50176 [02:34<05:11, 106.99it/s]
 34%|███▍      | 17152/50176 [02:36<05:03, 108.63it/s]
 35%|███▍      | 17408/50176 [02:38<04:55, 110.71it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.48it/s]
 36%|███▌      | 17920/50176 [02:43<04:39, 115.21it/s]
 36%|███▌      | 18176/50176 [02:45<04:38, 115.05it/s]
 37%|███▋      | 18432/50176 [02:47<04:37, 114.23it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.06it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.71it/s]
 38%|███▊      | 19200/50176 [02:54<04:43, 109.36it/s]
 39%|███▉      | 19456/50176 [02:57<04:49, 106.29it/s]
 39%|███▉      | 19712/50176 [02:59<04:51, 104.53it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.13it/s]
 40%|████      | 20224/50176 [03:04<04:32, 109.96it/s]
 41%|████      | 20480/50176 [03:06<04:19, 114.23it/s]
 41%|████▏     | 20736/50176 [03:08<04:17, 114.44it/s]
 42%|████▏     | 20992/50176 [03:10<04:04, 119.41it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.88it/s]
 43%|████▎     | 21504/50176 [03:15<04:10, 114.60it/s]
 43%|████▎     | 21760/50176 [03:17<04:08, 114.56it/s]
 44%|████▍     | 22016/50176 [03:19<03:55, 119.43it/s]
 44%|████▍     | 22272/50176 [03:21<03:55, 118.55it/s]
 45%|████▍     | 22528/50176 [03:23<03:54, 117.96it/s]
 45%|████▌     | 22784/50176 [03:26<04:02, 112.88it/s]
 46%|████▌     | 23040/50176 [03:28<04:03, 111.61it/s]
 46%|████▋     | 23296/50176 [03:31<04:08, 108.37it/s]
 47%|████▋     | 23552/50176 [03:33<04:09, 106.87it/s]
 47%|████▋     | 23808/50176 [03:35<03:55, 111.84it/s]
 48%|████▊     | 24064/50176 [03:37<03:49, 113.63it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.72it/s]
 49%|████▉     | 24576/50176 [03:42<03:46, 112.80it/s]
 49%|████▉     | 24832/50176 [03:44<03:42, 114.07it/s]
 50%|█████     | 25088/50176 [03:46<03:35, 116.39it/s]
 51%|█████     | 25344/50176 [03:48<03:30, 118.15it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 114.05it/s]
 52%|█████▏    | 25856/50176 [03:53<03:28, 116.40it/s]
 52%|█████▏    | 26112/50176 [03:55<03:28, 115.67it/s]
 53%|█████▎    | 26368/50176 [03:58<03:31, 112.41it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 115.31it/s]
 54%|█████▎    | 26880/50176 [04:02<03:23, 114.22it/s]
 54%|█████▍    | 27136/50176 [04:04<03:21, 114.62it/s]
 55%|█████▍    | 27392/50176 [04:06<03:16, 116.20it/s]
 55%|█████▌    | 27648/50176 [04:08<03:03, 122.59it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.22it/s]
 56%|█████▌    | 28160/50176 [04:13<03:20, 109.88it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.36it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.74it/s]
 58%|█████▊    | 28928/50176 [04:20<03:08, 112.52it/s]
 58%|█████▊    | 29184/50176 [04:23<03:09, 110.91it/s]
 59%|█████▊    | 29440/50176 [04:25<03:15, 105.83it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.23it/s]
 60%|█████▉    | 29952/50176 [04:30<03:11, 105.69it/s]
 60%|██████    | 30208/50176 [04:32<03:06, 107.18it/s]
 61%|██████    | 30464/50176 [04:34<02:56, 111.90it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.28it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.60it/s]
 62%|██████▏   | 31232/50176 [04:41<02:45, 114.57it/s]
 63%|██████▎   | 31488/50176 [04:43<02:41, 115.96it/s]
 63%|██████▎   | 31744/50176 [04:45<02:37, 117.16it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.81it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 116.93it/s]
 65%|██████▍   | 32512/50176 [04:52<02:38, 111.20it/s]
 65%|██████▌   | 32768/50176 [04:54<02:32, 114.46it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.73it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.91it/s]
 67%|██████▋   | 33536/50176 [05:01<02:30, 110.91it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.31it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.51it/s]
 68%|██████▊   | 34304/50176 [05:08<02:16, 116.64it/s]
 69%|██████▉   | 34560/50176 [05:10<02:19, 112.00it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.70it/s]
 70%|██████▉   | 35072/50176 [05:15<02:19, 108.57it/s]
 70%|███████   | 35328/50176 [05:18<02:13, 110.86it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 110.00it/s]
 71%|███████▏  | 35840/50176 [05:22<02:12, 108.46it/s]
 72%|███████▏  | 36096/50176 [05:25<02:07, 110.13it/s]
 72%|███████▏  | 36352/50176 [05:27<02:09, 107.11it/s]
 73%|███████▎  | 36608/50176 [05:29<02:05, 108.12it/s]
 73%|███████▎  | 36864/50176 [05:32<02:04, 107.34it/s]
 74%|███████▍  | 37120/50176 [05:34<02:02, 106.33it/s]
 74%|███████▍  | 37376/50176 [05:36<01:53, 112.28it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.51it/s]
 76%|███████▌  | 37888/50176 [05:41<01:51, 110.03it/s]
 76%|███████▌  | 38144/50176 [05:43<01:46, 112.92it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 110.99it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.40it/s]
 78%|███████▊  | 38912/50176 [05:50<01:42, 110.00it/s]
 78%|███████▊  | 39168/50176 [05:53<01:41, 108.87it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.83it/s]
 79%|███████▉  | 39680/50176 [05:57<01:37, 108.08it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.25it/s]
 80%|████████  | 40192/50176 [06:02<01:34, 105.88it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.11it/s]
 81%|████████  | 40704/50176 [06:08<01:33, 101.74it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.58it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.44it/s]
 83%|████████▎ | 41472/50176 [06:15<01:22, 105.96it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.75it/s]
 84%|████████▎ | 41984/50176 [06:19<01:15, 108.45it/s]
 84%|████████▍ | 42240/50176 [06:21<01:13, 108.44it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 110.01it/s]
 85%|████████▌ | 42752/50176 [06:26<01:07, 109.59it/s]
 86%|████████▌ | 43008/50176 [06:28<01:05, 110.01it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.48it/s]
 87%|████████▋ | 43520/50176 [06:33<01:02, 105.99it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.04it/s]
 88%|████████▊ | 44032/50176 [06:38<00:58, 105.29it/s]
 88%|████████▊ | 44288/50176 [06:41<00:54, 107.69it/s]
 89%|████████▉ | 44544/50176 [06:43<00:54, 104.08it/s]
 89%|████████▉ | 44800/50176 [06:46<00:51, 104.91it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.04it/s]
 90%|█████████ | 45312/50176 [06:50<00:45, 106.23it/s]
 91%|█████████ | 45568/50176 [06:52<00:41, 110.17it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.34it/s]
 92%|█████████▏| 46080/50176 [06:58<00:39, 104.44it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.54it/s]
 93%|█████████▎| 46592/50176 [07:02<00:33, 108.35it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.18it/s]
 94%|█████████▍| 47104/50176 [07:07<00:29, 102.41it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.34it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.86it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.04it/s]
 96%|█████████▌| 48128/50176 [07:17<00:19, 105.15it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.33it/s]
 97%|█████████▋| 48640/50176 [07:22<00:14, 105.28it/s]
 97%|█████████▋| 48896/50176 [07:24<00:12, 106.12it/s]
 98%|█████████▊| 49152/50176 [07:27<00:09, 103.97it/s]
 98%|█████████▊| 49408/50176 [07:30<00:07, 103.50it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.72it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.96it/s] 
100%|██████████| 50176/50176 [07:36<00:00, 127.46it/s]
100%|██████████| 50176/50176 [07:36<00:00, 110.01it/s]
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6101	imagenet-zeroshot-val-top5: 0.8697
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt
new file mode 100644
index 0000000000000000000000000000000000000000..e6e4f30faf213505d6587262716bac2c47b9e12c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c3aab2d35933d08e75d28fdb1e0a017c646ae36be6913503ac48d3be5ef9b01
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..bc7601b78b2b41486d3bb53ffe68f8589d8e609b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_39.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:42 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:42 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:43,  1.55it/s]
  1%|          | 6/1000 [00:00<01:37, 10.23it/s]
  1%|          | 11/1000 [00:00<00:54, 18.21it/s]
  2%|▏         | 17/1000 [00:00<00:36, 27.18it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.62it/s]
  3%|▎         | 29/1000 [00:01<00:23, 40.63it/s]
  4%|▎         | 35/1000 [00:01<00:21, 45.10it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.48it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.83it/s]
  5%|▌         | 53/1000 [00:01<00:17, 52.73it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.97it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.87it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.55it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.09it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.42it/s]
  9%|▉         | 89/1000 [00:02<00:16, 56.86it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.10it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.30it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.56it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.68it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 57.70it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.65it/s]
 13%|█▎        | 131/1000 [00:02<00:15, 57.61it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.56it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.60it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.72it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.62it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.51it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.54it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.54it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.39it/s]
 19%|█▉        | 191/1000 [00:03<00:14, 57.38it/s]
 20%|█▉        | 197/1000 [00:04<00:14, 57.32it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.21it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.08it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.13it/s]
 22%|██▏       | 221/1000 [00:04<00:15, 51.39it/s]
 23%|██▎       | 227/1000 [00:04<00:14, 53.15it/s]
 23%|██▎       | 233/1000 [00:04<00:14, 54.40it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 55.12it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 55.74it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 56.25it/s]
 26%|██▌       | 257/1000 [00:05<00:13, 56.74it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 56.99it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 56.82it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.25it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.66it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.69it/s]
 30%|███       | 305/1000 [00:06<00:12, 57.51it/s]
 31%|███       | 311/1000 [00:06<00:11, 57.71it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 57.75it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 57.74it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.56it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.65it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 57.71it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 57.83it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.64it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 57.76it/s]
 36%|███▋      | 365/1000 [00:07<00:11, 57.51it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 57.72it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.61it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.44it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.55it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.70it/s]
 41%|████      | 407/1000 [00:07<00:10, 57.79it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 57.79it/s]
 42%|████▏     | 419/1000 [00:07<00:10, 57.82it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.77it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.92it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 57.92it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.73it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 57.82it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 57.79it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 57.77it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 57.90it/s]
 48%|████▊     | 479/1000 [00:09<00:09, 57.82it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 57.80it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.77it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.67it/s]
 50%|█████     | 503/1000 [00:09<00:08, 57.78it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.66it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.71it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 57.76it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 57.86it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 57.76it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 57.81it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 57.87it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.75it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.74it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 57.84it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.63it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.61it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 57.81it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 57.76it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.71it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 57.82it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.73it/s]
 61%|██████    | 611/1000 [00:11<00:06, 57.78it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 57.87it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 57.86it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 57.75it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.67it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.59it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 57.81it/s]
 65%|██████▌   | 653/1000 [00:12<00:06, 57.74it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 57.86it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 57.85it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 57.86it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 57.90it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 57.86it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.70it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.62it/s]
 70%|███████   | 701/1000 [00:12<00:05, 57.76it/s]
 71%|███████   | 707/1000 [00:12<00:05, 57.79it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 57.81it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 57.86it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 57.82it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 57.78it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 57.84it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 57.80it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 57.87it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.01it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 57.81it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 57.86it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 57.86it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 57.90it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 57.93it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.70it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.71it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.74it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.75it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 57.73it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 57.81it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 57.81it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 57.99it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.08it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 57.91it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 57.96it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 57.91it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 57.97it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 57.80it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 57.79it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 57.85it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.72it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 57.84it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 57.88it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 57.85it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 57.83it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 57.88it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.66it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 57.84it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 57.95it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 57.95it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 57.95it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 57.97it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 57.94it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.01it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.07it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.11it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.13it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.06it/s]
100%|█████████▉| 995/1000 [00:17<00:00, 57.92it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.44it/s]
+2022-12-21,10:36:00 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:26, 79.65it/s]
  1%|          | 512/50176 [00:05<09:27, 87.56it/s]
  2%|▏         | 768/50176 [00:08<08:36, 95.57it/s]
  2%|▏         | 1024/50176 [00:10<08:25, 97.16it/s]
  3%|▎         | 1280/50176 [00:13<08:19, 97.84it/s]
  3%|▎         | 1536/50176 [00:16<08:55, 90.79it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.15it/s]
  4%|▍         | 2048/50176 [00:21<08:22, 95.78it/s]
  5%|▍         | 2304/50176 [00:24<08:21, 95.38it/s]
  5%|▌         | 2560/50176 [00:26<08:02, 98.77it/s]
  6%|▌         | 2816/50176 [00:29<08:13, 95.95it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.82it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.33it/s]
  7%|▋         | 3584/50176 [00:37<07:41, 100.95it/s]
  8%|▊         | 3840/50176 [00:39<07:49, 98.77it/s] 
  8%|▊         | 4096/50176 [00:42<08:07, 94.62it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.21it/s]
  9%|▉         | 4608/50176 [00:47<07:43, 98.26it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.34it/s]
 10%|█         | 5120/50176 [00:53<07:30, 99.97it/s]
 11%|█         | 5376/50176 [00:55<07:34, 98.61it/s]
 11%|█         | 5632/50176 [00:58<07:32, 98.36it/s]
 12%|█▏        | 5888/50176 [01:00<07:29, 98.62it/s]
 12%|█▏        | 6144/50176 [01:04<07:57, 92.26it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.85it/s]
 13%|█▎        | 6656/50176 [01:09<07:20, 98.80it/s]
 14%|█▍        | 6912/50176 [01:11<07:26, 96.80it/s]
 14%|█▍        | 7168/50176 [01:14<07:21, 97.48it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.86it/s]
 15%|█▌        | 7680/50176 [01:19<07:00, 100.98it/s]
 16%|█▌        | 7936/50176 [01:21<06:49, 103.08it/s]
 16%|█▋        | 8192/50176 [01:24<06:47, 103.02it/s]
 17%|█▋        | 8448/50176 [01:26<06:42, 103.55it/s]
 17%|█▋        | 8704/50176 [01:29<06:39, 103.77it/s]
 18%|█▊        | 8960/50176 [01:31<06:50, 100.30it/s]
 18%|█▊        | 9216/50176 [01:34<06:47, 100.55it/s]
 19%|█▉        | 9472/50176 [01:36<06:40, 101.67it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.22it/s]
 20%|█▉        | 9984/50176 [01:41<06:36, 101.28it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.47it/s]
 21%|██        | 10496/50176 [01:46<06:19, 104.62it/s]
 21%|██▏       | 10752/50176 [01:49<06:24, 102.50it/s]
 22%|██▏       | 11008/50176 [01:51<06:25, 101.51it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.35it/s]
 23%|██▎       | 11520/50176 [01:56<06:17, 102.33it/s]
 23%|██▎       | 11776/50176 [01:59<06:04, 105.27it/s]
 24%|██▍       | 12032/50176 [02:01<06:05, 104.50it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.43it/s]
 25%|██▌       | 12544/50176 [02:06<06:05, 102.87it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.67it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.74it/s]
 27%|██▋       | 13312/50176 [02:13<05:50, 105.12it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.86it/s] 
 28%|██▊       | 13824/50176 [02:19<06:19, 95.67it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.68it/s]
 29%|██▊       | 14336/50176 [02:25<06:16, 95.12it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.54it/s]
 30%|██▉       | 14848/50176 [02:30<06:16, 93.74it/s]
 30%|███       | 15104/50176 [02:33<06:07, 95.51it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.44it/s]
 31%|███       | 15616/50176 [02:38<05:47, 99.44it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.76it/s]
 32%|███▏      | 16128/50176 [02:43<05:42, 99.35it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.59it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.14it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.50it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.23it/s]
 35%|███▍      | 17408/50176 [02:56<05:30, 99.09it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.47it/s]
 36%|███▌      | 17920/50176 [03:00<05:12, 103.22it/s]
 36%|███▌      | 18176/50176 [03:03<05:11, 102.86it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.58it/s]
 37%|███▋      | 18688/50176 [03:08<05:12, 100.90it/s]
 38%|███▊      | 18944/50176 [03:11<05:11, 100.28it/s]
 38%|███▊      | 19200/50176 [03:13<05:19, 97.00it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.78it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.38it/s]
 40%|███▉      | 19968/50176 [03:22<05:18, 94.91it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.68it/s]
 41%|████      | 20480/50176 [03:26<04:55, 100.64it/s]
 41%|████▏     | 20736/50176 [03:29<04:51, 101.11it/s]
 42%|████▏     | 20992/50176 [03:31<04:38, 104.95it/s]
 42%|████▏     | 21248/50176 [03:34<04:41, 102.81it/s]
 43%|████▎     | 21504/50176 [03:36<04:35, 103.90it/s]
 43%|████▎     | 21760/50176 [03:39<04:32, 104.34it/s]
 44%|████▍     | 22016/50176 [03:41<04:23, 106.77it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.82it/s]
 45%|████▍     | 22528/50176 [03:46<04:18, 106.91it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.44it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.50it/s]
 46%|████▋     | 23296/50176 [03:54<04:30, 99.29it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.36it/s]
 47%|████▋     | 23808/50176 [03:59<04:17, 102.42it/s]
 48%|████▊     | 24064/50176 [04:01<04:13, 102.88it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.17it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 102.89it/s]
 49%|████▉     | 24832/50176 [04:08<03:59, 105.66it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 105.98it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.89it/s]
 51%|█████     | 25600/50176 [04:16<03:53, 105.20it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.33it/s]
 52%|█████▏    | 26112/50176 [04:21<03:50, 104.61it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.62it/s]
 53%|█████▎    | 26624/50176 [04:25<03:40, 106.58it/s]
 54%|█████▎    | 26880/50176 [04:28<03:44, 103.59it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.53it/s]
 55%|█████▍    | 27392/50176 [04:33<03:36, 105.16it/s]
 55%|█████▌    | 27648/50176 [04:35<03:25, 109.80it/s]
 56%|█████▌    | 27904/50176 [04:38<03:34, 103.99it/s]
 56%|█████▌    | 28160/50176 [04:40<03:34, 102.77it/s]
 57%|█████▋    | 28416/50176 [04:43<03:32, 102.32it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.79it/s]
 58%|█████▊    | 28928/50176 [04:48<03:28, 102.07it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.53it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.86it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.96it/s]
 60%|█████▉    | 29952/50176 [04:59<03:34, 94.25it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.09it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.82it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.46it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.39it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.70it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 106.06it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.31it/s]
 64%|██████▍   | 32000/50176 [05:18<02:49, 107.33it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.60it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.44it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.72it/s]
 66%|██████▌   | 33024/50176 [05:28<02:47, 102.17it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.52it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.16it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.34it/s]
 68%|██████▊   | 34048/50176 [05:38<02:31, 106.13it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.57it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.34it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.58it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.95it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.46it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.88it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.22it/s]
 72%|███████▏  | 36096/50176 [05:57<02:16, 103.47it/s]
 72%|███████▏  | 36352/50176 [06:00<02:15, 101.69it/s]
 73%|███████▎  | 36608/50176 [06:03<02:14, 100.83it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.44it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.62it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.27it/s]
 75%|███████▌  | 37632/50176 [06:12<01:58, 106.30it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.23it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.59it/s]
 77%|███████▋  | 38400/50176 [06:20<01:53, 103.79it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.39it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.08it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.73it/s]
 79%|███████▊  | 39424/50176 [06:30<01:45, 102.15it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.71it/s]
 80%|███████▉  | 39936/50176 [06:35<01:39, 103.18it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.82it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.58it/s] 
 81%|████████  | 40704/50176 [06:43<01:36, 97.95it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.50it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.81it/s]
 83%|████████▎ | 41472/50176 [06:50<01:24, 102.78it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 105.00it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.33it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.69it/s]
 85%|████████▍ | 42496/50176 [06:59<01:12, 106.64it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 103.98it/s]
 86%|████████▌ | 43008/50176 [07:04<01:09, 103.52it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.38it/s]
 87%|████████▋ | 43520/50176 [07:10<01:06, 100.63it/s]
 87%|████████▋ | 43776/50176 [07:12<01:04, 99.99it/s] 
 88%|████████▊ | 44032/50176 [07:15<01:00, 102.27it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.78it/s]
 89%|████████▉ | 44544/50176 [07:20<00:55, 101.38it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.18it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.24it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.08it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.47it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 104.08it/s]
 92%|█████████▏| 46080/50176 [07:35<00:40, 101.43it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.34it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.48it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 102.07it/s]
 94%|█████████▍| 47104/50176 [07:45<00:31, 98.48it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.81it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.00it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.25it/s]
 96%|█████████▌| 48128/50176 [07:55<00:19, 104.06it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.31it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.15it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.35it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.47it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.64it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.51it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 95.80it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 126.78it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.61it/s]
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6116	imagenet-zeroshot-val-top5: 0.8690
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6011349a5b9f04a37bb347520d99b8725677d444
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:06b4f80550f7bbf91afae20ddb1cb133e3cc674344e2a24239d12b5e0321cd96
+size 1795822369
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6629098c024e9cab1434bc4aa0d0c6c423b83f2a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:34 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_4.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:43 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:25,  1.46it/s]
  1%|          | 6/1000 [00:00<01:41,  9.81it/s]
  1%|          | 12/1000 [00:00<00:52, 19.00it/s]
  2%|▏         | 18/1000 [00:01<00:36, 27.23it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.41it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.26it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.78it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.19it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.72it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.48it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.82it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.75it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.43it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.83it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.35it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.67it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.06it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.25it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.51it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.65it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.64it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.59it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.62it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.57it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.61it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.53it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.39it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.38it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.31it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.23it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.18it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.14it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.97it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.98it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.93it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.16it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.14it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.07it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.14it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.24it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.32it/s]
 26%|██▋       | 264/1000 [00:05<00:14, 51.32it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 52.84it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 54.09it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 55.15it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 55.90it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.55it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.87it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.08it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.35it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.45it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.45it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.34it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.47it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.63it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.73it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.52it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.48it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.59it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.69it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.59it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.26it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.30it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.40it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.55it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.64it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.61it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.62it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.67it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.81it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.81it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.60it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.67it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.73it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.71it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.65it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.79it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.70it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.66it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.67it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.56it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.62it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.52it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.59it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.69it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.64it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.76it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.77it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.66it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.67it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.72it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.59it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.52it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.65it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.64it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.55it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.66it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.59it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.60it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.69it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.67it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.77it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.59it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.53it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.67it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.62it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.71it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.74it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.65it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.70it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.71it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.75it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.64it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.72it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.72it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.68it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.65it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.62it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.65it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.69it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.67it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.75it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.82it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.67it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.76it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.63it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.66it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.71it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.52it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.52it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.58it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.56it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.55it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.56it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.66it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.80it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.84it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.76it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.81it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.76it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.78it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.71it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.76it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.76it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.60it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.70it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.71it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.66it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.68it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.68it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.68it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.78it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.82it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.81it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.83it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.86it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.80it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.87it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.94it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.97it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.85it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.17it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 55.83it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.17it/s]
+2022-12-21,10:36:01 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:14, 89.99it/s]
  1%|          | 512/50176 [00:05<09:01, 91.68it/s]
  2%|▏         | 768/50176 [00:07<08:24, 97.99it/s]
  2%|▏         | 1024/50176 [00:10<08:19, 98.39it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.47it/s]
  3%|▎         | 1536/50176 [00:16<08:52, 91.29it/s]
  4%|▎         | 1792/50176 [00:18<08:37, 93.41it/s]
  4%|▍         | 2048/50176 [00:21<08:21, 95.94it/s]
  5%|▍         | 2304/50176 [00:24<08:21, 95.46it/s]
  5%|▌         | 2560/50176 [00:26<08:02, 98.66it/s]
  6%|▌         | 2816/50176 [00:29<08:12, 96.07it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.90it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.26it/s]
  7%|▋         | 3584/50176 [00:36<07:41, 101.01it/s]
  8%|▊         | 3840/50176 [00:39<07:49, 98.65it/s] 
  8%|▊         | 4096/50176 [00:42<08:07, 94.58it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.20it/s]
  9%|▉         | 4608/50176 [00:47<07:44, 98.21it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.36it/s]
 10%|█         | 5120/50176 [00:52<07:30, 100.10it/s]
 11%|█         | 5376/50176 [00:55<07:34, 98.58it/s] 
 11%|█         | 5632/50176 [00:58<07:33, 98.32it/s]
 12%|█▏        | 5888/50176 [01:00<07:29, 98.63it/s]
 12%|█▏        | 6144/50176 [01:03<07:57, 92.26it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.84it/s]
 13%|█▎        | 6656/50176 [01:08<07:20, 98.73it/s]
 14%|█▍        | 6912/50176 [01:11<07:27, 96.75it/s]
 14%|█▍        | 7168/50176 [01:14<07:20, 97.53it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.81it/s]
 15%|█▌        | 7680/50176 [01:19<07:01, 100.92it/s]
 16%|█▌        | 7936/50176 [01:21<06:49, 103.13it/s]
 16%|█▋        | 8192/50176 [01:23<06:47, 103.15it/s]
 17%|█▋        | 8448/50176 [01:26<06:43, 103.51it/s]
 17%|█▋        | 8704/50176 [01:28<06:39, 103.71it/s]
 18%|█▊        | 8960/50176 [01:31<06:51, 100.27it/s]
 18%|█▊        | 9216/50176 [01:34<06:47, 100.56it/s]
 19%|█▉        | 9472/50176 [01:36<06:39, 101.85it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.15it/s]
 20%|█▉        | 9984/50176 [01:41<06:36, 101.39it/s]
 20%|██        | 10240/50176 [01:43<06:25, 103.57it/s]
 21%|██        | 10496/50176 [01:46<06:19, 104.54it/s]
 21%|██▏       | 10752/50176 [01:48<06:25, 102.31it/s]
 22%|██▏       | 11008/50176 [01:51<06:26, 101.40it/s]
 22%|██▏       | 11264/50176 [01:53<06:20, 102.20it/s]
 23%|██▎       | 11520/50176 [01:56<06:16, 102.57it/s]
 23%|██▎       | 11776/50176 [01:58<06:04, 105.44it/s]
 24%|██▍       | 12032/50176 [02:01<06:05, 104.49it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.52it/s]
 25%|██▌       | 12544/50176 [02:06<06:05, 102.88it/s]
 26%|██▌       | 12800/50176 [02:08<06:07, 101.79it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.76it/s]
 27%|██▋       | 13312/50176 [02:13<05:51, 104.98it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.83it/s] 
 28%|██▊       | 13824/50176 [02:19<06:20, 95.63it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.70it/s]
 29%|██▊       | 14336/50176 [02:24<06:16, 95.19it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.64it/s]
 30%|██▉       | 14848/50176 [02:30<06:17, 93.60it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.52it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.43it/s]
 31%|███       | 15616/50176 [02:37<05:47, 99.51it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.76it/s]
 32%|███▏      | 16128/50176 [02:42<05:42, 99.39it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.64it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.09it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.44it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.36it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.12it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.57it/s]
 36%|███▌      | 17920/50176 [03:00<05:11, 103.40it/s]
 36%|███▌      | 18176/50176 [03:03<05:11, 102.74it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.70it/s]
 37%|███▋      | 18688/50176 [03:08<05:11, 101.16it/s]
 38%|███▊      | 18944/50176 [03:10<05:11, 100.12it/s]
 38%|███▊      | 19200/50176 [03:13<05:20, 96.65it/s] 
 39%|███▉      | 19456/50176 [03:16<05:21, 95.52it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.45it/s]
 40%|███▉      | 19968/50176 [03:21<05:18, 94.80it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.71it/s]
 41%|████      | 20480/50176 [03:26<04:55, 100.62it/s]
 41%|████▏     | 20736/50176 [03:29<04:51, 101.01it/s]
 42%|████▏     | 20992/50176 [03:31<04:38, 104.90it/s]
 42%|████▏     | 21248/50176 [03:33<04:41, 102.80it/s]
 43%|████▎     | 21504/50176 [03:36<04:35, 103.90it/s]
 43%|████▎     | 21760/50176 [03:38<04:32, 104.36it/s]
 44%|████▍     | 22016/50176 [03:41<04:23, 106.69it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.74it/s]
 45%|████▍     | 22528/50176 [03:45<04:18, 106.84it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.55it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.45it/s]
 46%|████▋     | 23296/50176 [03:53<04:30, 99.23it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.48it/s]
 47%|████▋     | 23808/50176 [03:58<04:17, 102.59it/s]
 48%|████▊     | 24064/50176 [04:01<04:13, 102.87it/s]
 48%|████▊     | 24320/50176 [04:03<04:15, 101.17it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 102.98it/s]
 49%|████▉     | 24832/50176 [04:08<04:00, 105.59it/s]
 50%|█████     | 25088/50176 [04:10<03:56, 105.98it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.70it/s]
 51%|█████     | 25600/50176 [04:15<03:53, 105.42it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.31it/s]
 52%|█████▏    | 26112/50176 [04:20<03:49, 104.63it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.72it/s]
 53%|█████▎    | 26624/50176 [04:25<03:41, 106.53it/s]
 54%|█████▎    | 26880/50176 [04:28<03:45, 103.40it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.74it/s]
 55%|█████▍    | 27392/50176 [04:32<03:36, 105.15it/s]
 55%|█████▌    | 27648/50176 [04:34<03:24, 109.94it/s]
 56%|█████▌    | 27904/50176 [04:37<03:34, 103.70it/s]
 56%|█████▌    | 28160/50176 [04:40<03:34, 102.72it/s]
 57%|█████▋    | 28416/50176 [04:42<03:32, 102.33it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.79it/s]
 58%|█████▊    | 28928/50176 [04:47<03:27, 102.21it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.53it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.83it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.98it/s]
 60%|█████▉    | 29952/50176 [04:58<03:34, 94.24it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.19it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.86it/s]
 61%|██████    | 30720/50176 [05:06<03:12, 101.31it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.52it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.65it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 105.96it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.22it/s]
 64%|██████▍   | 32000/50176 [05:18<02:48, 107.64it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.65it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.40it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.72it/s]
 66%|██████▌   | 33024/50176 [05:28<02:47, 102.10it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.66it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.17it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.28it/s]
 68%|██████▊   | 34048/50176 [05:37<02:31, 106.16it/s]
 68%|██████▊   | 34304/50176 [05:39<02:24, 109.54it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.28it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.69it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.69it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.60it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.90it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.04it/s]
 72%|███████▏  | 36096/50176 [05:57<02:15, 103.60it/s]
 72%|███████▏  | 36352/50176 [06:00<02:16, 101.64it/s]
 73%|███████▎  | 36608/50176 [06:02<02:14, 100.79it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.58it/s] 
 74%|███████▍  | 37120/50176 [06:07<02:11, 99.52it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.18it/s]
 75%|███████▌  | 37632/50176 [06:12<01:57, 106.36it/s]
 76%|███████▌  | 37888/50176 [06:14<01:54, 107.20it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.52it/s]
 77%|███████▋  | 38400/50176 [06:19<01:53, 103.79it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.39it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 104.06it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.66it/s]
 79%|███████▊  | 39424/50176 [06:29<01:45, 102.27it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.71it/s]
 80%|███████▉  | 39936/50176 [06:34<01:39, 103.24it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.87it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.67it/s] 
 81%|████████  | 40704/50176 [06:42<01:36, 97.77it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.47it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.92it/s]
 83%|████████▎ | 41472/50176 [06:49<01:24, 102.96it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 104.99it/s]
 84%|████████▎ | 41984/50176 [06:54<01:19, 103.55it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.69it/s]
 85%|████████▍ | 42496/50176 [06:59<01:11, 106.86it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 104.18it/s]
 86%|████████▌ | 43008/50176 [07:04<01:09, 103.80it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.51it/s]
 87%|████████▋ | 43520/50176 [07:09<01:06, 100.56it/s]
 87%|████████▋ | 43776/50176 [07:12<01:03, 100.24it/s]
 88%|████████▊ | 44032/50176 [07:14<00:59, 102.51it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.81it/s]
 89%|████████▉ | 44544/50176 [07:19<00:55, 101.44it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.13it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.20it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.18it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.31it/s]
 91%|█████████▏| 45824/50176 [07:31<00:41, 104.23it/s]
 92%|█████████▏| 46080/50176 [07:34<00:40, 101.41it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.46it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.44it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 101.99it/s]
 94%|█████████▍| 47104/50176 [07:44<00:31, 98.29it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.73it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.05it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.49it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 104.17it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.36it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.28it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.27it/s]
 98%|█████████▊| 49152/50176 [08:04<00:10, 100.42it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.45it/s]
 99%|█████████▉| 49664/50176 [08:09<00:05, 100.47it/s]
 99%|█████████▉| 49920/50176 [08:12<00:02, 95.96it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 126.84it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.68it/s]
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4249	imagenet-zeroshot-val-top5: 0.7049
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt
new file mode 100644
index 0000000000000000000000000000000000000000..be48328fd23d4c7f2402dae5f5171bb00d0f3dd2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba9a68c4f0ca3ff840ed612e4a1e6059d744d999ff34863bfe5ef58c38172c21
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8ee1ff6f7eb8318544f1b49f6ad265bef18af939
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_40.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:43 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:41,  1.56it/s]
  1%|          | 6/1000 [00:00<01:35, 10.38it/s]
  1%|          | 12/1000 [00:00<00:49, 20.12it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.69it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.86it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.43it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.77it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.89it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.35it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.92it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.25it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.19it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.85it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.33it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.86it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.16it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.52it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.68it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.94it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.85it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.85it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.92it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.99it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.10it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.97it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.84it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.85it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.78it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.70it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.68it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.40it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.30it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.99it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.30it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.37it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.45it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.57it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.70it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.77it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.60it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.84it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.97it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.18it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.16it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.14it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.27it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.26it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.15it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.02it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.13it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.00it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.82it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.88it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.98it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.08it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.99it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.67it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.87it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.03it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.11it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.12it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.13it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.27it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.26it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.05it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.18it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.15it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.11it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.26it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.18it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.16it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.13it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.04it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.08it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.11it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.22it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.18it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.05it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.01it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.15it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.13it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.17it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.20it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.29it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.10it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.04it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.20it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.14it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.24it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.30it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.23it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.28it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.29it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.17it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.25it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.22it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.16it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.18it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.19it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.27it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.34it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.19it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.27it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.23it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.28it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.32it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.08it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.09it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.14it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.11it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.86it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.94it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.08it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.31it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.29it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.23it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.24it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.19it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.24it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.23it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.12it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.22it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.22it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.17it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.20it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.21it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.21it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.34it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.34it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.34it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.37it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.30it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.37it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.44it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.45it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.40it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.20it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.91it/s]
+2022-12-21,10:36:01 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:05, 82.39it/s]
  1%|          | 512/50176 [00:05<09:20, 88.65it/s]
  2%|▏         | 768/50176 [00:08<08:37, 95.42it/s]
  2%|▏         | 1024/50176 [00:10<08:26, 97.05it/s]
  3%|▎         | 1280/50176 [00:13<08:19, 97.84it/s]
  3%|▎         | 1536/50176 [00:16<08:54, 90.93it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.09it/s]
  4%|▍         | 2048/50176 [00:21<08:22, 95.70it/s]
  5%|▍         | 2304/50176 [00:24<08:23, 95.12it/s]
  5%|▌         | 2560/50176 [00:26<08:02, 98.79it/s]
  6%|▌         | 2816/50176 [00:29<08:12, 96.17it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.92it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.42it/s]
  7%|▋         | 3584/50176 [00:37<07:41, 100.89it/s]
  8%|▊         | 3840/50176 [00:39<07:49, 98.72it/s] 
  8%|▊         | 4096/50176 [00:42<08:06, 94.69it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.22it/s]
  9%|▉         | 4608/50176 [00:47<07:44, 98.14it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.42it/s]
 10%|█         | 5120/50176 [00:53<07:31, 99.70it/s]
 11%|█         | 5376/50176 [00:55<07:33, 98.76it/s]
 11%|█         | 5632/50176 [00:58<07:32, 98.37it/s]
 12%|█▏        | 5888/50176 [01:00<07:28, 98.71it/s]
 12%|█▏        | 6144/50176 [01:04<07:56, 92.35it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.90it/s]
 13%|█▎        | 6656/50176 [01:08<07:21, 98.68it/s]
 14%|█▍        | 6912/50176 [01:11<07:26, 96.82it/s]
 14%|█▍        | 7168/50176 [01:14<07:21, 97.41it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.79it/s]
 15%|█▌        | 7680/50176 [01:19<07:00, 100.96it/s]
 16%|█▌        | 7936/50176 [01:21<06:49, 103.07it/s]
 16%|█▋        | 8192/50176 [01:24<06:47, 103.11it/s]
 17%|█▋        | 8448/50176 [01:26<06:43, 103.43it/s]
 17%|█▋        | 8704/50176 [01:28<06:39, 103.69it/s]
 18%|█▊        | 8960/50176 [01:31<06:51, 100.17it/s]
 18%|█▊        | 9216/50176 [01:34<06:46, 100.65it/s]
 19%|█▉        | 9472/50176 [01:36<06:39, 101.78it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.27it/s]
 20%|█▉        | 9984/50176 [01:41<06:38, 100.92it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.59it/s]
 21%|██        | 10496/50176 [01:46<06:18, 104.70it/s]
 21%|██▏       | 10752/50176 [01:49<06:24, 102.40it/s]
 22%|██▏       | 11008/50176 [01:51<06:25, 101.60it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.15it/s]
 23%|██▎       | 11520/50176 [01:56<06:16, 102.60it/s]
 23%|██▎       | 11776/50176 [01:58<06:05, 105.05it/s]
 24%|██▍       | 12032/50176 [02:01<06:06, 104.20it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.27it/s]
 25%|██▌       | 12544/50176 [02:06<06:06, 102.78it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.66it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.77it/s]
 27%|██▋       | 13312/50176 [02:13<05:51, 104.94it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.78it/s] 
 28%|██▊       | 13824/50176 [02:19<06:20, 95.63it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.74it/s]
 29%|██▊       | 14336/50176 [02:25<06:16, 95.14it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.62it/s]
 30%|██▉       | 14848/50176 [02:30<06:17, 93.66it/s]
 30%|███       | 15104/50176 [02:33<06:07, 95.56it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.35it/s]
 31%|███       | 15616/50176 [02:38<05:47, 99.45it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.77it/s]
 32%|███▏      | 16128/50176 [02:43<05:42, 99.32it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.67it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.06it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.54it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.26it/s]
 35%|███▍      | 17408/50176 [02:56<05:30, 99.03it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.49it/s]
 36%|███▌      | 17920/50176 [03:00<05:12, 103.38it/s]
 36%|███▌      | 18176/50176 [03:03<05:11, 102.77it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.62it/s]
 37%|███▋      | 18688/50176 [03:08<05:11, 100.95it/s]
 38%|███▊      | 18944/50176 [03:11<05:11, 100.27it/s]
 38%|███▊      | 19200/50176 [03:13<05:19, 96.86it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.84it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.36it/s]
 40%|███▉      | 19968/50176 [03:22<05:17, 95.04it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.79it/s]
 41%|████      | 20480/50176 [03:26<04:54, 100.92it/s]
 41%|████▏     | 20736/50176 [03:29<04:51, 101.04it/s]
 42%|████▏     | 20992/50176 [03:31<04:38, 104.86it/s]
 42%|████▏     | 21248/50176 [03:34<04:41, 102.84it/s]
 43%|████▎     | 21504/50176 [03:36<04:36, 103.78it/s]
 43%|████▎     | 21760/50176 [03:39<04:32, 104.28it/s]
 44%|████▍     | 22016/50176 [03:41<04:24, 106.63it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.87it/s]
 45%|████▍     | 22528/50176 [03:46<04:18, 107.01it/s]
 45%|████▌     | 22784/50176 [03:48<04:25, 103.34it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.71it/s]
 46%|████▋     | 23296/50176 [03:54<04:30, 99.29it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.27it/s]
 47%|████▋     | 23808/50176 [03:59<04:17, 102.42it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.75it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.10it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 102.90it/s]
 49%|████▉     | 24832/50176 [04:08<04:00, 105.57it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 105.94it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.93it/s]
 51%|█████     | 25600/50176 [04:16<03:53, 105.24it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.45it/s]
 52%|█████▏    | 26112/50176 [04:20<03:50, 104.61it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.61it/s]
 53%|█████▎    | 26624/50176 [04:25<03:41, 106.46it/s]
 54%|█████▎    | 26880/50176 [04:28<03:44, 103.64it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.65it/s]
 55%|█████▍    | 27392/50176 [04:33<03:36, 105.02it/s]
 55%|█████▌    | 27648/50176 [04:35<03:24, 109.91it/s]
 56%|█████▌    | 27904/50176 [04:38<03:33, 104.16it/s]
 56%|█████▌    | 28160/50176 [04:40<03:34, 102.73it/s]
 57%|█████▋    | 28416/50176 [04:43<03:32, 102.41it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.84it/s]
 58%|█████▊    | 28928/50176 [04:48<03:28, 102.06it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.43it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.81it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.93it/s]
 60%|█████▉    | 29952/50176 [04:59<03:34, 94.32it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.12it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.97it/s]
 61%|██████    | 30720/50176 [05:06<03:12, 101.24it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.54it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.58it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 106.01it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.14it/s]
 64%|██████▍   | 32000/50176 [05:18<02:49, 107.40it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.65it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.38it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.72it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 102.09it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.58it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.17it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.46it/s]
 68%|██████▊   | 34048/50176 [05:38<02:31, 106.12it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.61it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.51it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.64it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.80it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.48it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.82it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.17it/s]
 72%|███████▏  | 36096/50176 [05:57<02:16, 103.50it/s]
 72%|███████▏  | 36352/50176 [06:00<02:16, 101.54it/s]
 73%|███████▎  | 36608/50176 [06:03<02:14, 100.97it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.50it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.52it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.35it/s]
 75%|███████▌  | 37632/50176 [06:12<01:57, 106.32it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.13it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.58it/s]
 77%|███████▋  | 38400/50176 [06:20<01:53, 103.71it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.34it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 104.05it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.75it/s]
 79%|███████▊  | 39424/50176 [06:30<01:45, 102.22it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.69it/s]
 80%|███████▉  | 39936/50176 [06:35<01:39, 103.24it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.80it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.65it/s] 
 81%|████████  | 40704/50176 [06:43<01:36, 97.91it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.41it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.77it/s]
 83%|████████▎ | 41472/50176 [06:50<01:24, 102.86it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 104.94it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.43it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.68it/s]
 85%|████████▍ | 42496/50176 [06:59<01:11, 106.89it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 103.87it/s]
 86%|████████▌ | 43008/50176 [07:04<01:08, 103.89it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.53it/s]
 87%|████████▋ | 43520/50176 [07:10<01:06, 100.83it/s]
 87%|████████▋ | 43776/50176 [07:12<01:03, 100.27it/s]
 88%|████████▊ | 44032/50176 [07:15<00:59, 102.61it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.69it/s]
 89%|████████▉ | 44544/50176 [07:20<00:55, 101.60it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.27it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.21it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.04it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.57it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 103.94it/s]
 92%|█████████▏| 46080/50176 [07:34<00:40, 101.43it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.44it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.59it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 102.09it/s]
 94%|█████████▍| 47104/50176 [07:45<00:31, 98.26it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.73it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.07it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.52it/s]
 96%|█████████▌| 48128/50176 [07:55<00:19, 104.07it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.29it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.20it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.32it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.47it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.64it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.48it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 95.90it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 126.77it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.63it/s]
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6105	imagenet-zeroshot-val-top5: 0.8693
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt
new file mode 100644
index 0000000000000000000000000000000000000000..635ec71ca4fe15ea12e83d2451e06795bb6497ea
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a3268eefaa5e5c3587dc18f17d4a4b67698c4ea52447867a4379dc9df44f716
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5480a31f78b6d950d0f9f886369cb760c1903d17
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_41.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:43 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:43,  1.42it/s]
  1%|          | 6/1000 [00:00<01:43,  9.61it/s]
  1%|          | 12/1000 [00:00<00:51, 19.13it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.71it/s]
  2%|▏         | 24/1000 [00:01<00:27, 34.93it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.79it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.26it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.65it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.20it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.68it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.05it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.10it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.84it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.29it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.81it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.14it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.53it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.69it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.95it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.08it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.05it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.97it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.99it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.96it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.02it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.10it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.00it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.86it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.88it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.82it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.76it/s]
 19%|█▉        | 192/1000 [00:04<00:13, 57.74it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.67it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.45it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.33it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.42it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.64it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.56it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.61it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.71it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.82it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 52.59it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 53.91it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 55.04it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 55.98it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 56.64it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.25it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.50it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.69it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.94it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.00it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.98it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.85it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.88it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.01it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.11it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.93it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.05it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.15it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.06it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.75it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.81it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.92it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.06it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.13it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.12it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.14it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.17it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.28it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.03it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.11it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.07it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.22it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.12it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.10it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.02it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.06it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.05it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.10it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.20it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.14it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.24it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.21it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.10it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.05it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.00it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.15it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.15it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.16it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.09it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.19it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.29it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.79it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.82it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.03it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.04it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.22it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.23it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.27it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.16it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.24it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.22it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.20it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.16it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.16it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.23it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.32it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.17it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.28it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.22it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.26it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.28it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.06it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.05it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.00it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.02it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.13it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.32it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.29it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.23it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.19it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.89it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.05it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.11it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.02it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.17it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.13it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.23it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.28it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.31it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.34it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.40it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.42it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.38it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.02it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.60it/s]
+2022-12-21,10:36:01 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<09:50, 84.56it/s]
  1%|          | 512/50176 [00:05<09:13, 89.78it/s]
  2%|▏         | 768/50176 [00:08<08:32, 96.38it/s]
  2%|▏         | 1024/50176 [00:10<08:23, 97.57it/s]
  3%|▎         | 1280/50176 [00:13<08:19, 97.98it/s]
  3%|▎         | 1536/50176 [00:16<08:54, 90.93it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.17it/s]
  4%|▍         | 2048/50176 [00:21<08:22, 95.71it/s]
  5%|▍         | 2304/50176 [00:24<08:21, 95.38it/s]
  5%|▌         | 2560/50176 [00:26<08:01, 98.82it/s]
  6%|▌         | 2816/50176 [00:29<08:12, 96.16it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.80it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.34it/s]
  7%|▋         | 3584/50176 [00:37<07:41, 100.88it/s]
  8%|▊         | 3840/50176 [00:39<07:48, 98.80it/s] 
  8%|▊         | 4096/50176 [00:42<08:06, 94.62it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.26it/s]
  9%|▉         | 4608/50176 [00:47<07:43, 98.23it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.32it/s]
 10%|█         | 5120/50176 [00:52<07:30, 100.05it/s]
 11%|█         | 5376/50176 [00:55<07:34, 98.59it/s] 
 11%|█         | 5632/50176 [00:58<07:33, 98.25it/s]
 12%|█▏        | 5888/50176 [01:00<07:28, 98.78it/s]
 12%|█▏        | 6144/50176 [01:03<07:57, 92.25it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.85it/s]
 13%|█▎        | 6656/50176 [01:08<07:20, 98.75it/s]
 14%|█▍        | 6912/50176 [01:11<07:27, 96.75it/s]
 14%|█▍        | 7168/50176 [01:14<07:21, 97.39it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.92it/s]
 15%|█▌        | 7680/50176 [01:19<07:00, 101.00it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.98it/s]
 16%|█▋        | 8192/50176 [01:23<06:46, 103.20it/s]
 17%|█▋        | 8448/50176 [01:26<06:43, 103.45it/s]
 17%|█▋        | 8704/50176 [01:28<06:39, 103.68it/s]
 18%|█▊        | 8960/50176 [01:31<06:50, 100.36it/s]
 18%|█▊        | 9216/50176 [01:34<06:47, 100.53it/s]
 19%|█▉        | 9472/50176 [01:36<06:40, 101.71it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.17it/s]
 20%|█▉        | 9984/50176 [01:41<06:36, 101.36it/s]
 20%|██        | 10240/50176 [01:44<06:26, 103.44it/s]
 21%|██        | 10496/50176 [01:46<06:19, 104.53it/s]
 21%|██▏       | 10752/50176 [01:49<06:25, 102.33it/s]
 22%|██▏       | 11008/50176 [01:51<06:25, 101.49it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.19it/s]
 23%|██▎       | 11520/50176 [01:56<06:16, 102.54it/s]
 23%|██▎       | 11776/50176 [01:58<06:04, 105.35it/s]
 24%|██▍       | 12032/50176 [02:01<06:05, 104.34it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.45it/s]
 25%|██▌       | 12544/50176 [02:06<06:05, 102.84it/s]
 26%|██▌       | 12800/50176 [02:08<06:07, 101.79it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.73it/s]
 27%|██▋       | 13312/50176 [02:13<05:51, 104.95it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.87it/s] 
 28%|██▊       | 13824/50176 [02:19<06:20, 95.62it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.68it/s]
 29%|██▊       | 14336/50176 [02:24<06:16, 95.24it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.50it/s]
 30%|██▉       | 14848/50176 [02:30<06:17, 93.69it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.54it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.35it/s]
 31%|███       | 15616/50176 [02:37<05:47, 99.48it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.82it/s]
 32%|███▏      | 16128/50176 [02:43<05:42, 99.32it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.67it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.08it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.47it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.37it/s]
 35%|███▍      | 17408/50176 [02:56<05:30, 99.13it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.53it/s]
 36%|███▌      | 17920/50176 [03:00<05:12, 103.23it/s]
 36%|███▌      | 18176/50176 [03:03<05:10, 102.90it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.60it/s]
 37%|███▋      | 18688/50176 [03:08<05:11, 100.95it/s]
 38%|███▊      | 18944/50176 [03:10<05:11, 100.17it/s]
 38%|███▊      | 19200/50176 [03:13<05:19, 96.81it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.77it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.43it/s]
 40%|███▉      | 19968/50176 [03:22<05:18, 94.92it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.72it/s]
 41%|████      | 20480/50176 [03:26<04:54, 100.68it/s]
 41%|████▏     | 20736/50176 [03:29<04:50, 101.17it/s]
 42%|████▏     | 20992/50176 [03:31<04:37, 105.01it/s]
 42%|████▏     | 21248/50176 [03:34<04:41, 102.86it/s]
 43%|████▎     | 21504/50176 [03:36<04:36, 103.86it/s]
 43%|████▎     | 21760/50176 [03:38<04:32, 104.34it/s]
 44%|████▍     | 22016/50176 [03:41<04:24, 106.57it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.87it/s]
 45%|████▍     | 22528/50176 [03:46<04:18, 106.89it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.43it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.44it/s]
 46%|████▋     | 23296/50176 [03:53<04:30, 99.19it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.46it/s]
 47%|████▋     | 23808/50176 [03:58<04:17, 102.53it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.76it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.09it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 103.01it/s]
 49%|████▉     | 24832/50176 [04:08<04:00, 105.52it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 105.92it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.81it/s]
 51%|█████     | 25600/50176 [04:15<03:53, 105.26it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.36it/s]
 52%|█████▏    | 26112/50176 [04:20<03:50, 104.59it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.73it/s]
 53%|█████▎    | 26624/50176 [04:25<03:41, 106.48it/s]
 54%|█████▎    | 26880/50176 [04:28<03:45, 103.52it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.62it/s]
 55%|█████▍    | 27392/50176 [04:33<03:36, 105.07it/s]
 55%|█████▌    | 27648/50176 [04:35<03:25, 109.86it/s]
 56%|█████▌    | 27904/50176 [04:37<03:34, 103.84it/s]
 56%|█████▌    | 28160/50176 [04:40<03:33, 102.93it/s]
 57%|█████▋    | 28416/50176 [04:43<03:32, 102.33it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.78it/s]
 58%|█████▊    | 28928/50176 [04:48<03:28, 102.11it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.60it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.93it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.91it/s]
 60%|█████▉    | 29952/50176 [04:59<03:34, 94.25it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.08it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.83it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.35it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.52it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.76it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 105.99it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.16it/s]
 64%|██████▍   | 32000/50176 [05:18<02:49, 107.44it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.60it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.52it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.68it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 102.08it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.53it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.17it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.29it/s]
 68%|██████▊   | 34048/50176 [05:37<02:31, 106.21it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.56it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.31it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.59it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.84it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.50it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.91it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.26it/s]
 72%|███████▏  | 36096/50176 [05:57<02:16, 103.46it/s]
 72%|███████▏  | 36352/50176 [06:00<02:16, 101.65it/s]
 73%|███████▎  | 36608/50176 [06:02<02:14, 100.82it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.47it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.55it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.31it/s]
 75%|███████▌  | 37632/50176 [06:12<01:57, 106.32it/s]
 76%|███████▌  | 37888/50176 [06:14<01:54, 107.33it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.67it/s]
 77%|███████▋  | 38400/50176 [06:19<01:53, 103.83it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.43it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 103.95it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.71it/s]
 79%|███████▊  | 39424/50176 [06:30<01:45, 102.25it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.72it/s]
 80%|███████▉  | 39936/50176 [06:35<01:39, 103.12it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.86it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.55it/s] 
 81%|████████  | 40704/50176 [06:42<01:36, 97.94it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.48it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.88it/s]
 83%|████████▎ | 41472/50176 [06:50<01:24, 102.77it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 105.08it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.41it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.74it/s]
 85%|████████▍ | 42496/50176 [06:59<01:11, 106.92it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 104.19it/s]
 86%|████████▌ | 43008/50176 [07:04<01:09, 103.65it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.55it/s]
 87%|████████▋ | 43520/50176 [07:10<01:06, 100.75it/s]
 87%|████████▋ | 43776/50176 [07:12<01:03, 100.22it/s]
 88%|████████▊ | 44032/50176 [07:14<01:00, 102.36it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.83it/s]
 89%|████████▉ | 44544/50176 [07:19<00:55, 101.40it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.20it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.30it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.22it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.37it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 104.05it/s]
 92%|█████████▏| 46080/50176 [07:34<00:40, 101.51it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.45it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.45it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 102.05it/s]
 94%|█████████▍| 47104/50176 [07:45<00:31, 98.41it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.77it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.02it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.37it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 104.11it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.33it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.17it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.33it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.45it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.60it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.51it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 95.80it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 126.94it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.65it/s]
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6177	imagenet-zeroshot-val-top5: 0.8741
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt
new file mode 100644
index 0000000000000000000000000000000000000000..14cdf7121b7938b29faa64bbfb320b89d146f60f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:65ec328c4895214fa5e537125e59fe9ba5fea7b4384a46b71ff444458c007c2d
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0250f93ae48d40405abd9015da6c0e767d5a6fa9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt).
+2022-12-21,10:44:38 | INFO | Model:
+2022-12-21,10:44:38 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:38 | INFO | Params:
+2022-12-21,10:44:38 | INFO |   batch_size: 256
+2022-12-21,10:44:38 | INFO |   beta1: 0.9
+2022-12-21,10:44:38 | INFO |   beta2: 0.98
+2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/checkpoints
+2022-12-21,10:44:38 | INFO |   copy_codebase: False
+2022-12-21,10:44:38 | INFO |   csv_caption_key: title
+2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:38 | INFO |   csv_separator: 	
+2022-12-21,10:44:38 | INFO |   dataset_resampled: False
+2022-12-21,10:44:38 | INFO |   dataset_type: auto
+2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:38 | INFO |   debug: False
+2022-12-21,10:44:38 | INFO |   device: cuda:0
+2022-12-21,10:44:38 | INFO |   dist_backend: nccl
+2022-12-21,10:44:38 | INFO |   dist_url: env://
+2022-12-21,10:44:38 | INFO |   distributed: False
+2022-12-21,10:44:38 | INFO |   epochs: 32
+2022-12-21,10:44:38 | INFO |   eps: 1e-06
+2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:38 | INFO |   gather_with_grad: False
+2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:38 | INFO |   horovod: False
+2022-12-21,10:44:38 | INFO |   image_mean: None
+2022-12-21,10:44:38 | INFO |   image_std: None
+2022-12-21,10:44:38 | INFO |   imagenet_v2: None
+2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:38 | INFO |   local_loss: False
+2022-12-21,10:44:38 | INFO |   local_rank: 0
+2022-12-21,10:44:38 | INFO |   lock_image: False
+2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:38 | INFO |   log_level: 20
+2022-12-21,10:44:38 | INFO |   log_local: False
+2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/out.log
+2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:38 | INFO |   lr: 0.0005
+2022-12-21,10:44:38 | INFO |   model: ViT-B-16
+2022-12-21,10:44:38 | INFO |   name: epoch_42.pt
+2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt
+2022-12-21,10:44:38 | INFO |   pretrained_image: False
+2022-12-21,10:44:38 | INFO |   rank: 0
+2022-12-21,10:44:38 | INFO |   report_to: 
+2022-12-21,10:44:38 | INFO |   resume: None
+2022-12-21,10:44:38 | INFO |   save_frequency: 1
+2022-12-21,10:44:38 | INFO |   save_most_recent: False
+2022-12-21,10:44:38 | INFO |   seed: 0
+2022-12-21,10:44:38 | INFO |   skip_scheduler: False
+2022-12-21,10:44:38 | INFO |   tensorboard: False
+2022-12-21,10:44:38 | INFO |   tensorboard_path: 
+2022-12-21,10:44:38 | INFO |   torchscript: False
+2022-12-21,10:44:38 | INFO |   trace: False
+2022-12-21,10:44:38 | INFO |   train_data: None
+2022-12-21,10:44:38 | INFO |   train_num_samples: None
+2022-12-21,10:44:38 | INFO |   use_bn_sync: False
+2022-12-21,10:44:38 | INFO |   val_data: None
+2022-12-21,10:44:38 | INFO |   val_frequency: 1
+2022-12-21,10:44:38 | INFO |   val_num_samples: None
+2022-12-21,10:44:38 | INFO |   wandb: False
+2022-12-21,10:44:38 | INFO |   wandb_notes: 
+2022-12-21,10:44:38 | INFO |   warmup: 10000
+2022-12-21,10:44:38 | INFO |   wd: 0.2
+2022-12-21,10:44:38 | INFO |   workers: 1
+2022-12-21,10:44:38 | INFO |   world_size: 1
+2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:06,  1.50it/s]
  1%|          | 6/1000 [00:00<01:39, 10.03it/s]
  1%|          | 12/1000 [00:00<00:49, 19.80it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.29it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.35it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.15it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.35it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.43it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.05it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.95it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.32it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.34it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.86it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.38it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.90it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.23it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.63it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.84it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.09it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.23it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.18it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.17it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.08it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.04it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.13it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.15it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.02it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.91it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.90it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.85it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.81it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.74it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.56it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.46it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.56it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.55it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.74it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.69it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.61it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.67it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.78it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.89it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.93it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.77it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.81it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 58.01it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.11it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.31it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.27it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.23it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.36it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.32it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.24it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.05it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.11it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.18it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.28it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.90it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.92it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.05it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.18it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.08it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.79it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.88it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 58.00it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.16it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.24it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.24it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.25it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.27it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.38it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.38it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.17it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.23it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.28it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.26it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.21it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.36it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.28it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.25it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.25it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.17it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.21it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.13it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.98it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.05it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.17it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.13it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.25it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.27it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.16it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.17it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.24it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.13it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.07it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.23it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.13it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.24it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.15it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.29it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.29it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.38it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.19it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.13it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.27it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.23it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.32it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.37it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.30it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.34it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.36it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.23it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.31it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.30it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.29it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.21it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.16it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.26it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.24it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.32it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.40it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.24it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.34it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.30it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.32it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.36it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.17it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.23it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.19it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.19it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.23it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.32it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.42it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.46it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.39it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.43it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.39it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.39it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.33it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.38it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.36it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.24it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.32it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.33it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.30it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.32it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.28it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.26it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.37it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.41it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.42it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.43it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.44it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.39it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.48it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.54it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.56it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.48it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.19it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.01it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.94it/s]
+2022-12-21,10:44:56 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<09:57, 83.60it/s]
  1%|          | 512/50176 [00:05<09:13, 89.73it/s]
  2%|▏         | 768/50176 [00:08<08:28, 97.15it/s]
  2%|▏         | 1024/50176 [00:10<08:22, 97.90it/s]
  3%|▎         | 1280/50176 [00:13<08:18, 98.01it/s]
  3%|▎         | 1536/50176 [00:16<08:42, 93.17it/s]
  4%|▎         | 1792/50176 [00:19<08:35, 93.84it/s]
  4%|▍         | 2048/50176 [00:21<08:16, 97.03it/s]
  5%|▍         | 2304/50176 [00:23<08:03, 98.98it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.46it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.07it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.80it/s]
  7%|▋         | 3328/50176 [00:34<07:43, 100.99it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.01it/s]
  8%|▊         | 3840/50176 [00:39<07:41, 100.51it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.88it/s] 
  9%|▊         | 4352/50176 [00:44<07:52, 96.97it/s]
  9%|▉         | 4608/50176 [00:47<07:37, 99.66it/s]
 10%|▉         | 4864/50176 [00:49<07:33, 99.98it/s]
 10%|█         | 5120/50176 [00:52<07:23, 101.66it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.27it/s]
 11%|█         | 5632/50176 [00:57<07:19, 101.46it/s]
 12%|█▏        | 5888/50176 [00:59<07:18, 101.07it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.46it/s] 
 13%|█▎        | 6400/50176 [01:05<07:32, 96.64it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.82it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.67it/s] 
 14%|█▍        | 7168/50176 [01:12<07:14, 98.91it/s]
 15%|█▍        | 7424/50176 [01:15<07:04, 100.70it/s]
 15%|█▌        | 7680/50176 [01:17<06:54, 102.62it/s]
 16%|█▌        | 7936/50176 [01:20<06:43, 104.62it/s]
 16%|█▋        | 8192/50176 [01:22<06:41, 104.52it/s]
 17%|█▋        | 8448/50176 [01:24<06:37, 105.02it/s]
 17%|█▋        | 8704/50176 [01:27<06:38, 104.02it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.09it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 102.96it/s]
 19%|█▉        | 9472/50176 [01:34<06:30, 104.13it/s]
 19%|█▉        | 9728/50176 [01:37<06:36, 102.06it/s]
 20%|█▉        | 9984/50176 [01:40<06:44, 99.27it/s] 
 20%|██        | 10240/50176 [01:42<06:28, 102.87it/s]
 21%|██        | 10496/50176 [01:45<06:22, 103.87it/s]
 21%|██▏       | 10752/50176 [01:47<06:35, 99.74it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.75it/s]
 22%|██▏       | 11264/50176 [01:52<06:23, 101.40it/s]
 23%|██▎       | 11520/50176 [01:55<06:16, 102.57it/s]
 23%|██▎       | 11776/50176 [01:57<06:19, 101.13it/s]
 24%|██▍       | 12032/50176 [02:00<06:11, 102.56it/s]
 24%|██▍       | 12288/50176 [02:02<06:00, 104.99it/s]
 25%|██▌       | 12544/50176 [02:05<06:04, 103.20it/s]
 26%|██▌       | 12800/50176 [02:07<06:05, 102.14it/s]
 26%|██▌       | 13056/50176 [02:10<06:07, 100.87it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 102.02it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.29it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.84it/s]
 28%|██▊       | 14080/50176 [02:21<06:18, 95.33it/s]
 29%|██▊       | 14336/50176 [02:23<06:16, 95.25it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.81it/s]
 30%|██▉       | 14848/50176 [02:29<06:16, 93.72it/s]
 30%|███       | 15104/50176 [02:31<06:09, 94.81it/s]
 31%|███       | 15360/50176 [02:34<06:04, 95.51it/s]
 31%|███       | 15616/50176 [02:36<05:52, 97.93it/s]
 32%|███▏      | 15872/50176 [02:39<05:55, 96.56it/s]
 32%|███▏      | 16128/50176 [02:42<05:46, 98.39it/s]
 33%|███▎      | 16384/50176 [02:44<05:39, 99.39it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.88it/s]
 34%|███▎      | 16896/50176 [02:49<05:36, 98.77it/s]
 34%|███▍      | 17152/50176 [02:52<05:30, 99.93it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.29it/s]
 35%|███▌      | 17664/50176 [02:57<05:18, 101.98it/s]
 36%|███▌      | 17920/50176 [02:59<05:13, 102.89it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.62it/s]
 37%|███▋      | 18432/50176 [03:04<05:07, 103.36it/s]
 37%|███▋      | 18688/50176 [03:07<05:12, 100.92it/s]
 38%|███▊      | 18944/50176 [03:10<05:10, 100.64it/s]
 38%|███▊      | 19200/50176 [03:12<05:17, 97.68it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.60it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.99it/s]
 40%|███▉      | 19968/50176 [03:20<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:22<04:50, 103.22it/s]
 41%|████      | 20480/50176 [03:25<04:51, 101.99it/s]
 41%|████▏     | 20736/50176 [03:27<04:45, 102.95it/s]
 42%|████▏     | 20992/50176 [03:30<04:32, 106.94it/s]
 42%|████▏     | 21248/50176 [03:32<04:41, 102.78it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.62it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.23it/s]
 44%|████▍     | 22016/50176 [03:39<04:19, 108.62it/s]
 44%|████▍     | 22272/50176 [03:42<04:15, 109.14it/s]
 45%|████▍     | 22528/50176 [03:44<04:16, 107.59it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.11it/s]
 46%|████▌     | 23040/50176 [03:49<04:22, 103.53it/s]
 46%|████▋     | 23296/50176 [03:52<04:25, 101.38it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.15it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.91it/s]
 48%|████▊     | 24064/50176 [03:59<04:09, 104.79it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.97it/s]
 49%|████▉     | 24576/50176 [04:04<04:07, 103.40it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.73it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.12it/s]
 51%|█████     | 25344/50176 [04:11<03:49, 108.15it/s]
 51%|█████     | 25600/50176 [04:15<04:16, 95.70it/s] 
 52%|█████▏    | 25856/50176 [04:17<04:15, 95.22it/s]
 52%|█████▏    | 26112/50176 [04:20<04:08, 96.90it/s]
 53%|█████▎    | 26368/50176 [04:22<03:59, 99.25it/s]
 53%|█████▎    | 26624/50176 [04:25<03:53, 101.00it/s]
 54%|█████▎    | 26880/50176 [04:28<03:56, 98.43it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.60it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.13it/s]
 55%|█████▌    | 27648/50176 [04:34<03:27, 108.34it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.31it/s]
 56%|█████▌    | 28160/50176 [04:40<03:41, 99.51it/s] 
 57%|█████▋    | 28416/50176 [04:42<03:33, 101.81it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 101.43it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.19it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.38it/s]
 59%|█████▊    | 29440/50176 [04:53<03:31, 98.27it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.60it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.38it/s]
 60%|██████    | 30208/50176 [05:00<03:20, 99.47it/s]
 61%|██████    | 30464/50176 [05:03<03:07, 105.41it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.48it/s]
 62%|██████▏   | 30976/50176 [05:07<03:01, 105.99it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.03it/s]
 63%|██████▎   | 31488/50176 [05:12<02:50, 109.30it/s]
 63%|██████▎   | 31744/50176 [05:14<02:50, 107.99it/s]
 64%|██████▍   | 32000/50176 [05:17<02:46, 109.20it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.88it/s]
 65%|██████▍   | 32512/50176 [05:22<02:50, 103.70it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 107.05it/s]
 66%|██████▌   | 33024/50176 [05:26<02:39, 107.69it/s]
 66%|██████▋   | 33280/50176 [05:29<02:37, 107.29it/s]
 67%|██████▋   | 33536/50176 [05:31<02:42, 102.27it/s]
 67%|██████▋   | 33792/50176 [05:34<02:37, 103.84it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 106.00it/s]
 68%|██████▊   | 34304/50176 [05:38<02:26, 108.54it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.38it/s]
 69%|██████▉   | 34816/50176 [05:44<02:27, 104.09it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.79it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.09it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.78it/s]
 71%|███████▏  | 35840/50176 [05:53<02:18, 103.39it/s]
 72%|███████▏  | 36096/50176 [05:56<02:13, 105.10it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.46it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 104.56it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.72it/s]
 74%|███████▍  | 37120/50176 [06:06<02:06, 103.35it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 106.93it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.11it/s]
 76%|███████▌  | 37888/50176 [06:13<01:55, 106.71it/s]
 76%|███████▌  | 38144/50176 [06:15<01:52, 107.30it/s]
 77%|███████▋  | 38400/50176 [06:18<01:52, 104.55it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.51it/s]
 78%|███████▊  | 38912/50176 [06:22<01:46, 105.31it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.47it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.13it/s]
 79%|███████▉  | 39680/50176 [06:30<01:38, 106.81it/s]
 80%|███████▉  | 39936/50176 [06:32<01:33, 109.59it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.15it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 104.09it/s]
 81%|████████  | 40704/50176 [06:39<01:33, 101.79it/s]
 82%|████████▏ | 40960/50176 [06:42<01:26, 106.33it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 104.85it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.05it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.73it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.92it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.42it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.36it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 107.97it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.12it/s]
 86%|████████▌ | 43264/50176 [07:03<01:05, 106.02it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.30it/s]
 87%|████████▋ | 43776/50176 [07:09<01:02, 102.55it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 104.28it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.81it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.71it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.55it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 105.04it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.76it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 107.99it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.71it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.13it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.07it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.09it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.01it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 99.92it/s] 
 94%|█████████▍| 47360/50176 [07:43<00:28, 99.83it/s]
 95%|█████████▍| 47616/50176 [07:46<00:26, 98.46it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 101.00it/s]
 96%|█████████▌| 48128/50176 [07:51<00:20, 102.40it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.52it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.56it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.13it/s]
 98%|█████████▊| 49152/50176 [08:01<00:10, 101.95it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.34it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.40it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 96.05it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.88it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.44it/s]
+2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6169	imagenet-zeroshot-val-top5: 0.8739
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt
new file mode 100644
index 0000000000000000000000000000000000000000..76f5aec2f0cb3768c1f577d46012878e8877cebb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dc22faaeda93e134492720b3df9a087b5603b3e9cbfbef0ec60a34a239336738
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9de4e09f99b86d1df7175e32895f0ed90a308b0f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt).
+2022-12-21,10:44:38 | INFO | Model:
+2022-12-21,10:44:38 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:38 | INFO | Params:
+2022-12-21,10:44:38 | INFO |   batch_size: 256
+2022-12-21,10:44:38 | INFO |   beta1: 0.9
+2022-12-21,10:44:38 | INFO |   beta2: 0.98
+2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/checkpoints
+2022-12-21,10:44:38 | INFO |   copy_codebase: False
+2022-12-21,10:44:38 | INFO |   csv_caption_key: title
+2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:38 | INFO |   csv_separator: 	
+2022-12-21,10:44:38 | INFO |   dataset_resampled: False
+2022-12-21,10:44:38 | INFO |   dataset_type: auto
+2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:38 | INFO |   debug: False
+2022-12-21,10:44:38 | INFO |   device: cuda:0
+2022-12-21,10:44:38 | INFO |   dist_backend: nccl
+2022-12-21,10:44:38 | INFO |   dist_url: env://
+2022-12-21,10:44:38 | INFO |   distributed: False
+2022-12-21,10:44:38 | INFO |   epochs: 32
+2022-12-21,10:44:38 | INFO |   eps: 1e-06
+2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:38 | INFO |   gather_with_grad: False
+2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:38 | INFO |   horovod: False
+2022-12-21,10:44:38 | INFO |   image_mean: None
+2022-12-21,10:44:38 | INFO |   image_std: None
+2022-12-21,10:44:38 | INFO |   imagenet_v2: None
+2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:38 | INFO |   local_loss: False
+2022-12-21,10:44:38 | INFO |   local_rank: 0
+2022-12-21,10:44:38 | INFO |   lock_image: False
+2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:38 | INFO |   log_level: 20
+2022-12-21,10:44:38 | INFO |   log_local: False
+2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/out.log
+2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:38 | INFO |   lr: 0.0005
+2022-12-21,10:44:38 | INFO |   model: ViT-B-16
+2022-12-21,10:44:38 | INFO |   name: epoch_43.pt
+2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt
+2022-12-21,10:44:38 | INFO |   pretrained_image: False
+2022-12-21,10:44:38 | INFO |   rank: 0
+2022-12-21,10:44:38 | INFO |   report_to: 
+2022-12-21,10:44:38 | INFO |   resume: None
+2022-12-21,10:44:38 | INFO |   save_frequency: 1
+2022-12-21,10:44:38 | INFO |   save_most_recent: False
+2022-12-21,10:44:38 | INFO |   seed: 0
+2022-12-21,10:44:38 | INFO |   skip_scheduler: False
+2022-12-21,10:44:38 | INFO |   tensorboard: False
+2022-12-21,10:44:38 | INFO |   tensorboard_path: 
+2022-12-21,10:44:38 | INFO |   torchscript: False
+2022-12-21,10:44:38 | INFO |   trace: False
+2022-12-21,10:44:38 | INFO |   train_data: None
+2022-12-21,10:44:38 | INFO |   train_num_samples: None
+2022-12-21,10:44:38 | INFO |   use_bn_sync: False
+2022-12-21,10:44:38 | INFO |   val_data: None
+2022-12-21,10:44:38 | INFO |   val_frequency: 1
+2022-12-21,10:44:38 | INFO |   val_num_samples: None
+2022-12-21,10:44:38 | INFO |   wandb: False
+2022-12-21,10:44:38 | INFO |   wandb_notes: 
+2022-12-21,10:44:38 | INFO |   warmup: 10000
+2022-12-21,10:44:38 | INFO |   wd: 0.2
+2022-12-21,10:44:38 | INFO |   workers: 1
+2022-12-21,10:44:38 | INFO |   world_size: 1
+2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:10,  1.64it/s]
  1%|          | 6/1000 [00:00<01:32, 10.79it/s]
  1%|          | 12/1000 [00:00<00:48, 20.49it/s]
  2%|▏         | 18/1000 [00:00<00:33, 28.97it/s]
  2%|▏         | 24/1000 [00:01<00:27, 36.13it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.83it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.15it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.36it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.42it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.21it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.48it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.90it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.39it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.95it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.29it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.70it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.87it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.12it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.25it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.16it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.17it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.11it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.14it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.21it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.23it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.11it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.99it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 58.00it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.97it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.81it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.79it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.47it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.31it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.45it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.68it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.71it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.64it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.72it/s]
 25%|██▌       | 252/1000 [00:04<00:12, 57.82it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.88it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.94it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.78it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.84it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 58.03it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.14it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.35it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.33it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.30it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.43it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.42it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.32it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.14it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.21it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.32it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.41it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.17it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 58.17it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.25it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.32it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.20it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.88it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.95it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 58.03it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.12it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.23it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.25it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.25it/s]
 43%|████▎     | 426/1000 [00:07<00:09, 58.29it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.42it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.42it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.17it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.24it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.31it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.29it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.23it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.38it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.30it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.28it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.24it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.16it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.22it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.24it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.28it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.37it/s]
 53%|█████▎    | 534/1000 [00:09<00:07, 58.32it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.41it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.37it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.27it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.30it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.36it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.25it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.19it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.33it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.31it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.23it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.33it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.24it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.23it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.35it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.35it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.42it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.25it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.19it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.33it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.27it/s]
 66%|██████▌   | 660/1000 [00:11<00:05, 58.35it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.39it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.31it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.28it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.32it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.22it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.33it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.31it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.29it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.31it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.27it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.29it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.30it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.28it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.36it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.44it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.23it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.34it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.32it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.37it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.43it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.08it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.11it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.18it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.16it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.19it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.31it/s]
 83%|████████▎ | 834/1000 [00:14<00:02, 58.45it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.49it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.40it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.43it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.38it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.35it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.29it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.35it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.23it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.34it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.31it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.35it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.33it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.31it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.40it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.44it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.44it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.47it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.50it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.43it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.51it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.58it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.61it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.59it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.08it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.76it/s]
100%|██████████| 1000/1000 [00:17<00:00, 56.12it/s]
+2022-12-21,10:44:56 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:42, 85.63it/s]
  1%|          | 512/50176 [00:05<09:07, 90.79it/s]
  2%|▏         | 768/50176 [00:08<08:26, 97.61it/s]
  2%|▏         | 1024/50176 [00:10<08:21, 98.05it/s]
  3%|▎         | 1280/50176 [00:13<08:18, 98.18it/s]
  3%|▎         | 1536/50176 [00:16<08:41, 93.21it/s]
  4%|▎         | 1792/50176 [00:18<08:34, 93.95it/s]
  4%|▍         | 2048/50176 [00:21<08:15, 97.14it/s]
  5%|▍         | 2304/50176 [00:23<08:04, 98.81it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.63it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.24it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.81it/s]
  7%|▋         | 3328/50176 [00:34<07:43, 101.00it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.06it/s]
  8%|▊         | 3840/50176 [00:39<07:41, 100.49it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.91it/s] 
  9%|▊         | 4352/50176 [00:44<07:52, 96.98it/s]
  9%|▉         | 4608/50176 [00:47<07:38, 99.47it/s]
 10%|▉         | 4864/50176 [00:49<07:33, 100.00it/s]
 10%|█         | 5120/50176 [00:52<07:22, 101.76it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.30it/s]
 11%|█         | 5632/50176 [00:57<07:18, 101.51it/s]
 12%|█▏        | 5888/50176 [00:59<07:17, 101.30it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.50it/s] 
 13%|█▎        | 6400/50176 [01:05<07:33, 96.63it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.79it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.54it/s] 
 14%|█▍        | 7168/50176 [01:12<07:14, 98.94it/s]
 15%|█▍        | 7424/50176 [01:15<07:04, 100.65it/s]
 15%|█▌        | 7680/50176 [01:17<06:54, 102.56it/s]
 16%|█▌        | 7936/50176 [01:19<06:43, 104.69it/s]
 16%|█▋        | 8192/50176 [01:22<06:41, 104.61it/s]
 17%|█▋        | 8448/50176 [01:24<06:37, 105.03it/s]
 17%|█▋        | 8704/50176 [01:27<06:38, 104.13it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.15it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 102.94it/s]
 19%|█▉        | 9472/50176 [01:34<06:31, 104.03it/s]
 19%|█▉        | 9728/50176 [01:37<06:36, 102.09it/s]
 20%|█▉        | 9984/50176 [01:40<06:45, 99.17it/s] 
 20%|██        | 10240/50176 [01:42<06:28, 102.86it/s]
 21%|██        | 10496/50176 [01:44<06:22, 103.70it/s]
 21%|██▏       | 10752/50176 [01:47<06:35, 99.77it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.95it/s]
 22%|██▏       | 11264/50176 [01:52<06:23, 101.42it/s]
 23%|██▎       | 11520/50176 [01:55<06:17, 102.52it/s]
 23%|██▎       | 11776/50176 [01:57<06:19, 101.12it/s]
 24%|██▍       | 12032/50176 [02:00<06:12, 102.53it/s]
 24%|██▍       | 12288/50176 [02:02<06:01, 104.95it/s]
 25%|██▌       | 12544/50176 [02:05<06:04, 103.27it/s]
 26%|██▌       | 12800/50176 [02:07<06:06, 102.05it/s]
 26%|██▌       | 13056/50176 [02:10<06:07, 100.92it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 102.02it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.35it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.81it/s]
 28%|██▊       | 14080/50176 [02:21<06:18, 95.34it/s]
 29%|██▊       | 14336/50176 [02:23<06:16, 95.25it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.81it/s]
 30%|██▉       | 14848/50176 [02:29<06:16, 93.74it/s]
 30%|███       | 15104/50176 [02:31<06:09, 94.86it/s]
 31%|███       | 15360/50176 [02:34<06:04, 95.50it/s]
 31%|███       | 15616/50176 [02:36<05:53, 97.85it/s]
 32%|███▏      | 15872/50176 [02:39<05:55, 96.58it/s]
 32%|███▏      | 16128/50176 [02:42<05:46, 98.34it/s]
 33%|███▎      | 16384/50176 [02:44<05:40, 99.33it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.85it/s]
 34%|███▎      | 16896/50176 [02:49<05:36, 98.85it/s]
 34%|███▍      | 17152/50176 [02:52<05:30, 99.86it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.25it/s]
 35%|███▌      | 17664/50176 [02:57<05:18, 102.01it/s]
 36%|███▌      | 17920/50176 [02:59<05:13, 102.94it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.63it/s]
 37%|███▋      | 18432/50176 [03:04<05:07, 103.37it/s]
 37%|███▋      | 18688/50176 [03:07<05:11, 100.98it/s]
 38%|███▊      | 18944/50176 [03:09<05:10, 100.65it/s]
 38%|███▊      | 19200/50176 [03:12<05:17, 97.61it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.62it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.99it/s]
 40%|███▉      | 19968/50176 [03:20<05:09, 97.47it/s]
 40%|████      | 20224/50176 [03:22<04:50, 103.20it/s]
 41%|████      | 20480/50176 [03:25<04:51, 101.95it/s]
 41%|████▏     | 20736/50176 [03:27<04:46, 102.91it/s]
 42%|████▏     | 20992/50176 [03:30<04:32, 106.93it/s]
 42%|████▏     | 21248/50176 [03:32<04:41, 102.73it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.57it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.16it/s]
 44%|████▍     | 22016/50176 [03:39<04:19, 108.63it/s]
 44%|████▍     | 22272/50176 [03:42<04:15, 109.20it/s]
 45%|████▍     | 22528/50176 [03:44<04:16, 107.59it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.04it/s]
 46%|████▌     | 23040/50176 [03:49<04:22, 103.52it/s]
 46%|████▋     | 23296/50176 [03:52<04:24, 101.46it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.11it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.89it/s]
 48%|████▊     | 24064/50176 [03:59<04:09, 104.83it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.96it/s]
 49%|████▉     | 24576/50176 [04:04<04:07, 103.43it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.75it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.22it/s]
 51%|█████     | 25344/50176 [04:11<03:49, 108.08it/s]
 51%|█████     | 25600/50176 [04:15<04:16, 95.66it/s] 
 52%|█████▏    | 25856/50176 [04:17<04:15, 95.21it/s]
 52%|█████▏    | 26112/50176 [04:20<04:07, 97.09it/s]
 53%|█████▎    | 26368/50176 [04:22<03:59, 99.35it/s]
 53%|█████▎    | 26624/50176 [04:25<03:53, 100.93it/s]
 54%|█████▎    | 26880/50176 [04:28<03:56, 98.38it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.60it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.13it/s]
 55%|█████▌    | 27648/50176 [04:34<03:27, 108.34it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.44it/s]
 56%|█████▌    | 28160/50176 [04:40<03:41, 99.40it/s] 
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.64it/s]
 57%|█████▋    | 28672/50176 [04:45<03:31, 101.50it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.24it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.39it/s]
 59%|█████▊    | 29440/50176 [04:53<03:31, 98.23it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.67it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.35it/s]
 60%|██████    | 30208/50176 [05:00<03:20, 99.51it/s]
 61%|██████    | 30464/50176 [05:02<03:07, 105.38it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.41it/s]
 62%|██████▏   | 30976/50176 [05:07<03:01, 106.02it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.14it/s]
 63%|██████▎   | 31488/50176 [05:12<02:51, 109.26it/s]
 63%|██████▎   | 31744/50176 [05:14<02:50, 107.96it/s]
 64%|██████▍   | 32000/50176 [05:17<02:46, 109.17it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.76it/s]
 65%|██████▍   | 32512/50176 [05:22<02:50, 103.73it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 107.05it/s]
 66%|██████▌   | 33024/50176 [05:26<02:39, 107.47it/s]
 66%|██████▋   | 33280/50176 [05:29<02:37, 107.03it/s]
 67%|██████▋   | 33536/50176 [05:31<02:42, 102.17it/s]
 67%|██████▋   | 33792/50176 [05:34<02:38, 103.67it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 106.01it/s]
 68%|██████▊   | 34304/50176 [05:38<02:26, 108.41it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.29it/s]
 69%|██████▉   | 34816/50176 [05:43<02:27, 104.04it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.77it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.21it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.72it/s]
 71%|███████▏  | 35840/50176 [05:53<02:18, 103.22it/s]
 72%|███████▏  | 36096/50176 [05:56<02:14, 105.07it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.34it/s]
 73%|███████▎  | 36608/50176 [06:00<02:09, 104.71it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.78it/s]
 74%|███████▍  | 37120/50176 [06:06<02:06, 103.45it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 106.92it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.12it/s]
 76%|███████▌  | 37888/50176 [06:13<01:55, 106.73it/s]
 76%|███████▌  | 38144/50176 [06:15<01:52, 107.22it/s]
 77%|███████▋  | 38400/50176 [06:17<01:52, 104.63it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.31it/s]
 78%|███████▊  | 38912/50176 [06:22<01:46, 105.28it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.50it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.28it/s]
 79%|███████▉  | 39680/50176 [06:29<01:38, 106.96it/s]
 80%|███████▉  | 39936/50176 [06:32<01:33, 109.55it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.04it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 104.22it/s]
 81%|████████  | 40704/50176 [06:39<01:33, 101.68it/s]
 82%|████████▏ | 40960/50176 [06:42<01:26, 106.26it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 104.85it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.01it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.78it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.89it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.25it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.38it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 108.13it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.19it/s]
 86%|████████▌ | 43264/50176 [07:03<01:05, 106.05it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.28it/s]
 87%|████████▋ | 43776/50176 [07:08<01:02, 102.70it/s]
 88%|████████▊ | 44032/50176 [07:11<00:59, 103.94it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.87it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.59it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.67it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 105.13it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.87it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 108.09it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.73it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.07it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.14it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.31it/s]
 93%|█████████▎| 46848/50176 [07:38<00:32, 103.74it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 99.77it/s] 
 94%|█████████▍| 47360/50176 [07:43<00:28, 99.51it/s]
 95%|█████████▍| 47616/50176 [07:46<00:26, 98.42it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 101.11it/s]
 96%|█████████▌| 48128/50176 [07:51<00:20, 102.38it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.39it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.41it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.17it/s]
 98%|█████████▊| 49152/50176 [08:01<00:10, 101.93it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.53it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.22it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 96.17it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.78it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.45it/s]
+2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6219	imagenet-zeroshot-val-top5: 0.8762
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt
new file mode 100644
index 0000000000000000000000000000000000000000..56f2142a88b0b1f309ddd0c9865b8aa635380338
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:537179988416b0968837ed7bc95bf20c5c35413599a6646c466cbdbaa0c86335
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d6fe382d04ce638600aadf8962dba5e5806a67c5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt).
+2022-12-21,10:44:37 | INFO | Model:
+2022-12-21,10:44:37 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:37 | INFO | Params:
+2022-12-21,10:44:37 | INFO |   batch_size: 256
+2022-12-21,10:44:37 | INFO |   beta1: 0.9
+2022-12-21,10:44:37 | INFO |   beta2: 0.98
+2022-12-21,10:44:37 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/checkpoints
+2022-12-21,10:44:37 | INFO |   copy_codebase: False
+2022-12-21,10:44:37 | INFO |   csv_caption_key: title
+2022-12-21,10:44:37 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:37 | INFO |   csv_separator: 	
+2022-12-21,10:44:37 | INFO |   dataset_resampled: False
+2022-12-21,10:44:37 | INFO |   dataset_type: auto
+2022-12-21,10:44:37 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:37 | INFO |   debug: False
+2022-12-21,10:44:37 | INFO |   device: cuda:0
+2022-12-21,10:44:37 | INFO |   dist_backend: nccl
+2022-12-21,10:44:37 | INFO |   dist_url: env://
+2022-12-21,10:44:37 | INFO |   distributed: False
+2022-12-21,10:44:37 | INFO |   epochs: 32
+2022-12-21,10:44:37 | INFO |   eps: 1e-06
+2022-12-21,10:44:37 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:37 | INFO |   gather_with_grad: False
+2022-12-21,10:44:37 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:37 | INFO |   horovod: False
+2022-12-21,10:44:37 | INFO |   image_mean: None
+2022-12-21,10:44:37 | INFO |   image_std: None
+2022-12-21,10:44:37 | INFO |   imagenet_v2: None
+2022-12-21,10:44:37 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:37 | INFO |   local_loss: False
+2022-12-21,10:44:37 | INFO |   local_rank: 0
+2022-12-21,10:44:37 | INFO |   lock_image: False
+2022-12-21,10:44:37 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:37 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:37 | INFO |   log_level: 20
+2022-12-21,10:44:37 | INFO |   log_local: False
+2022-12-21,10:44:37 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/out.log
+2022-12-21,10:44:37 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:37 | INFO |   lr: 0.0005
+2022-12-21,10:44:37 | INFO |   model: ViT-B-16
+2022-12-21,10:44:37 | INFO |   name: epoch_44.pt
+2022-12-21,10:44:37 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:37 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:37 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:37 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt
+2022-12-21,10:44:37 | INFO |   pretrained_image: False
+2022-12-21,10:44:37 | INFO |   rank: 0
+2022-12-21,10:44:37 | INFO |   report_to: 
+2022-12-21,10:44:37 | INFO |   resume: None
+2022-12-21,10:44:37 | INFO |   save_frequency: 1
+2022-12-21,10:44:37 | INFO |   save_most_recent: False
+2022-12-21,10:44:37 | INFO |   seed: 0
+2022-12-21,10:44:37 | INFO |   skip_scheduler: False
+2022-12-21,10:44:37 | INFO |   tensorboard: False
+2022-12-21,10:44:37 | INFO |   tensorboard_path: 
+2022-12-21,10:44:37 | INFO |   torchscript: False
+2022-12-21,10:44:37 | INFO |   trace: False
+2022-12-21,10:44:37 | INFO |   train_data: None
+2022-12-21,10:44:37 | INFO |   train_num_samples: None
+2022-12-21,10:44:37 | INFO |   use_bn_sync: False
+2022-12-21,10:44:37 | INFO |   val_data: None
+2022-12-21,10:44:37 | INFO |   val_frequency: 1
+2022-12-21,10:44:37 | INFO |   val_num_samples: None
+2022-12-21,10:44:37 | INFO |   wandb: False
+2022-12-21,10:44:37 | INFO |   wandb_notes: 
+2022-12-21,10:44:37 | INFO |   warmup: 10000
+2022-12-21,10:44:37 | INFO |   wd: 0.2
+2022-12-21,10:44:37 | INFO |   workers: 1
+2022-12-21,10:44:37 | INFO |   world_size: 1
+2022-12-21,10:44:37 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:07,  1.50it/s]
  1%|          | 6/1000 [00:00<01:40,  9.90it/s]
  1%|          | 11/1000 [00:00<00:55, 17.87it/s]
  2%|▏         | 16/1000 [00:00<00:39, 24.88it/s]
  2%|▏         | 22/1000 [00:01<00:29, 33.01it/s]
  3%|▎         | 28/1000 [00:01<00:24, 39.51it/s]
  3%|▎         | 34/1000 [00:01<00:21, 44.47it/s]
  4%|▍         | 40/1000 [00:01<00:19, 48.23it/s]
  5%|▍         | 46/1000 [00:01<00:18, 50.85it/s]
  5%|▌         | 52/1000 [00:01<00:17, 52.82it/s]
  6%|▌         | 58/1000 [00:01<00:17, 54.15it/s]
  6%|▋         | 64/1000 [00:01<00:16, 55.21it/s]
  7%|▋         | 70/1000 [00:01<00:16, 55.93it/s]
  8%|▊         | 76/1000 [00:02<00:16, 56.49it/s]
  8%|▊         | 82/1000 [00:02<00:16, 57.07it/s]
  9%|▉         | 88/1000 [00:02<00:15, 57.39it/s]
  9%|▉         | 94/1000 [00:02<00:15, 57.61it/s]
 10%|█         | 100/1000 [00:02<00:15, 57.83it/s]
 11%|█         | 106/1000 [00:02<00:15, 58.08it/s]
 11%|█         | 112/1000 [00:02<00:15, 58.18it/s]
 12%|█▏        | 118/1000 [00:02<00:15, 58.19it/s]
 12%|█▏        | 124/1000 [00:02<00:15, 58.06it/s]
 13%|█▎        | 130/1000 [00:02<00:14, 58.16it/s]
 14%|█▎        | 136/1000 [00:03<00:14, 58.16it/s]
 14%|█▍        | 142/1000 [00:03<00:14, 58.06it/s]
 15%|█▍        | 148/1000 [00:03<00:14, 58.15it/s]
 15%|█▌        | 154/1000 [00:03<00:14, 58.22it/s]
 16%|█▌        | 160/1000 [00:03<00:14, 58.10it/s]
 17%|█▋        | 166/1000 [00:03<00:14, 58.04it/s]
 17%|█▋        | 172/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 178/1000 [00:03<00:14, 57.99it/s]
 18%|█▊        | 184/1000 [00:03<00:14, 57.84it/s]
 19%|█▉        | 190/1000 [00:03<00:14, 57.68it/s]
 20%|█▉        | 196/1000 [00:04<00:13, 57.68it/s]
 20%|██        | 202/1000 [00:04<00:13, 57.65it/s]
 21%|██        | 208/1000 [00:04<00:13, 57.38it/s]
 21%|██▏       | 214/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 220/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 226/1000 [00:04<00:13, 57.63it/s]
 23%|██▎       | 232/1000 [00:04<00:13, 57.76it/s]
 24%|██▍       | 238/1000 [00:04<00:13, 57.70it/s]
 24%|██▍       | 244/1000 [00:04<00:13, 57.60it/s]
 25%|██▌       | 250/1000 [00:05<00:12, 57.70it/s]
 26%|██▌       | 256/1000 [00:05<00:12, 57.92it/s]
 26%|██▌       | 262/1000 [00:05<00:12, 57.96it/s]
 27%|██▋       | 268/1000 [00:05<00:12, 57.87it/s]
 27%|██▋       | 274/1000 [00:05<00:13, 52.08it/s]
 28%|██▊       | 280/1000 [00:05<00:13, 53.83it/s]
 29%|██▊       | 286/1000 [00:05<00:12, 55.06it/s]
 29%|██▉       | 292/1000 [00:05<00:12, 56.05it/s]
 30%|██▉       | 298/1000 [00:05<00:12, 56.71it/s]
 30%|███       | 304/1000 [00:05<00:12, 57.19it/s]
 31%|███       | 310/1000 [00:06<00:11, 57.60it/s]
 32%|███▏      | 316/1000 [00:06<00:11, 57.84it/s]
 32%|███▏      | 322/1000 [00:06<00:11, 58.01it/s]
 33%|███▎      | 328/1000 [00:06<00:11, 57.89it/s]
 33%|███▎      | 334/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 340/1000 [00:06<00:11, 58.09it/s]
 35%|███▍      | 346/1000 [00:06<00:11, 58.28it/s]
 35%|███▌      | 352/1000 [00:06<00:11, 58.15it/s]
 36%|███▌      | 358/1000 [00:06<00:11, 58.24it/s]
 36%|███▋      | 364/1000 [00:07<00:10, 58.25it/s]
 37%|███▋      | 370/1000 [00:07<00:10, 58.26it/s]
 38%|███▊      | 376/1000 [00:07<00:10, 58.19it/s]
 38%|███▊      | 382/1000 [00:07<00:10, 57.99it/s]
 39%|███▉      | 388/1000 [00:07<00:10, 57.90it/s]
 39%|███▉      | 394/1000 [00:07<00:10, 58.00it/s]
 40%|████      | 400/1000 [00:07<00:10, 58.19it/s]
 41%|████      | 406/1000 [00:07<00:10, 58.22it/s]
 41%|████      | 412/1000 [00:07<00:10, 58.11it/s]
 42%|████▏     | 418/1000 [00:07<00:10, 58.20it/s]
 42%|████▏     | 424/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 430/1000 [00:08<00:09, 58.30it/s]
 44%|████▎     | 436/1000 [00:08<00:09, 58.36it/s]
 44%|████▍     | 442/1000 [00:08<00:09, 58.15it/s]
 45%|████▍     | 448/1000 [00:08<00:09, 58.19it/s]
 45%|████▌     | 454/1000 [00:08<00:09, 58.27it/s]
 46%|████▌     | 460/1000 [00:08<00:09, 58.26it/s]
 47%|████▋     | 466/1000 [00:08<00:09, 58.35it/s]
 47%|████▋     | 472/1000 [00:08<00:09, 58.32it/s]
 48%|████▊     | 478/1000 [00:08<00:08, 58.37it/s]
 48%|████▊     | 484/1000 [00:09<00:08, 58.21it/s]
 49%|████▉     | 490/1000 [00:09<00:08, 58.21it/s]
 50%|████▉     | 496/1000 [00:09<00:08, 58.16it/s]
 50%|█████     | 502/1000 [00:09<00:08, 58.22it/s]
 51%|█████     | 508/1000 [00:09<00:08, 58.15it/s]
 51%|█████▏    | 514/1000 [00:09<00:08, 58.20it/s]
 52%|█████▏    | 520/1000 [00:09<00:08, 58.27it/s]
 53%|█████▎    | 526/1000 [00:09<00:08, 58.32it/s]
 53%|█████▎    | 532/1000 [00:09<00:08, 58.30it/s]
 54%|█████▍    | 538/1000 [00:10<00:07, 58.35it/s]
 54%|█████▍    | 544/1000 [00:10<00:07, 58.43it/s]
 55%|█████▌    | 550/1000 [00:10<00:07, 58.34it/s]
 56%|█████▌    | 556/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 562/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 568/1000 [00:10<00:07, 58.11it/s]
 57%|█████▋    | 574/1000 [00:10<00:07, 58.06it/s]
 58%|█████▊    | 580/1000 [00:10<00:07, 58.22it/s]
 59%|█████▊    | 586/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 592/1000 [00:10<00:07, 58.16it/s]
 60%|█████▉    | 598/1000 [00:11<00:06, 58.21it/s]
 60%|██████    | 604/1000 [00:11<00:06, 58.12it/s]
 61%|██████    | 610/1000 [00:11<00:06, 58.26it/s]
 62%|██████▏   | 616/1000 [00:11<00:06, 58.26it/s]
 62%|██████▏   | 622/1000 [00:11<00:06, 58.33it/s]
 63%|██████▎   | 628/1000 [00:11<00:06, 58.39it/s]
 63%|██████▎   | 634/1000 [00:11<00:06, 58.24it/s]
 64%|██████▍   | 640/1000 [00:11<00:06, 58.17it/s]
 65%|██████▍   | 646/1000 [00:11<00:06, 58.31it/s]
 65%|██████▌   | 652/1000 [00:11<00:05, 58.31it/s]
 66%|██████▌   | 658/1000 [00:12<00:05, 58.35it/s]
 66%|██████▋   | 664/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 670/1000 [00:12<00:05, 58.31it/s]
 68%|██████▊   | 676/1000 [00:12<00:05, 58.37it/s]
 68%|██████▊   | 682/1000 [00:12<00:05, 58.34it/s]
 69%|██████▉   | 688/1000 [00:12<00:05, 58.40it/s]
 69%|██████▉   | 694/1000 [00:12<00:05, 58.26it/s]
 70%|███████   | 700/1000 [00:12<00:05, 58.37it/s]
 71%|███████   | 706/1000 [00:12<00:05, 58.30it/s]
 71%|███████   | 712/1000 [00:12<00:04, 58.36it/s]
 72%|███████▏  | 718/1000 [00:13<00:04, 58.34it/s]
 72%|███████▏  | 724/1000 [00:13<00:04, 58.30it/s]
 73%|███████▎  | 730/1000 [00:13<00:04, 58.27it/s]
 74%|███████▎  | 736/1000 [00:13<00:04, 58.33it/s]
 74%|███████▍  | 742/1000 [00:13<00:04, 58.26it/s]
 75%|███████▍  | 748/1000 [00:13<00:04, 58.16it/s]
 75%|███████▌  | 754/1000 [00:13<00:04, 58.34it/s]
 76%|███████▌  | 760/1000 [00:13<00:04, 58.22it/s]
 77%|███████▋  | 766/1000 [00:13<00:04, 58.30it/s]
 77%|███████▋  | 772/1000 [00:14<00:03, 58.30it/s]
 78%|███████▊  | 778/1000 [00:14<00:03, 58.37it/s]
 78%|███████▊  | 784/1000 [00:14<00:03, 58.37it/s]
 79%|███████▉  | 790/1000 [00:14<00:03, 58.21it/s]
 80%|███████▉  | 796/1000 [00:14<00:03, 58.27it/s]
 80%|████████  | 802/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 808/1000 [00:14<00:03, 58.21it/s]
 81%|████████▏ | 814/1000 [00:14<00:03, 58.20it/s]
 82%|████████▏ | 820/1000 [00:14<00:03, 58.30it/s]
 83%|████████▎ | 826/1000 [00:14<00:02, 58.26it/s]
 83%|████████▎ | 832/1000 [00:15<00:02, 58.41it/s]
 84%|████████▍ | 838/1000 [00:15<00:02, 58.51it/s]
 84%|████████▍ | 844/1000 [00:15<00:02, 58.40it/s]
 85%|████████▌ | 850/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 856/1000 [00:15<00:02, 58.39it/s]
 86%|████████▌ | 862/1000 [00:15<00:02, 58.37it/s]
 87%|████████▋ | 868/1000 [00:15<00:02, 58.31it/s]
 87%|████████▋ | 874/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 880/1000 [00:15<00:02, 58.36it/s]
 89%|████████▊ | 886/1000 [00:15<00:01, 58.23it/s]
 89%|████████▉ | 892/1000 [00:16<00:01, 58.33it/s]
 90%|████████▉ | 898/1000 [00:16<00:01, 58.38it/s]
 90%|█████████ | 904/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 910/1000 [00:16<00:01, 58.32it/s]
 92%|█████████▏| 916/1000 [00:16<00:01, 58.30it/s]
 92%|█████████▏| 922/1000 [00:16<00:01, 58.25it/s]
 93%|█████████▎| 928/1000 [00:16<00:01, 58.41it/s]
 93%|█████████▎| 934/1000 [00:16<00:01, 58.48it/s]
 94%|█████████▍| 940/1000 [00:16<00:01, 58.50it/s]
 95%|█████████▍| 946/1000 [00:17<00:00, 58.38it/s]
 95%|█████████▌| 952/1000 [00:17<00:00, 58.42it/s]
 96%|█████████▌| 958/1000 [00:17<00:00, 58.37it/s]
 96%|█████████▋| 964/1000 [00:17<00:00, 58.46it/s]
 97%|█████████▋| 970/1000 [00:17<00:00, 58.48it/s]
 98%|█████████▊| 976/1000 [00:17<00:00, 58.51it/s]
 98%|█████████▊| 982/1000 [00:17<00:00, 58.54it/s]
 99%|█████████▉| 988/1000 [00:17<00:00, 58.46it/s]
 99%|█████████▉| 994/1000 [00:17<00:00, 58.41it/s]
100%|██████████| 1000/1000 [00:17<00:00, 58.20it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.77it/s]
+2022-12-21,10:44:55 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:36, 78.39it/s]
  1%|          | 512/50176 [00:06<09:33, 86.55it/s]
  2%|▏         | 768/50176 [00:08<08:38, 95.22it/s]
  2%|▏         | 1024/50176 [00:10<08:27, 96.82it/s]
  3%|▎         | 1280/50176 [00:13<08:23, 97.18it/s]
  3%|▎         | 1536/50176 [00:16<08:45, 92.61it/s]
  4%|▎         | 1792/50176 [00:19<08:37, 93.42it/s]
  4%|▍         | 2048/50176 [00:21<08:17, 96.80it/s]
  5%|▍         | 2304/50176 [00:24<08:05, 98.69it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.59it/s]
  6%|▌         | 2816/50176 [00:29<08:03, 97.92it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.78it/s]
  7%|▋         | 3328/50176 [00:34<07:43, 101.05it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.03it/s]
  8%|▊         | 3840/50176 [00:39<07:41, 100.37it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.86it/s] 
  9%|▊         | 4352/50176 [00:44<07:52, 96.98it/s]
  9%|▉         | 4608/50176 [00:47<07:38, 99.46it/s]
 10%|▉         | 4864/50176 [00:49<07:32, 100.03it/s]
 10%|█         | 5120/50176 [00:52<07:23, 101.68it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.30it/s]
 11%|█         | 5632/50176 [00:57<07:19, 101.44it/s]
 12%|█▏        | 5888/50176 [00:59<07:17, 101.27it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.47it/s] 
 13%|█▎        | 6400/50176 [01:05<07:33, 96.63it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.94it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.53it/s] 
 14%|█▍        | 7168/50176 [01:13<07:14, 98.88it/s]
 15%|█▍        | 7424/50176 [01:15<07:04, 100.75it/s]
 15%|█▌        | 7680/50176 [01:17<06:53, 102.88it/s]
 16%|█▌        | 7936/50176 [01:20<06:44, 104.50it/s]
 16%|█▋        | 8192/50176 [01:22<06:40, 104.92it/s]
 17%|█▋        | 8448/50176 [01:25<06:37, 104.88it/s]
 17%|█▋        | 8704/50176 [01:27<06:37, 104.24it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.11it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 102.92it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.97it/s]
 19%|█▉        | 9728/50176 [01:37<06:37, 101.81it/s]
 20%|█▉        | 9984/50176 [01:40<06:44, 99.48it/s] 
 20%|██        | 10240/50176 [01:42<06:29, 102.56it/s]
 21%|██        | 10496/50176 [01:45<06:21, 103.91it/s]
 21%|██▏       | 10752/50176 [01:48<06:34, 99.84it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.72it/s]
 22%|██▏       | 11264/50176 [01:53<06:23, 101.45it/s]
 23%|██▎       | 11520/50176 [01:55<06:16, 102.55it/s]
 23%|██▎       | 11776/50176 [01:58<06:19, 101.10it/s]
 24%|██▍       | 12032/50176 [02:00<06:12, 102.51it/s]
 24%|██▍       | 12288/50176 [02:02<06:01, 104.89it/s]
 25%|██▌       | 12544/50176 [02:05<06:03, 103.40it/s]
 26%|██▌       | 12800/50176 [02:07<06:05, 102.34it/s]
 26%|██▌       | 13056/50176 [02:10<06:08, 100.85it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 102.05it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.34it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.89it/s]
 28%|██▊       | 14080/50176 [02:21<06:18, 95.31it/s]
 29%|██▊       | 14336/50176 [02:24<06:16, 95.26it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.87it/s]
 30%|██▉       | 14848/50176 [02:29<06:17, 93.60it/s]
 30%|███       | 15104/50176 [02:32<06:10, 94.63it/s]
 31%|███       | 15360/50176 [02:34<06:04, 95.43it/s]
 31%|███       | 15616/50176 [02:37<05:52, 97.94it/s]
 32%|███▏      | 15872/50176 [02:39<05:56, 96.33it/s]
 32%|███▏      | 16128/50176 [02:42<05:45, 98.44it/s]
 33%|███▎      | 16384/50176 [02:44<05:40, 99.32it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.89it/s]
 34%|███▎      | 16896/50176 [02:50<05:36, 98.80it/s]
 34%|███▍      | 17152/50176 [02:52<05:30, 99.85it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.19it/s]
 35%|███▌      | 17664/50176 [02:57<05:18, 102.10it/s]
 36%|███▌      | 17920/50176 [03:00<05:13, 102.79it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.67it/s]
 37%|███▋      | 18432/50176 [03:05<05:06, 103.46it/s]
 37%|███▋      | 18688/50176 [03:07<05:12, 100.85it/s]
 38%|███▊      | 18944/50176 [03:10<05:10, 100.71it/s]
 38%|███▊      | 19200/50176 [03:13<05:17, 97.65it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.58it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.88it/s]
 40%|███▉      | 19968/50176 [03:21<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:23<04:49, 103.34it/s]
 41%|████      | 20480/50176 [03:25<04:51, 101.94it/s]
 41%|████▏     | 20736/50176 [03:28<04:45, 102.99it/s]
 42%|████▏     | 20992/50176 [03:30<04:33, 106.80it/s]
 42%|████▏     | 21248/50176 [03:33<04:41, 102.77it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.62it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.21it/s]
 44%|████▍     | 22016/50176 [03:40<04:19, 108.54it/s]
 44%|████▍     | 22272/50176 [03:42<04:15, 109.21it/s]
 45%|████▍     | 22528/50176 [03:44<04:16, 107.61it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.19it/s]
 46%|████▌     | 23040/50176 [03:50<04:21, 103.64it/s]
 46%|████▋     | 23296/50176 [03:52<04:25, 101.34it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.22it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.86it/s]
 48%|████▊     | 24064/50176 [04:00<04:09, 104.67it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.94it/s]
 49%|████▉     | 24576/50176 [04:05<04:07, 103.32it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.77it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.26it/s]
 51%|█████     | 25344/50176 [04:12<03:49, 108.12it/s]
 51%|█████     | 25600/50176 [04:15<04:16, 95.79it/s] 
 52%|█████▏    | 25856/50176 [04:18<04:15, 95.16it/s]
 52%|█████▏    | 26112/50176 [04:20<04:07, 97.07it/s]
 53%|█████▎    | 26368/50176 [04:23<04:00, 99.14it/s]
 53%|█████▎    | 26624/50176 [04:25<03:53, 100.98it/s]
 54%|█████▎    | 26880/50176 [04:28<03:56, 98.52it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.59it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.16it/s]
 55%|█████▌    | 27648/50176 [04:35<03:28, 108.16it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.23it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 99.76it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:33, 101.99it/s]
 57%|█████▋    | 28672/50176 [04:45<03:31, 101.49it/s]
 58%|█████▊    | 28928/50176 [04:47<03:26, 103.10it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.26it/s]
 59%|█████▊    | 29440/50176 [04:53<03:31, 98.25it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:32, 96.52it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.29it/s]
 60%|██████    | 30208/50176 [05:01<03:20, 99.46it/s]
 61%|██████    | 30464/50176 [05:03<03:06, 105.67it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.68it/s]
 62%|██████▏   | 30976/50176 [05:08<03:01, 105.92it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 105.94it/s]
 63%|██████▎   | 31488/50176 [05:12<02:50, 109.36it/s]
 63%|██████▎   | 31744/50176 [05:15<02:51, 107.76it/s]
 64%|██████▍   | 32000/50176 [05:17<02:46, 109.25it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.81it/s]
 65%|██████▍   | 32512/50176 [05:22<02:50, 103.64it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 107.09it/s]
 66%|██████▌   | 33024/50176 [05:27<02:39, 107.60it/s]
 66%|██████▋   | 33280/50176 [05:29<02:37, 107.26it/s]
 67%|██████▋   | 33536/50176 [05:32<02:42, 102.32it/s]
 67%|██████▋   | 33792/50176 [05:34<02:37, 103.88it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 105.95it/s]
 68%|██████▊   | 34304/50176 [05:39<02:26, 108.49it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.50it/s]
 69%|██████▉   | 34816/50176 [05:44<02:27, 104.13it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.76it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.09it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.56it/s]
 71%|███████▏  | 35840/50176 [05:54<02:19, 103.12it/s]
 72%|███████▏  | 36096/50176 [05:56<02:14, 105.03it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.26it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 104.76it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.88it/s]
 74%|███████▍  | 37120/50176 [06:06<02:06, 103.43it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 107.09it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.18it/s]
 76%|███████▌  | 37888/50176 [06:13<01:55, 106.79it/s]
 76%|███████▌  | 38144/50176 [06:15<01:51, 107.43it/s]
 77%|███████▋  | 38400/50176 [06:18<01:52, 104.40it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.63it/s]
 78%|███████▊  | 38912/50176 [06:23<01:46, 105.32it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.47it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.16it/s]
 79%|███████▉  | 39680/50176 [06:30<01:38, 106.92it/s]
 80%|███████▉  | 39936/50176 [06:32<01:33, 109.64it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.10it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 103.85it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.67it/s]
 82%|████████▏ | 40960/50176 [06:42<01:26, 106.41it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 105.05it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 104.93it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.79it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.96it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.50it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.34it/s]
 85%|████████▌ | 42752/50176 [06:59<01:08, 107.82it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.07it/s]
 86%|████████▌ | 43264/50176 [07:04<01:05, 106.14it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.41it/s]
 87%|████████▋ | 43776/50176 [07:09<01:02, 102.37it/s]
 88%|████████▊ | 44032/50176 [07:11<00:59, 104.05it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 108.08it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.69it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.46it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 105.14it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.75it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 107.83it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.56it/s]
 92%|█████████▏| 46080/50176 [07:31<00:39, 103.28it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.12it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.17it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.01it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 100.00it/s]
 94%|█████████▍| 47360/50176 [07:43<00:28, 99.92it/s] 
 95%|█████████▍| 47616/50176 [07:46<00:26, 98.34it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.98it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 102.45it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.54it/s]
 97%|█████████▋| 48640/50176 [07:56<00:14, 104.73it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.06it/s]
 98%|█████████▊| 49152/50176 [08:01<00:10, 101.93it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.34it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.38it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 96.06it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.70it/s]
100%|██████████| 50176/50176 [08:10<00:00, 102.39it/s]
+2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6259	imagenet-zeroshot-val-top5: 0.8789
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt
new file mode 100644
index 0000000000000000000000000000000000000000..34dd7e36549ec451df8fd2413da140d852d5bc94
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5071605d65556eb8c98773e6dd5608f98154724f37597cc6ab139db3ecf52318
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0601d8e8075fab5783ca2bc32646cd6e12852000
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt).
+2022-12-21,10:44:38 | INFO | Model:
+2022-12-21,10:44:38 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:38 | INFO | Params:
+2022-12-21,10:44:38 | INFO |   batch_size: 256
+2022-12-21,10:44:38 | INFO |   beta1: 0.9
+2022-12-21,10:44:38 | INFO |   beta2: 0.98
+2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/checkpoints
+2022-12-21,10:44:38 | INFO |   copy_codebase: False
+2022-12-21,10:44:38 | INFO |   csv_caption_key: title
+2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:38 | INFO |   csv_separator: 	
+2022-12-21,10:44:38 | INFO |   dataset_resampled: False
+2022-12-21,10:44:38 | INFO |   dataset_type: auto
+2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:38 | INFO |   debug: False
+2022-12-21,10:44:38 | INFO |   device: cuda:0
+2022-12-21,10:44:38 | INFO |   dist_backend: nccl
+2022-12-21,10:44:38 | INFO |   dist_url: env://
+2022-12-21,10:44:38 | INFO |   distributed: False
+2022-12-21,10:44:38 | INFO |   epochs: 32
+2022-12-21,10:44:38 | INFO |   eps: 1e-06
+2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:38 | INFO |   gather_with_grad: False
+2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:38 | INFO |   horovod: False
+2022-12-21,10:44:38 | INFO |   image_mean: None
+2022-12-21,10:44:38 | INFO |   image_std: None
+2022-12-21,10:44:38 | INFO |   imagenet_v2: None
+2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:38 | INFO |   local_loss: False
+2022-12-21,10:44:38 | INFO |   local_rank: 0
+2022-12-21,10:44:38 | INFO |   lock_image: False
+2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:38 | INFO |   log_level: 20
+2022-12-21,10:44:38 | INFO |   log_local: False
+2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/out.log
+2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:38 | INFO |   lr: 0.0005
+2022-12-21,10:44:38 | INFO |   model: ViT-B-16
+2022-12-21,10:44:38 | INFO |   name: epoch_45.pt
+2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt
+2022-12-21,10:44:38 | INFO |   pretrained_image: False
+2022-12-21,10:44:38 | INFO |   rank: 0
+2022-12-21,10:44:38 | INFO |   report_to: 
+2022-12-21,10:44:38 | INFO |   resume: None
+2022-12-21,10:44:38 | INFO |   save_frequency: 1
+2022-12-21,10:44:38 | INFO |   save_most_recent: False
+2022-12-21,10:44:38 | INFO |   seed: 0
+2022-12-21,10:44:38 | INFO |   skip_scheduler: False
+2022-12-21,10:44:38 | INFO |   tensorboard: False
+2022-12-21,10:44:38 | INFO |   tensorboard_path: 
+2022-12-21,10:44:38 | INFO |   torchscript: False
+2022-12-21,10:44:38 | INFO |   trace: False
+2022-12-21,10:44:38 | INFO |   train_data: None
+2022-12-21,10:44:38 | INFO |   train_num_samples: None
+2022-12-21,10:44:38 | INFO |   use_bn_sync: False
+2022-12-21,10:44:38 | INFO |   val_data: None
+2022-12-21,10:44:38 | INFO |   val_frequency: 1
+2022-12-21,10:44:38 | INFO |   val_num_samples: None
+2022-12-21,10:44:38 | INFO |   wandb: False
+2022-12-21,10:44:38 | INFO |   wandb_notes: 
+2022-12-21,10:44:38 | INFO |   warmup: 10000
+2022-12-21,10:44:38 | INFO |   wd: 0.2
+2022-12-21,10:44:38 | INFO |   workers: 1
+2022-12-21,10:44:38 | INFO |   world_size: 1
+2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:55,  1.40it/s]
  1%|          | 6/1000 [00:00<01:45,  9.45it/s]
  1%|          | 12/1000 [00:00<00:53, 18.49it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.43it/s]
  2%|▏         | 24/1000 [00:01<00:29, 33.64it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.58it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.18it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.76it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.37it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.68it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.81it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.48it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.01it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.58it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.92it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.33it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.50it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.88it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.87it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.80it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.82it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.78it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.82it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.88it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.92it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.77it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.61it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.58it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.49it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.45it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.42it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.20it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.09it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.23it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.46it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.43it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.41it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.49it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.59it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.63it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.45it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.48it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.63it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.73it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.95it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.92it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.69it/s]
 31%|███       | 312/1000 [00:06<00:13, 49.48it/s]
 32%|███▏      | 318/1000 [00:06<00:13, 51.78it/s]
 32%|███▏      | 324/1000 [00:06<00:12, 53.43it/s]
 33%|███▎      | 330/1000 [00:06<00:12, 54.55it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 55.55it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 56.32it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 56.89it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.03it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.28it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.53it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.73it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.71it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.55it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.83it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.91it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.91it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.93it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.97it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.07it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.07it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.87it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.92it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.99it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.95it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.90it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.04it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.95it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.93it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.93it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.86it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.91it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.82it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.96it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.09it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.10it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.99it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.05it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.91it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.86it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.98it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.96it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.88it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.99it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.90it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.91it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.02it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.02it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.11it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.93it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.88it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.03it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.99it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.06it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.07it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.99it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.05it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.06it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.08it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.97it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.06it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.04it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.00it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.02it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.98it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.03it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.07it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.06it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.14it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.22it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.04it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.12it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.83it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.93it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.00it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.82it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.84it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.92it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.89it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.92it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.01it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.15it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.19it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.10it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.16it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.13it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.14it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.07it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.08it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.95it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.07it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.05it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.99it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.01it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.01it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.00it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.09it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.12it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.13it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.16it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.19it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.15it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.22it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.28it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.03it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.84it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.90it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.47it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.30it/s]
+2022-12-21,10:44:56 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:09, 90.82it/s]
  1%|          | 512/50176 [00:05<08:55, 92.67it/s]
  2%|▏         | 768/50176 [00:07<08:21, 98.50it/s]
  2%|▏         | 1024/50176 [00:10<08:15, 99.28it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.51it/s]
  3%|▎         | 1536/50176 [00:16<08:40, 93.52it/s]
  4%|▎         | 1792/50176 [00:18<08:33, 94.25it/s]
  4%|▍         | 2048/50176 [00:21<08:14, 97.32it/s]
  5%|▍         | 2304/50176 [00:23<08:02, 99.18it/s]
  5%|▌         | 2560/50176 [00:26<07:54, 100.33it/s]
  6%|▌         | 2816/50176 [00:28<08:02, 98.23it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.93it/s]
  7%|▋         | 3328/50176 [00:33<07:43, 101.11it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.03it/s]
  8%|▊         | 3840/50176 [00:38<07:41, 100.48it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.89it/s] 
  9%|▊         | 4352/50176 [00:44<07:51, 97.21it/s]
  9%|▉         | 4608/50176 [00:46<07:37, 99.66it/s]
 10%|▉         | 4864/50176 [00:49<07:33, 100.00it/s]
 10%|█         | 5120/50176 [00:51<07:23, 101.57it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.32it/s]
 11%|█         | 5632/50176 [00:56<07:19, 101.41it/s]
 12%|█▏        | 5888/50176 [00:59<07:17, 101.33it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.31it/s] 
 13%|█▎        | 6400/50176 [01:05<07:32, 96.66it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.78it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.65it/s] 
 14%|█▍        | 7168/50176 [01:12<07:15, 98.83it/s]
 15%|█▍        | 7424/50176 [01:15<07:05, 100.46it/s]
 15%|█▌        | 7680/50176 [01:17<06:55, 102.39it/s]
 16%|█▌        | 7936/50176 [01:19<06:43, 104.81it/s]
 16%|█▋        | 8192/50176 [01:22<06:41, 104.50it/s]
 17%|█▋        | 8448/50176 [01:24<06:38, 104.82it/s]
 17%|█▋        | 8704/50176 [01:27<06:38, 104.14it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.11it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 103.03it/s]
 19%|█▉        | 9472/50176 [01:34<06:31, 104.04it/s]
 19%|█▉        | 9728/50176 [01:37<06:36, 101.98it/s]
 20%|█▉        | 9984/50176 [01:40<06:45, 99.22it/s] 
 20%|██        | 10240/50176 [01:42<06:27, 103.12it/s]
 21%|██        | 10496/50176 [01:44<06:22, 103.64it/s]
 21%|██▏       | 10752/50176 [01:47<06:35, 99.72it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.83it/s]
 22%|██▏       | 11264/50176 [01:52<06:23, 101.38it/s]
 23%|██▎       | 11520/50176 [01:54<06:16, 102.72it/s]
 23%|██▎       | 11776/50176 [01:57<06:19, 101.25it/s]
 24%|██▍       | 12032/50176 [02:00<06:11, 102.67it/s]
 24%|██▍       | 12288/50176 [02:02<06:00, 105.15it/s]
 25%|██▌       | 12544/50176 [02:04<06:04, 103.38it/s]
 26%|██▌       | 12800/50176 [02:07<06:06, 102.07it/s]
 26%|██▌       | 13056/50176 [02:10<06:08, 100.62it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 101.84it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.21it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.79it/s]
 28%|██▊       | 14080/50176 [02:20<06:18, 95.31it/s]
 29%|██▊       | 14336/50176 [02:23<06:15, 95.33it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.83it/s]
 30%|██▉       | 14848/50176 [02:29<06:17, 93.67it/s]
 30%|███       | 15104/50176 [02:31<06:09, 94.90it/s]
 31%|███       | 15360/50176 [02:34<06:03, 95.68it/s]
 31%|███       | 15616/50176 [02:36<05:52, 97.94it/s]
 32%|███▏      | 15872/50176 [02:39<05:54, 96.66it/s]
 32%|███▏      | 16128/50176 [02:41<05:46, 98.33it/s]
 33%|███▎      | 16384/50176 [02:44<05:40, 99.33it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.95it/s]
 34%|███▎      | 16896/50176 [02:49<05:37, 98.57it/s]
 34%|███▍      | 17152/50176 [02:52<05:29, 100.12it/s]
 35%|███▍      | 17408/50176 [02:54<05:29, 99.40it/s] 
 35%|███▌      | 17664/50176 [02:57<05:18, 101.92it/s]
 36%|███▌      | 17920/50176 [02:59<05:13, 102.94it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.81it/s]
 37%|███▋      | 18432/50176 [03:04<05:07, 103.29it/s]
 37%|███▋      | 18688/50176 [03:07<05:12, 100.86it/s]
 38%|███▊      | 18944/50176 [03:09<05:10, 100.66it/s]
 38%|███▊      | 19200/50176 [03:12<05:17, 97.69it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.63it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.96it/s]
 40%|███▉      | 19968/50176 [03:20<05:10, 97.43it/s]
 40%|████      | 20224/50176 [03:22<04:50, 103.28it/s]
 41%|████      | 20480/50176 [03:25<04:51, 102.00it/s]
 41%|████▏     | 20736/50176 [03:27<04:46, 102.93it/s]
 42%|████▏     | 20992/50176 [03:29<04:33, 106.80it/s]
 42%|████▏     | 21248/50176 [03:32<04:41, 102.66it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.55it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.18it/s]
 44%|████▍     | 22016/50176 [03:39<04:19, 108.59it/s]
 44%|████▍     | 22272/50176 [03:41<04:15, 109.14it/s]
 45%|████▍     | 22528/50176 [03:44<04:17, 107.58it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.10it/s]
 46%|████▌     | 23040/50176 [03:49<04:22, 103.55it/s]
 46%|████▋     | 23296/50176 [03:52<04:24, 101.46it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.06it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.94it/s]
 48%|████▊     | 24064/50176 [03:59<04:09, 104.73it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.98it/s]
 49%|████▉     | 24576/50176 [04:04<04:07, 103.39it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.74it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.22it/s]
 51%|█████     | 25344/50176 [04:11<03:49, 108.06it/s]
 51%|█████     | 25600/50176 [04:14<04:16, 95.69it/s] 
 52%|█████▏    | 25856/50176 [04:17<04:15, 95.19it/s]
 52%|█████▏    | 26112/50176 [04:20<04:08, 96.84it/s]
 53%|█████▎    | 26368/50176 [04:22<03:59, 99.26it/s]
 53%|█████▎    | 26624/50176 [04:25<03:52, 101.09it/s]
 54%|█████▎    | 26880/50176 [04:27<03:56, 98.36it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.60it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.18it/s]
 55%|█████▌    | 27648/50176 [04:34<03:27, 108.35it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.27it/s]
 56%|█████▌    | 28160/50176 [04:40<03:41, 99.32it/s] 
 57%|█████▋    | 28416/50176 [04:42<03:33, 101.85it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 101.40it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.17it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.34it/s]
 59%|█████▊    | 29440/50176 [04:52<03:30, 98.32it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:32, 96.59it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.40it/s]
 60%|██████    | 30208/50176 [05:00<03:20, 99.43it/s]
 61%|██████    | 30464/50176 [05:02<03:06, 105.43it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.35it/s]
 62%|██████▏   | 30976/50176 [05:07<03:00, 106.29it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.03it/s]
 63%|██████▎   | 31488/50176 [05:12<02:51, 109.22it/s]
 63%|██████▎   | 31744/50176 [05:14<02:50, 108.13it/s]
 64%|██████▍   | 32000/50176 [05:16<02:46, 109.43it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.86it/s]
 65%|██████▍   | 32512/50176 [05:21<02:50, 103.77it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 106.98it/s]
 66%|██████▌   | 33024/50176 [05:26<02:39, 107.81it/s]
 66%|██████▋   | 33280/50176 [05:28<02:37, 107.32it/s]
 67%|██████▋   | 33536/50176 [05:31<02:43, 101.89it/s]
 67%|██████▋   | 33792/50176 [05:34<02:37, 104.05it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 105.99it/s]
 68%|██████▊   | 34304/50176 [05:38<02:26, 108.60it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.62it/s]
 69%|██████▉   | 34816/50176 [05:43<02:27, 104.16it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.80it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.22it/s]
 71%|███████   | 35584/50176 [05:50<02:16, 106.71it/s]
 71%|███████▏  | 35840/50176 [05:53<02:18, 103.27it/s]
 72%|███████▏  | 36096/50176 [05:55<02:13, 105.13it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.42it/s]
 73%|███████▎  | 36608/50176 [06:00<02:09, 104.64it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.78it/s]
 74%|███████▍  | 37120/50176 [06:05<02:06, 103.41it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 107.01it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.16it/s]
 76%|███████▌  | 37888/50176 [06:12<01:55, 106.75it/s]
 76%|███████▌  | 38144/50176 [06:15<01:52, 107.35it/s]
 77%|███████▋  | 38400/50176 [06:17<01:52, 104.79it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.40it/s]
 78%|███████▊  | 38912/50176 [06:22<01:47, 105.27it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.44it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.13it/s]
 79%|███████▉  | 39680/50176 [06:29<01:38, 106.93it/s]
 80%|███████▉  | 39936/50176 [06:31<01:33, 109.57it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.14it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 104.10it/s]
 81%|████████  | 40704/50176 [06:39<01:33, 101.69it/s]
 82%|████████▏ | 40960/50176 [06:41<01:26, 106.44it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 104.91it/s]
 83%|████████▎ | 41472/50176 [06:46<01:22, 105.05it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.69it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.90it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.49it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.30it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 107.87it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.01it/s]
 86%|████████▌ | 43264/50176 [07:03<01:05, 105.95it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.42it/s]
 87%|████████▋ | 43776/50176 [07:08<01:02, 102.64it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 104.28it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.93it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.62it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.55it/s]
 90%|████████▉ | 45056/50176 [07:20<00:48, 105.04it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.69it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 108.09it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.69it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.10it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 103.91it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.30it/s]
 93%|█████████▎| 46848/50176 [07:38<00:32, 103.77it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 99.11it/s] 
 94%|█████████▍| 47360/50176 [07:43<00:28, 100.04it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 98.70it/s] 
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.93it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 102.65it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.74it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.00it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.30it/s]
 98%|█████████▊| 49152/50176 [08:00<00:10, 102.16it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.47it/s]
 99%|█████████▉| 49664/50176 [08:05<00:05, 101.42it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 96.14it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.76it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.49it/s]
+2022-12-21,10:53:05 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:05 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6278	imagenet-zeroshot-val-top5: 0.8803
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt
new file mode 100644
index 0000000000000000000000000000000000000000..cd30b58789c73e3c4e4cf5ca1383fb5759b9986a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0bc96fdd2839f5707510b857032bde852d484a3ae5e4d82107738bdbb113abf6
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1d4ee4d1ea9433b7e23eee5ff770648090803a11
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt).
+2022-12-21,10:53:30 | INFO | Model:
+2022-12-21,10:53:30 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:30 | INFO | Params:
+2022-12-21,10:53:30 | INFO |   batch_size: 256
+2022-12-21,10:53:30 | INFO |   beta1: 0.9
+2022-12-21,10:53:30 | INFO |   beta2: 0.98
+2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/checkpoints
+2022-12-21,10:53:30 | INFO |   copy_codebase: False
+2022-12-21,10:53:30 | INFO |   csv_caption_key: title
+2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:30 | INFO |   csv_separator: 	
+2022-12-21,10:53:30 | INFO |   dataset_resampled: False
+2022-12-21,10:53:30 | INFO |   dataset_type: auto
+2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:30 | INFO |   debug: False
+2022-12-21,10:53:30 | INFO |   device: cuda:0
+2022-12-21,10:53:30 | INFO |   dist_backend: nccl
+2022-12-21,10:53:30 | INFO |   dist_url: env://
+2022-12-21,10:53:30 | INFO |   distributed: False
+2022-12-21,10:53:30 | INFO |   epochs: 32
+2022-12-21,10:53:30 | INFO |   eps: 1e-06
+2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:30 | INFO |   gather_with_grad: False
+2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:30 | INFO |   horovod: False
+2022-12-21,10:53:30 | INFO |   image_mean: None
+2022-12-21,10:53:30 | INFO |   image_std: None
+2022-12-21,10:53:30 | INFO |   imagenet_v2: None
+2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:30 | INFO |   local_loss: False
+2022-12-21,10:53:30 | INFO |   local_rank: 0
+2022-12-21,10:53:30 | INFO |   lock_image: False
+2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:30 | INFO |   log_level: 20
+2022-12-21,10:53:30 | INFO |   log_local: False
+2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/out.log
+2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:30 | INFO |   lr: 0.0005
+2022-12-21,10:53:30 | INFO |   model: ViT-B-16
+2022-12-21,10:53:30 | INFO |   name: epoch_46.pt
+2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt
+2022-12-21,10:53:30 | INFO |   pretrained_image: False
+2022-12-21,10:53:30 | INFO |   rank: 0
+2022-12-21,10:53:30 | INFO |   report_to: 
+2022-12-21,10:53:30 | INFO |   resume: None
+2022-12-21,10:53:30 | INFO |   save_frequency: 1
+2022-12-21,10:53:30 | INFO |   save_most_recent: False
+2022-12-21,10:53:30 | INFO |   seed: 0
+2022-12-21,10:53:30 | INFO |   skip_scheduler: False
+2022-12-21,10:53:30 | INFO |   tensorboard: False
+2022-12-21,10:53:30 | INFO |   tensorboard_path: 
+2022-12-21,10:53:30 | INFO |   torchscript: False
+2022-12-21,10:53:30 | INFO |   trace: False
+2022-12-21,10:53:30 | INFO |   train_data: None
+2022-12-21,10:53:30 | INFO |   train_num_samples: None
+2022-12-21,10:53:30 | INFO |   use_bn_sync: False
+2022-12-21,10:53:30 | INFO |   val_data: None
+2022-12-21,10:53:30 | INFO |   val_frequency: 1
+2022-12-21,10:53:30 | INFO |   val_num_samples: None
+2022-12-21,10:53:30 | INFO |   wandb: False
+2022-12-21,10:53:30 | INFO |   wandb_notes: 
+2022-12-21,10:53:30 | INFO |   warmup: 10000
+2022-12-21,10:53:30 | INFO |   wd: 0.2
+2022-12-21,10:53:30 | INFO |   workers: 1
+2022-12-21,10:53:30 | INFO |   world_size: 1
+2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:43,  1.42it/s]
  1%|          | 6/1000 [00:00<01:43,  9.58it/s]
  1%|          | 12/1000 [00:00<00:52, 18.83it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.30it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.36it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.12it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.53it/s]
  4%|▍         | 42/1000 [00:01<00:19, 47.92it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.49it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.54it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.58it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.28it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.79it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.12it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.45it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.87it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.06it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.32it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.45it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.43it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.36it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.39it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.35it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.43it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.55it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.57it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.45it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.33it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.33it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.29it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.22it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.17it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.10it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.90it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.75it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.85it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.80it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.01it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.02it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.94it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.01it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.11it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.22it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.26it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.06it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.09it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.28it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.53it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.50it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.47it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.63it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.54it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.44it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.26it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.34it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.46it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.54it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.34it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.34it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.42it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.50it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.39it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.09it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.16it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.26it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.40it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.46it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.44it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.46it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.46it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.58it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.59it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.38it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.43it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.46it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.45it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.43it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.60it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.53it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.52it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.51it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.41it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.46it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.37it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.44it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.48it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.56it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.43it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.57it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.58it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.50it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.50it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.57it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.45it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.40it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.55it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.53it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.43it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.55it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.46it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.46it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.59it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.57it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.65it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.47it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.40it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.55it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.50it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.59it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.64it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.58it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.61it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.60it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.62it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.52it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.61it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.59it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.57it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.57it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.52it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.32it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.38it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.40it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.53it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.60it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.45it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.57it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.54it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.58it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.62it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.43it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.42it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.48it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.46it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.43it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.45it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.55it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.68it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.72it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.62it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.67it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.62it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.65it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.60it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.66it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.66it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.53it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.63it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.62it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.58it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.59it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.54it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.54it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.64it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.68it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.68it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.70it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.71it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.65it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.72it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.79it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.57it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.32it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.28it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.14it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.15it/s]
+2022-12-21,10:53:48 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<08:48, 94.45it/s]
  1%|          | 512/50176 [00:05<08:47, 94.08it/s]
  2%|▏         | 768/50176 [00:07<08:21, 98.48it/s]
  2%|▏         | 1024/50176 [00:10<08:19, 98.49it/s]
  3%|▎         | 1280/50176 [00:13<08:31, 95.52it/s]
  3%|▎         | 1536/50176 [00:16<09:07, 88.79it/s]
  4%|▎         | 1792/50176 [00:19<08:46, 91.82it/s]
  4%|▍         | 2048/50176 [00:21<08:23, 95.64it/s]
  5%|▍         | 2304/50176 [00:24<08:06, 98.31it/s]
  5%|▌         | 2560/50176 [00:26<07:56, 99.84it/s]
  6%|▌         | 2816/50176 [00:29<08:18, 95.09it/s]
  6%|▌         | 3072/50176 [00:32<08:04, 97.15it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.28it/s]
  7%|▋         | 3584/50176 [00:36<07:42, 100.71it/s]
  8%|▊         | 3840/50176 [00:39<07:52, 97.96it/s] 
  8%|▊         | 4096/50176 [00:42<08:04, 95.11it/s]
  9%|▊         | 4352/50176 [00:45<07:49, 97.50it/s]
  9%|▉         | 4608/50176 [00:47<07:34, 100.25it/s]
 10%|▉         | 4864/50176 [00:49<07:32, 100.06it/s]
 10%|█         | 5120/50176 [00:52<07:21, 101.97it/s]
 11%|█         | 5376/50176 [00:54<07:23, 100.93it/s]
 11%|█         | 5632/50176 [00:57<07:32, 98.52it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.18it/s]
 12%|█▏        | 6144/50176 [01:03<07:56, 92.34it/s]
 13%|█▎        | 6400/50176 [01:06<07:41, 94.94it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.35it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.94it/s]
 14%|█▍        | 7168/50176 [01:13<07:16, 98.64it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.64it/s]
 15%|█▌        | 7680/50176 [01:18<07:03, 100.45it/s]
 16%|█▌        | 7936/50176 [01:20<06:50, 102.85it/s]
 16%|█▋        | 8192/50176 [01:23<06:48, 102.74it/s]
 17%|█▋        | 8448/50176 [01:25<06:44, 103.14it/s]
 17%|█▋        | 8704/50176 [01:28<06:54, 99.96it/s] 
 18%|█▊        | 8960/50176 [01:31<06:54, 99.46it/s]
 18%|█▊        | 9216/50176 [01:33<06:47, 100.63it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.88it/s]
 19%|█▉        | 9728/50176 [01:38<06:48, 99.07it/s] 
 20%|█▉        | 9984/50176 [01:41<06:51, 97.71it/s]
 20%|██        | 10240/50176 [01:43<06:30, 102.21it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.46it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.09it/s] 
 22%|██▏       | 11008/50176 [01:51<06:33, 99.48it/s]
 22%|██▏       | 11264/50176 [01:54<06:21, 102.00it/s]
 23%|██▎       | 11520/50176 [01:56<06:21, 101.29it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.79it/s]
 24%|██▍       | 12032/50176 [02:01<06:16, 101.43it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.91it/s]
 25%|██▌       | 12544/50176 [02:06<06:09, 101.82it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.68it/s]
 26%|██▌       | 13056/50176 [02:11<06:01, 102.82it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.09it/s]
 27%|██▋       | 13568/50176 [02:16<06:11, 98.50it/s] 
 28%|██▊       | 13824/50176 [02:19<06:19, 95.70it/s]
 28%|██▊       | 14080/50176 [02:22<06:19, 95.17it/s]
 29%|██▊       | 14336/50176 [02:25<06:21, 93.97it/s]
 29%|██▉       | 14592/50176 [02:27<06:07, 96.91it/s]
 30%|██▉       | 14848/50176 [02:30<06:20, 92.94it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.29it/s]
 31%|███       | 15360/50176 [02:35<05:58, 97.24it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.78it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.07it/s]
 32%|███▏      | 16128/50176 [02:43<05:43, 99.18it/s]
 33%|███▎      | 16384/50176 [02:46<05:38, 99.91it/s]
 33%|███▎      | 16640/50176 [02:48<05:48, 96.25it/s]
 34%|███▎      | 16896/50176 [02:51<05:41, 97.43it/s]
 34%|███▍      | 17152/50176 [02:53<05:30, 99.94it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 101.02it/s]
 35%|███▌      | 17664/50176 [02:58<05:16, 102.59it/s]
 36%|███▌      | 17920/50176 [03:01<05:11, 103.43it/s]
 36%|███▌      | 18176/50176 [03:03<05:09, 103.43it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.34it/s]
 37%|███▋      | 18688/50176 [03:08<05:03, 103.74it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.64it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.12it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.95it/s]
 39%|███▉      | 19712/50176 [03:19<05:17, 95.82it/s]
 40%|███▉      | 19968/50176 [03:22<05:10, 97.42it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.29it/s]
 41%|████      | 20480/50176 [03:26<04:43, 104.62it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.05it/s]
 42%|████▏     | 20992/50176 [03:31<04:32, 107.28it/s]
 42%|████▏     | 21248/50176 [03:33<04:35, 105.08it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.33it/s]
 43%|████▎     | 21760/50176 [03:38<04:27, 106.38it/s]
 44%|████▍     | 22016/50176 [03:40<04:20, 108.09it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.19it/s]
 45%|████▍     | 22528/50176 [03:45<04:15, 108.10it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.18it/s]
 46%|████▌     | 23040/50176 [03:50<04:26, 101.92it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.59it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.88it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.34it/s]
 48%|████▊     | 24064/50176 [04:00<04:14, 102.53it/s]
 48%|████▊     | 24320/50176 [04:03<04:16, 100.92it/s]
 49%|████▉     | 24576/50176 [04:06<04:12, 101.23it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 105.05it/s]
 50%|█████     | 25088/50176 [04:10<03:56, 106.28it/s]
 51%|█████     | 25344/50176 [04:12<03:51, 107.46it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.17it/s]
 52%|█████▏    | 25856/50176 [04:17<03:46, 107.55it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.20it/s]
 53%|█████▎    | 26368/50176 [04:22<03:44, 105.95it/s]
 53%|█████▎    | 26624/50176 [04:25<03:42, 105.66it/s]
 54%|█████▎    | 26880/50176 [04:27<03:41, 105.06it/s]
 54%|█████▍    | 27136/50176 [04:29<03:40, 104.66it/s]
 55%|█████▍    | 27392/50176 [04:32<03:33, 106.47it/s]
 55%|█████▌    | 27648/50176 [04:34<03:25, 109.52it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.34it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.00it/s]
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.55it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 101.08it/s]
 58%|█████▊    | 28928/50176 [04:47<03:28, 102.15it/s]
 58%|█████▊    | 29184/50176 [04:49<03:23, 103.23it/s]
 59%|█████▊    | 29440/50176 [04:52<03:30, 98.65it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.97it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.69it/s]
 60%|██████    | 30208/50176 [05:00<03:22, 98.65it/s]
 61%|██████    | 30464/50176 [05:02<03:07, 105.27it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 102.98it/s]
 62%|██████▏   | 30976/50176 [05:07<03:03, 104.85it/s]
 62%|██████▏   | 31232/50176 [05:09<02:58, 105.97it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.41it/s]
 63%|██████▎   | 31744/50176 [05:14<02:49, 108.55it/s]
 64%|██████▍   | 32000/50176 [05:16<02:47, 108.25it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.80it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.31it/s]
 65%|██████▌   | 32768/50176 [05:24<02:46, 104.67it/s]
 66%|██████▌   | 33024/50176 [05:26<02:44, 103.97it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.58it/s]
 67%|██████▋   | 33536/50176 [05:31<02:41, 102.98it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.68it/s]
 68%|██████▊   | 34048/50176 [05:36<02:30, 107.46it/s]
 68%|██████▊   | 34304/50176 [05:38<02:22, 111.43it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.42it/s]
 69%|██████▉   | 34816/50176 [05:43<02:25, 105.91it/s]
 70%|██████▉   | 35072/50176 [05:46<02:23, 105.60it/s]
 70%|███████   | 35328/50176 [05:48<02:20, 106.02it/s]
 71%|███████   | 35584/50176 [05:50<02:16, 106.82it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.14it/s]
 72%|███████▏  | 36096/50176 [05:55<02:12, 106.25it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.59it/s]
 73%|███████▎  | 36608/50176 [06:00<02:11, 103.01it/s]
 73%|███████▎  | 36864/50176 [06:03<02:12, 100.28it/s]
 74%|███████▍  | 37120/50176 [06:05<02:06, 102.83it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.50it/s]
 75%|███████▌  | 37632/50176 [06:10<01:58, 105.85it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.07it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.43it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.70it/s]
 77%|███████▋  | 38656/50176 [06:20<01:49, 105.62it/s]
 78%|███████▊  | 38912/50176 [06:22<01:47, 105.20it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.29it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 105.93it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.36it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.85it/s]
 80%|████████  | 40192/50176 [06:34<01:34, 105.87it/s]
 81%|████████  | 40448/50176 [06:37<01:34, 102.64it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.17it/s]
 82%|████████▏ | 40960/50176 [06:42<01:27, 104.81it/s]
 82%|████████▏ | 41216/50176 [06:44<01:24, 105.59it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 104.07it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 106.69it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.98it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.86it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 108.82it/s]
 85%|████████▌ | 42752/50176 [06:58<01:07, 109.61it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.14it/s]
 86%|████████▌ | 43264/50176 [07:03<01:04, 107.38it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.44it/s]
 87%|████████▋ | 43776/50176 [07:08<01:01, 103.23it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 105.81it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.88it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 104.04it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.95it/s]
 90%|████████▉ | 45056/50176 [07:20<00:47, 107.92it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.14it/s]
 91%|█████████ | 45568/50176 [07:25<00:43, 106.72it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.97it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 104.84it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.62it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 104.76it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.17it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 101.19it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.10it/s]
 95%|█████████▍| 47616/50176 [07:45<00:25, 100.78it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.73it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 102.51it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.60it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.97it/s]
 97%|█████████▋| 48896/50176 [07:57<00:12, 106.32it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.89it/s]
 98%|█████████▊| 49408/50176 [08:02<00:07, 105.54it/s]
 99%|█████████▉| 49664/50176 [08:05<00:04, 102.62it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.24it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 126.44it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.60it/s]
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6270	imagenet-zeroshot-val-top5: 0.8812
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1e72ad8f50f4f6d3ffb722e655fd89e537b42209
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ad1e1d28310eb4fd9497a77ead21ba20f9fecb8ed8d7c83fb9b67b97c87263f7
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0ca462130d0a191206143b66b2e86e3842a29757
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt).
+2022-12-21,10:53:30 | INFO | Model:
+2022-12-21,10:53:30 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:30 | INFO | Params:
+2022-12-21,10:53:30 | INFO |   batch_size: 256
+2022-12-21,10:53:30 | INFO |   beta1: 0.9
+2022-12-21,10:53:30 | INFO |   beta2: 0.98
+2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/checkpoints
+2022-12-21,10:53:30 | INFO |   copy_codebase: False
+2022-12-21,10:53:30 | INFO |   csv_caption_key: title
+2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:30 | INFO |   csv_separator: 	
+2022-12-21,10:53:30 | INFO |   dataset_resampled: False
+2022-12-21,10:53:30 | INFO |   dataset_type: auto
+2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:30 | INFO |   debug: False
+2022-12-21,10:53:30 | INFO |   device: cuda:0
+2022-12-21,10:53:30 | INFO |   dist_backend: nccl
+2022-12-21,10:53:30 | INFO |   dist_url: env://
+2022-12-21,10:53:30 | INFO |   distributed: False
+2022-12-21,10:53:30 | INFO |   epochs: 32
+2022-12-21,10:53:30 | INFO |   eps: 1e-06
+2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:30 | INFO |   gather_with_grad: False
+2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:30 | INFO |   horovod: False
+2022-12-21,10:53:30 | INFO |   image_mean: None
+2022-12-21,10:53:30 | INFO |   image_std: None
+2022-12-21,10:53:30 | INFO |   imagenet_v2: None
+2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:30 | INFO |   local_loss: False
+2022-12-21,10:53:30 | INFO |   local_rank: 0
+2022-12-21,10:53:30 | INFO |   lock_image: False
+2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:30 | INFO |   log_level: 20
+2022-12-21,10:53:30 | INFO |   log_local: False
+2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/out.log
+2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:30 | INFO |   lr: 0.0005
+2022-12-21,10:53:30 | INFO |   model: ViT-B-16
+2022-12-21,10:53:30 | INFO |   name: epoch_47.pt
+2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt
+2022-12-21,10:53:30 | INFO |   pretrained_image: False
+2022-12-21,10:53:30 | INFO |   rank: 0
+2022-12-21,10:53:30 | INFO |   report_to: 
+2022-12-21,10:53:30 | INFO |   resume: None
+2022-12-21,10:53:30 | INFO |   save_frequency: 1
+2022-12-21,10:53:30 | INFO |   save_most_recent: False
+2022-12-21,10:53:30 | INFO |   seed: 0
+2022-12-21,10:53:30 | INFO |   skip_scheduler: False
+2022-12-21,10:53:30 | INFO |   tensorboard: False
+2022-12-21,10:53:30 | INFO |   tensorboard_path: 
+2022-12-21,10:53:30 | INFO |   torchscript: False
+2022-12-21,10:53:30 | INFO |   trace: False
+2022-12-21,10:53:30 | INFO |   train_data: None
+2022-12-21,10:53:30 | INFO |   train_num_samples: None
+2022-12-21,10:53:30 | INFO |   use_bn_sync: False
+2022-12-21,10:53:30 | INFO |   val_data: None
+2022-12-21,10:53:30 | INFO |   val_frequency: 1
+2022-12-21,10:53:30 | INFO |   val_num_samples: None
+2022-12-21,10:53:30 | INFO |   wandb: False
+2022-12-21,10:53:30 | INFO |   wandb_notes: 
+2022-12-21,10:53:30 | INFO |   warmup: 10000
+2022-12-21,10:53:30 | INFO |   wd: 0.2
+2022-12-21,10:53:30 | INFO |   workers: 1
+2022-12-21,10:53:30 | INFO |   world_size: 1
+2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:23,  1.60it/s]
  1%|          | 6/1000 [00:00<01:33, 10.62it/s]
  1%|          | 12/1000 [00:00<00:48, 20.47it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.23it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.34it/s]
  3%|▎         | 30/1000 [00:01<00:23, 42.04it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.16it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.31it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.67it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.46it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.07it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.06it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.81it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.28it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.78it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.86it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.28it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.49it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.81it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.85it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.82it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.85it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.91it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.93it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.80it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.66it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.67it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.65it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.47it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.45it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.44it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.26it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.09it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.22it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.23it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.25it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.26it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.22it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.32it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.43it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.58it/s]
 26%|██▋       | 264/1000 [00:05<00:15, 47.83it/s]
 27%|██▋       | 270/1000 [00:05<00:14, 50.25it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 52.21it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 53.80it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 54.96it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 55.92it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.46it/s]
 31%|███       | 306/1000 [00:06<00:12, 56.84it/s]
 31%|███       | 312/1000 [00:06<00:12, 57.27it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.44it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.46it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.36it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.54it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.73it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.88it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.70it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.73it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.82it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.93it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.51it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.58it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.70it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.85it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.93it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.90it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.92it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.94it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.06it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.06it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.84it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.94it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.91it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.87it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.02it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.92it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.91it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.91it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.81it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.84it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.74it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.89it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.97it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.92it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.01it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.99it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.86it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.86it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.91it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.81it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.75it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.87it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.87it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.81it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.92it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.83it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.84it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.95it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.96it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.04it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.88it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.82it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.72it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.73it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.81it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.88it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.83it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.90it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.92it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.84it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.94it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.94it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.92it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.89it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.89it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.90it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.90it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.98it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.06it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.90it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.01it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.95it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.97it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.02it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.80it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.78it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.83it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.79it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.78it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.81it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.92it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.02it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.07it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.99it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.03it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.99it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.00it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.94it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.99it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 57.98it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.85it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.96it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.93it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.95it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.94it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.93it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.98it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.05it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.09it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.11it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.05it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.10it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.17it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.11it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.94it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.82it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.56it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.55it/s]
+2022-12-21,10:53:48 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:13, 90.19it/s]
  1%|          | 512/50176 [00:05<08:58, 92.17it/s]
  2%|▏         | 768/50176 [00:08<08:27, 97.31it/s]
  2%|▏         | 1024/50176 [00:10<08:23, 97.69it/s]
  3%|▎         | 1280/50176 [00:13<08:34, 95.07it/s]
  3%|▎         | 1536/50176 [00:16<09:09, 88.50it/s]
  4%|▎         | 1792/50176 [00:19<08:47, 91.67it/s]
  4%|▍         | 2048/50176 [00:21<08:24, 95.43it/s]
  5%|▍         | 2304/50176 [00:24<08:06, 98.33it/s]
  5%|▌         | 2560/50176 [00:26<07:58, 99.61it/s]
  6%|▌         | 2816/50176 [00:29<08:18, 95.07it/s]
  6%|▌         | 3072/50176 [00:32<08:05, 97.00it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.25it/s]
  7%|▋         | 3584/50176 [00:37<07:42, 100.74it/s]
  8%|▊         | 3840/50176 [00:39<07:53, 97.88it/s] 
  8%|▊         | 4096/50176 [00:42<08:04, 95.10it/s]
  9%|▊         | 4352/50176 [00:45<07:49, 97.62it/s]
  9%|▉         | 4608/50176 [00:47<07:34, 100.23it/s]
 10%|▉         | 4864/50176 [00:50<07:33, 99.95it/s] 
 10%|█         | 5120/50176 [00:52<07:21, 101.95it/s]
 11%|█         | 5376/50176 [00:55<07:24, 100.84it/s]
 11%|█         | 5632/50176 [00:57<07:31, 98.64it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.11it/s]
 12%|█▏        | 6144/50176 [01:03<07:56, 92.34it/s]
 13%|█▎        | 6400/50176 [01:06<07:40, 95.10it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.33it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.94it/s]
 14%|█▍        | 7168/50176 [01:13<07:16, 98.49it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.63it/s]
 15%|█▌        | 7680/50176 [01:18<07:03, 100.32it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.99it/s]
 16%|█▋        | 8192/50176 [01:23<06:49, 102.53it/s]
 17%|█▋        | 8448/50176 [01:26<06:44, 103.19it/s]
 17%|█▋        | 8704/50176 [01:28<06:54, 100.05it/s]
 18%|█▊        | 8960/50176 [01:31<06:53, 99.68it/s] 
 18%|█▊        | 9216/50176 [01:33<06:47, 100.50it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.94it/s]
 19%|█▉        | 9728/50176 [01:39<06:48, 99.02it/s] 
 20%|█▉        | 9984/50176 [01:41<06:50, 97.85it/s]
 20%|██        | 10240/50176 [01:44<06:30, 102.17it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.40it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.19it/s] 
 22%|██▏       | 11008/50176 [01:51<06:34, 99.39it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.17it/s]
 23%|██▎       | 11520/50176 [01:56<06:21, 101.34it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.80it/s]
 24%|██▍       | 12032/50176 [02:01<06:16, 101.45it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.80it/s]
 25%|██▌       | 12544/50176 [02:06<06:09, 101.81it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.75it/s]
 26%|██▌       | 13056/50176 [02:11<06:00, 102.97it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.08it/s]
 27%|██▋       | 13568/50176 [02:17<06:11, 98.49it/s] 
 28%|██▊       | 13824/50176 [02:19<06:19, 95.72it/s]
 28%|██▊       | 14080/50176 [02:22<06:19, 95.10it/s]
 29%|██▊       | 14336/50176 [02:25<06:20, 94.09it/s]
 29%|██▉       | 14592/50176 [02:27<06:07, 96.85it/s]
 30%|██▉       | 14848/50176 [02:30<06:20, 92.85it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.30it/s]
 31%|███       | 15360/50176 [02:36<05:58, 97.15it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.84it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.14it/s]
 32%|███▏      | 16128/50176 [02:43<05:43, 99.12it/s]
 33%|███▎      | 16384/50176 [02:46<05:37, 99.98it/s]
 33%|███▎      | 16640/50176 [02:49<05:47, 96.37it/s]
 34%|███▎      | 16896/50176 [02:51<05:41, 97.42it/s]
 34%|███▍      | 17152/50176 [02:54<05:30, 99.83it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 101.01it/s]
 35%|███▌      | 17664/50176 [02:58<05:17, 102.55it/s]
 36%|███▌      | 17920/50176 [03:01<05:11, 103.42it/s]
 36%|███▌      | 18176/50176 [03:03<05:09, 103.45it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.29it/s]
 37%|███▋      | 18688/50176 [03:08<05:03, 103.65it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.73it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.12it/s] 
 39%|███▉      | 19456/50176 [03:16<05:19, 96.00it/s]
 39%|███▉      | 19712/50176 [03:19<05:18, 95.68it/s]
 40%|███▉      | 19968/50176 [03:22<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.35it/s]
 41%|████      | 20480/50176 [03:26<04:44, 104.46it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.07it/s]
 42%|████▏     | 20992/50176 [03:31<04:31, 107.31it/s]
 42%|████▏     | 21248/50176 [03:33<04:35, 104.98it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.36it/s]
 43%|████▎     | 21760/50176 [03:38<04:26, 106.48it/s]
 44%|████▍     | 22016/50176 [03:41<04:20, 107.99it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.20it/s]
 45%|████▍     | 22528/50176 [03:45<04:15, 108.03it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.20it/s]
 46%|████▌     | 23040/50176 [03:51<04:25, 102.08it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.58it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.86it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.33it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.53it/s]
 48%|████▊     | 24320/50176 [04:03<04:16, 100.96it/s]
 49%|████▉     | 24576/50176 [04:06<04:13, 101.12it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 104.95it/s]
 50%|█████     | 25088/50176 [04:10<03:55, 106.33it/s]
 51%|█████     | 25344/50176 [04:13<03:51, 107.41it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.22it/s]
 52%|█████▏    | 25856/50176 [04:17<03:45, 107.65it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.21it/s]
 53%|█████▎    | 26368/50176 [04:22<03:44, 105.90it/s]
 53%|█████▎    | 26624/50176 [04:25<03:43, 105.60it/s]
 54%|█████▎    | 26880/50176 [04:27<03:41, 105.16it/s]
 54%|█████▍    | 27136/50176 [04:30<03:40, 104.66it/s]
 55%|█████▍    | 27392/50176 [04:32<03:34, 106.39it/s]
 55%|█████▌    | 27648/50176 [04:34<03:25, 109.61it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.28it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.06it/s]
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.57it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 100.96it/s]
 58%|█████▊    | 28928/50176 [04:47<03:27, 102.16it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.25it/s]
 59%|█████▊    | 29440/50176 [04:52<03:30, 98.73it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.85it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.81it/s]
 60%|██████    | 30208/50176 [05:00<03:22, 98.58it/s]
 61%|██████    | 30464/50176 [05:02<03:07, 105.30it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 103.03it/s]
 62%|██████▏   | 30976/50176 [05:07<03:03, 104.72it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.14it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.51it/s]
 63%|██████▎   | 31744/50176 [05:14<02:49, 108.55it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.22it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.81it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.17it/s]
 65%|██████▌   | 32768/50176 [05:24<02:46, 104.68it/s]
 66%|██████▌   | 33024/50176 [05:27<02:45, 103.90it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.55it/s]
 67%|██████▋   | 33536/50176 [05:32<02:41, 103.12it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.68it/s]
 68%|██████▊   | 34048/50176 [05:36<02:30, 107.29it/s]
 68%|██████▊   | 34304/50176 [05:38<02:22, 111.59it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.55it/s]
 69%|██████▉   | 34816/50176 [05:43<02:24, 106.02it/s]
 70%|██████▉   | 35072/50176 [05:46<02:23, 105.54it/s]
 70%|███████   | 35328/50176 [05:48<02:19, 106.15it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.71it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.10it/s]
 72%|███████▏  | 36096/50176 [05:55<02:12, 106.30it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.63it/s]
 73%|███████▎  | 36608/50176 [06:01<02:11, 102.85it/s]
 73%|███████▎  | 36864/50176 [06:03<02:12, 100.30it/s]
 74%|███████▍  | 37120/50176 [06:06<02:07, 102.67it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.46it/s]
 75%|███████▌  | 37632/50176 [06:10<01:58, 105.78it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.13it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.29it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.74it/s]
 77%|███████▋  | 38656/50176 [06:20<01:49, 105.67it/s]
 78%|███████▊  | 38912/50176 [06:22<01:46, 105.40it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.35it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 105.99it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.34it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.87it/s]
 80%|████████  | 40192/50176 [06:35<01:34, 105.97it/s]
 81%|████████  | 40448/50176 [06:37<01:34, 102.51it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.20it/s]
 82%|████████▏ | 40960/50176 [06:42<01:28, 104.63it/s]
 82%|████████▏ | 41216/50176 [06:44<01:24, 105.78it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 103.94it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 106.73it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.94it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.79it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 108.85it/s]
 85%|████████▌ | 42752/50176 [06:59<01:07, 109.73it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.23it/s]
 86%|████████▌ | 43264/50176 [07:03<01:04, 107.43it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.24it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 103.33it/s]
 88%|████████▊ | 44032/50176 [07:11<00:57, 105.94it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.78it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 103.91it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.84it/s]
 90%|████████▉ | 45056/50176 [07:20<00:47, 107.92it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.11it/s]
 91%|█████████ | 45568/50176 [07:25<00:43, 106.82it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.92it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 104.80it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.51it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 104.95it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.20it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 101.16it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.15it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 100.87it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.79it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 102.52it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.62it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 105.00it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 106.13it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.99it/s]
 98%|█████████▊| 49408/50176 [08:02<00:07, 105.52it/s]
 99%|█████████▉| 49664/50176 [08:05<00:04, 102.63it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.23it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 126.73it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.57it/s]
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6319	imagenet-zeroshot-val-top5: 0.8823
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt
new file mode 100644
index 0000000000000000000000000000000000000000..38e76144eba717fd195a15fb122ec171577460e2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:668f0d64e47e0bc50252527b8932985da8efab3ced65e79fab530024498f14fe
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..933fbb792346b3637fcb04b176c6eb36ad0cf082
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt).
+2022-12-21,10:53:29 | INFO | Model:
+2022-12-21,10:53:29 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:29 | INFO | Params:
+2022-12-21,10:53:29 | INFO |   batch_size: 256
+2022-12-21,10:53:29 | INFO |   beta1: 0.9
+2022-12-21,10:53:29 | INFO |   beta2: 0.98
+2022-12-21,10:53:29 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/checkpoints
+2022-12-21,10:53:29 | INFO |   copy_codebase: False
+2022-12-21,10:53:29 | INFO |   csv_caption_key: title
+2022-12-21,10:53:29 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:29 | INFO |   csv_separator: 	
+2022-12-21,10:53:29 | INFO |   dataset_resampled: False
+2022-12-21,10:53:29 | INFO |   dataset_type: auto
+2022-12-21,10:53:29 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:29 | INFO |   debug: False
+2022-12-21,10:53:29 | INFO |   device: cuda:0
+2022-12-21,10:53:29 | INFO |   dist_backend: nccl
+2022-12-21,10:53:29 | INFO |   dist_url: env://
+2022-12-21,10:53:29 | INFO |   distributed: False
+2022-12-21,10:53:29 | INFO |   epochs: 32
+2022-12-21,10:53:29 | INFO |   eps: 1e-06
+2022-12-21,10:53:29 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:29 | INFO |   gather_with_grad: False
+2022-12-21,10:53:29 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:29 | INFO |   horovod: False
+2022-12-21,10:53:29 | INFO |   image_mean: None
+2022-12-21,10:53:29 | INFO |   image_std: None
+2022-12-21,10:53:29 | INFO |   imagenet_v2: None
+2022-12-21,10:53:29 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:29 | INFO |   local_loss: False
+2022-12-21,10:53:29 | INFO |   local_rank: 0
+2022-12-21,10:53:29 | INFO |   lock_image: False
+2022-12-21,10:53:29 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:29 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:29 | INFO |   log_level: 20
+2022-12-21,10:53:29 | INFO |   log_local: False
+2022-12-21,10:53:29 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/out.log
+2022-12-21,10:53:29 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:29 | INFO |   lr: 0.0005
+2022-12-21,10:53:29 | INFO |   model: ViT-B-16
+2022-12-21,10:53:29 | INFO |   name: epoch_48.pt
+2022-12-21,10:53:29 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:29 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:29 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:29 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt
+2022-12-21,10:53:29 | INFO |   pretrained_image: False
+2022-12-21,10:53:29 | INFO |   rank: 0
+2022-12-21,10:53:29 | INFO |   report_to: 
+2022-12-21,10:53:29 | INFO |   resume: None
+2022-12-21,10:53:29 | INFO |   save_frequency: 1
+2022-12-21,10:53:29 | INFO |   save_most_recent: False
+2022-12-21,10:53:29 | INFO |   seed: 0
+2022-12-21,10:53:29 | INFO |   skip_scheduler: False
+2022-12-21,10:53:29 | INFO |   tensorboard: False
+2022-12-21,10:53:29 | INFO |   tensorboard_path: 
+2022-12-21,10:53:29 | INFO |   torchscript: False
+2022-12-21,10:53:29 | INFO |   trace: False
+2022-12-21,10:53:29 | INFO |   train_data: None
+2022-12-21,10:53:29 | INFO |   train_num_samples: None
+2022-12-21,10:53:29 | INFO |   use_bn_sync: False
+2022-12-21,10:53:29 | INFO |   val_data: None
+2022-12-21,10:53:29 | INFO |   val_frequency: 1
+2022-12-21,10:53:29 | INFO |   val_num_samples: None
+2022-12-21,10:53:29 | INFO |   wandb: False
+2022-12-21,10:53:29 | INFO |   wandb_notes: 
+2022-12-21,10:53:29 | INFO |   warmup: 10000
+2022-12-21,10:53:29 | INFO |   wd: 0.2
+2022-12-21,10:53:29 | INFO |   workers: 1
+2022-12-21,10:53:29 | INFO |   world_size: 1
+2022-12-21,10:53:29 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:01,  1.51it/s]
  1%|          | 6/1000 [00:00<01:39, 10.01it/s]
  1%|          | 11/1000 [00:00<00:54, 18.01it/s]
  2%|▏         | 17/1000 [00:00<00:36, 27.06it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.60it/s]
  3%|▎         | 29/1000 [00:01<00:23, 40.63it/s]
  4%|▎         | 35/1000 [00:01<00:21, 45.17it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.72it/s]
  5%|▍         | 47/1000 [00:01<00:18, 51.11it/s]
  5%|▌         | 53/1000 [00:01<00:17, 53.05it/s]
  6%|▌         | 59/1000 [00:01<00:17, 54.27it/s]
  6%|▋         | 65/1000 [00:01<00:16, 55.24it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.78it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.30it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.73it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.15it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.37it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.59it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.80it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.95it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 57.98it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.93it/s]
 13%|█▎        | 131/1000 [00:02<00:14, 58.02it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.97it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.01it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.94it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.82it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.67it/s]
 19%|█▉        | 191/1000 [00:03<00:14, 57.66it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.50it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.34it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 221/1000 [00:04<00:15, 50.46it/s]
 23%|██▎       | 227/1000 [00:04<00:14, 52.51it/s]
 23%|██▎       | 233/1000 [00:04<00:14, 54.00it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 54.89it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 55.66it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 56.27it/s]
 26%|██▌       | 257/1000 [00:05<00:13, 56.85it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.16it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.12it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.61it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.76it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.94it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.97it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.02it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.17it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.16it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.12it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.93it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.05it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.17it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.98it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.12it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.05it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.19it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.03it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.78it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.86it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.00it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.13it/s]
 42%|████▏     | 419/1000 [00:07<00:09, 58.16it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.10it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.23it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.23it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.03it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.13it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.11it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.22it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.14it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.13it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.08it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.98it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.10it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.98it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.04it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.09it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.17it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.08it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.20it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.22it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.09it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.09it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.04it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.98it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.15it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 593/1000 [00:10<00:07, 58.04it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.15it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.06it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.13it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.21it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.24it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.12it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.99it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.19it/s]
 65%|██████▌   | 653/1000 [00:11<00:05, 58.11it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.21it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.20it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.21it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.26it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.13it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.23it/s]
 71%|███████   | 707/1000 [00:12<00:05, 58.24it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.23it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.26it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.19it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.16it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.17it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.38it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.11it/s]
 77%|███████▋  | 767/1000 [00:13<00:04, 58.16it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.17it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.22it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.26it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.06it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.07it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.10it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.08it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.16it/s]
 83%|████████▎ | 827/1000 [00:14<00:02, 58.15it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.33it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.41it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.26it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.30it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.24it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.28it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.21it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.26it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 58.11it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.20it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.23it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.18it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.20it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.13it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.25it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.33it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.32it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.29it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.32it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.26it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.45it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.38it/s]
100%|█████████▉| 995/1000 [00:17<00:00, 58.24it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.68it/s]
+2022-12-21,10:53:47 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:37, 78.24it/s]
  1%|          | 512/50176 [00:05<09:32, 86.79it/s]
  2%|▏         | 768/50176 [00:08<08:44, 94.18it/s]
  2%|▏         | 1024/50176 [00:11<08:33, 95.63it/s]
  3%|▎         | 1280/50176 [00:13<08:40, 93.88it/s]
  3%|▎         | 1536/50176 [00:17<09:13, 87.81it/s]
  4%|▎         | 1792/50176 [00:19<08:50, 91.13it/s]
  4%|▍         | 2048/50176 [00:22<08:25, 95.14it/s]
  5%|▍         | 2304/50176 [00:24<08:09, 97.80it/s]
  5%|▌         | 2560/50176 [00:27<07:58, 99.61it/s]
  6%|▌         | 2816/50176 [00:30<08:18, 94.98it/s]
  6%|▌         | 3072/50176 [00:32<08:05, 97.06it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.22it/s]
  7%|▋         | 3584/50176 [00:37<07:43, 100.60it/s]
  8%|▊         | 3840/50176 [00:40<07:53, 97.79it/s] 
  8%|▊         | 4096/50176 [00:43<08:04, 95.14it/s]
  9%|▊         | 4352/50176 [00:45<07:49, 97.53it/s]
  9%|▉         | 4608/50176 [00:47<07:35, 100.13it/s]
 10%|▉         | 4864/50176 [00:50<07:33, 99.81it/s] 
 10%|█         | 5120/50176 [00:52<07:21, 101.94it/s]
 11%|█         | 5376/50176 [00:55<07:23, 101.13it/s]
 11%|█         | 5632/50176 [00:58<07:31, 98.57it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.16it/s]
 12%|█▏        | 6144/50176 [01:04<07:57, 92.24it/s]
 13%|█▎        | 6400/50176 [01:06<07:40, 94.99it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.39it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.98it/s]
 14%|█▍        | 7168/50176 [01:14<07:16, 98.56it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.72it/s]
 15%|█▌        | 7680/50176 [01:19<07:03, 100.34it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.92it/s]
 16%|█▋        | 8192/50176 [01:24<06:48, 102.72it/s]
 17%|█▋        | 8448/50176 [01:26<06:44, 103.17it/s]
 17%|█▋        | 8704/50176 [01:29<06:54, 100.03it/s]
 18%|█▊        | 8960/50176 [01:31<06:54, 99.52it/s] 
 18%|█▊        | 9216/50176 [01:34<06:47, 100.59it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.84it/s]
 19%|█▉        | 9728/50176 [01:39<06:47, 99.14it/s] 
 20%|█▉        | 9984/50176 [01:42<06:52, 97.54it/s]
 20%|██        | 10240/50176 [01:44<06:30, 102.22it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.50it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.12it/s] 
 22%|██▏       | 11008/50176 [01:52<06:34, 99.41it/s]
 22%|██▏       | 11264/50176 [01:54<06:21, 102.09it/s]
 23%|██▎       | 11520/50176 [01:57<06:21, 101.29it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.84it/s]
 24%|██▍       | 12032/50176 [02:02<06:15, 101.48it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.95it/s]
 25%|██▌       | 12544/50176 [02:07<06:09, 101.76it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.71it/s]
 26%|██▌       | 13056/50176 [02:12<06:00, 102.86it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.05it/s]
 27%|██▋       | 13568/50176 [02:17<06:11, 98.43it/s] 
 28%|██▊       | 13824/50176 [02:20<06:19, 95.73it/s]
 28%|██▊       | 14080/50176 [02:23<06:18, 95.24it/s]
 29%|██▊       | 14336/50176 [02:25<06:21, 94.03it/s]
 29%|██▉       | 14592/50176 [02:28<06:07, 96.90it/s]
 30%|██▉       | 14848/50176 [02:31<06:20, 92.92it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.29it/s]
 31%|███       | 15360/50176 [02:36<05:58, 97.12it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.80it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.07it/s]
 32%|███▏      | 16128/50176 [02:44<05:43, 99.21it/s]
 33%|███▎      | 16384/50176 [02:46<05:38, 99.96it/s]
 33%|███▎      | 16640/50176 [02:49<05:48, 96.31it/s]
 34%|███▎      | 16896/50176 [02:52<05:41, 97.40it/s]
 34%|███▍      | 17152/50176 [02:54<05:30, 99.94it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 101.00it/s]
 35%|███▌      | 17664/50176 [02:59<05:16, 102.61it/s]
 36%|███▌      | 17920/50176 [03:01<05:12, 103.34it/s]
 36%|███▌      | 18176/50176 [03:04<05:09, 103.46it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.36it/s]
 37%|███▋      | 18688/50176 [03:09<05:04, 103.42it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.70it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.22it/s] 
 39%|███▉      | 19456/50176 [03:17<05:20, 95.99it/s]
 39%|███▉      | 19712/50176 [03:20<05:18, 95.74it/s]
 40%|███▉      | 19968/50176 [03:22<05:10, 97.41it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.34it/s]
 41%|████      | 20480/50176 [03:27<04:44, 104.47it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.09it/s]
 42%|████▏     | 20992/50176 [03:31<04:32, 107.29it/s]
 42%|████▏     | 21248/50176 [03:34<04:35, 104.99it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.37it/s]
 43%|████▎     | 21760/50176 [03:39<04:26, 106.46it/s]
 44%|████▍     | 22016/50176 [03:41<04:20, 108.00it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.25it/s]
 45%|████▍     | 22528/50176 [03:46<04:15, 108.09it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.20it/s]
 46%|████▌     | 23040/50176 [03:51<04:25, 102.04it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.52it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.85it/s] 
 47%|████▋     | 23808/50176 [03:59<04:17, 102.39it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.50it/s]
 48%|████▊     | 24320/50176 [04:04<04:16, 100.92it/s]
 49%|████▉     | 24576/50176 [04:06<04:13, 101.14it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 104.97it/s]
 50%|█████     | 25088/50176 [04:11<03:55, 106.38it/s]
 51%|█████     | 25344/50176 [04:13<03:51, 107.44it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.18it/s]
 52%|█████▏    | 25856/50176 [04:18<03:45, 107.72it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.12it/s]
 53%|█████▎    | 26368/50176 [04:23<03:44, 105.96it/s]
 53%|█████▎    | 26624/50176 [04:25<03:43, 105.49it/s]
 54%|█████▎    | 26880/50176 [04:28<03:41, 105.16it/s]
 54%|█████▍    | 27136/50176 [04:30<03:40, 104.68it/s]
 55%|█████▍    | 27392/50176 [04:32<03:34, 106.46it/s]
 55%|█████▌    | 27648/50176 [04:35<03:25, 109.49it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.21it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.07it/s]
 57%|█████▋    | 28416/50176 [04:43<03:34, 101.57it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 100.99it/s]
 58%|█████▊    | 28928/50176 [04:48<03:27, 102.20it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.29it/s]
 59%|█████▊    | 29440/50176 [04:53<03:30, 98.69it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:31, 96.89it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.80it/s]
 60%|██████    | 30208/50176 [05:01<03:22, 98.62it/s]
 61%|██████    | 30464/50176 [05:03<03:07, 105.29it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 103.04it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.73it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.19it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.33it/s]
 63%|██████▎   | 31744/50176 [05:15<02:49, 108.61it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.24it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.79it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.22it/s]
 65%|██████▌   | 32768/50176 [05:25<02:46, 104.76it/s]
 66%|██████▌   | 33024/50176 [05:27<02:45, 103.91it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.61it/s]
 67%|██████▋   | 33536/50176 [05:32<02:41, 103.08it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.72it/s]
 68%|██████▊   | 34048/50176 [05:37<02:30, 107.32it/s]
 68%|██████▊   | 34304/50176 [05:39<02:22, 111.59it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.48it/s]
 69%|██████▉   | 34816/50176 [05:44<02:25, 105.86it/s]
 70%|██████▉   | 35072/50176 [05:46<02:23, 105.52it/s]
 70%|███████   | 35328/50176 [05:49<02:20, 105.97it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.87it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.02it/s]
 72%|███████▏  | 36096/50176 [05:56<02:12, 106.39it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.65it/s]
 73%|███████▎  | 36608/50176 [06:01<02:11, 102.87it/s]
 73%|███████▎  | 36864/50176 [06:04<02:12, 100.19it/s]
 74%|███████▍  | 37120/50176 [06:06<02:07, 102.66it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.49it/s]
 75%|███████▌  | 37632/50176 [06:11<01:58, 105.83it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.21it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.34it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.68it/s]
 77%|███████▋  | 38656/50176 [06:20<01:48, 105.73it/s]
 78%|███████▊  | 38912/50176 [06:23<01:46, 105.37it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.26it/s]
 79%|███████▊  | 39424/50176 [06:28<01:41, 105.95it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.29it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.89it/s]
 80%|████████  | 40192/50176 [06:35<01:34, 106.02it/s]
 81%|████████  | 40448/50176 [06:38<01:34, 102.54it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.15it/s]
 82%|████████▏ | 40960/50176 [06:42<01:27, 104.75it/s]
 82%|████████▏ | 41216/50176 [06:45<01:24, 105.65it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 104.10it/s]
 83%|████████▎ | 41728/50176 [06:50<01:19, 106.80it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.95it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.80it/s]
 85%|████████▍ | 42496/50176 [06:57<01:10, 108.79it/s]
 85%|████████▌ | 42752/50176 [06:59<01:07, 109.65it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.12it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 107.38it/s]
 87%|████████▋ | 43520/50176 [07:07<01:04, 103.31it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 103.33it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 105.84it/s]
 88%|████████▊ | 44288/50176 [07:14<00:54, 107.80it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 103.90it/s]
 89%|████████▉ | 44800/50176 [07:19<00:51, 104.97it/s]
 90%|████████▉ | 45056/50176 [07:21<00:47, 107.93it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.13it/s]
 91%|█████████ | 45568/50176 [07:26<00:43, 106.74it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.94it/s]
 92%|█████████▏| 46080/50176 [07:31<00:39, 104.59it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.60it/s]
 93%|█████████▎| 46592/50176 [07:36<00:34, 105.03it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.32it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 101.16it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.11it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 100.90it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.71it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 102.52it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.59it/s]
 97%|█████████▋| 48640/50176 [07:56<00:14, 105.02it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 106.19it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.97it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 105.56it/s]
 99%|█████████▉| 49664/50176 [08:06<00:04, 102.68it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.17it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 126.76it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.48it/s]
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6387	imagenet-zeroshot-val-top5: 0.8844
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt
new file mode 100644
index 0000000000000000000000000000000000000000..5f45f28a4270ffb64f4946bec4e7f2680eadf130
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f47830888249214d59a9c8d56ed06c8a9b03c26c2033fd77e21a8bd6a6533c4
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..68decec6bd505b4fed3be75a7064f56cb7728729
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,10:53:19 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:19 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt).
+2022-12-21,10:53:30 | INFO | Model:
+2022-12-21,10:53:30 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:30 | INFO | Params:
+2022-12-21,10:53:30 | INFO |   batch_size: 256
+2022-12-21,10:53:30 | INFO |   beta1: 0.9
+2022-12-21,10:53:30 | INFO |   beta2: 0.98
+2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/checkpoints
+2022-12-21,10:53:30 | INFO |   copy_codebase: False
+2022-12-21,10:53:30 | INFO |   csv_caption_key: title
+2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:30 | INFO |   csv_separator: 	
+2022-12-21,10:53:30 | INFO |   dataset_resampled: False
+2022-12-21,10:53:30 | INFO |   dataset_type: auto
+2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:30 | INFO |   debug: False
+2022-12-21,10:53:30 | INFO |   device: cuda:0
+2022-12-21,10:53:30 | INFO |   dist_backend: nccl
+2022-12-21,10:53:30 | INFO |   dist_url: env://
+2022-12-21,10:53:30 | INFO |   distributed: False
+2022-12-21,10:53:30 | INFO |   epochs: 32
+2022-12-21,10:53:30 | INFO |   eps: 1e-06
+2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:30 | INFO |   gather_with_grad: False
+2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:30 | INFO |   horovod: False
+2022-12-21,10:53:30 | INFO |   image_mean: None
+2022-12-21,10:53:30 | INFO |   image_std: None
+2022-12-21,10:53:30 | INFO |   imagenet_v2: None
+2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:30 | INFO |   local_loss: False
+2022-12-21,10:53:30 | INFO |   local_rank: 0
+2022-12-21,10:53:30 | INFO |   lock_image: False
+2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:30 | INFO |   log_level: 20
+2022-12-21,10:53:30 | INFO |   log_local: False
+2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/out.log
+2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:30 | INFO |   lr: 0.0005
+2022-12-21,10:53:30 | INFO |   model: ViT-B-16
+2022-12-21,10:53:30 | INFO |   name: epoch_49.pt
+2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt
+2022-12-21,10:53:30 | INFO |   pretrained_image: False
+2022-12-21,10:53:30 | INFO |   rank: 0
+2022-12-21,10:53:30 | INFO |   report_to: 
+2022-12-21,10:53:30 | INFO |   resume: None
+2022-12-21,10:53:30 | INFO |   save_frequency: 1
+2022-12-21,10:53:30 | INFO |   save_most_recent: False
+2022-12-21,10:53:30 | INFO |   seed: 0
+2022-12-21,10:53:30 | INFO |   skip_scheduler: False
+2022-12-21,10:53:30 | INFO |   tensorboard: False
+2022-12-21,10:53:30 | INFO |   tensorboard_path: 
+2022-12-21,10:53:30 | INFO |   torchscript: False
+2022-12-21,10:53:30 | INFO |   trace: False
+2022-12-21,10:53:30 | INFO |   train_data: None
+2022-12-21,10:53:30 | INFO |   train_num_samples: None
+2022-12-21,10:53:30 | INFO |   use_bn_sync: False
+2022-12-21,10:53:30 | INFO |   val_data: None
+2022-12-21,10:53:30 | INFO |   val_frequency: 1
+2022-12-21,10:53:30 | INFO |   val_num_samples: None
+2022-12-21,10:53:30 | INFO |   wandb: False
+2022-12-21,10:53:30 | INFO |   wandb_notes: 
+2022-12-21,10:53:30 | INFO |   warmup: 10000
+2022-12-21,10:53:30 | INFO |   wd: 0.2
+2022-12-21,10:53:30 | INFO |   workers: 1
+2022-12-21,10:53:30 | INFO |   world_size: 1
+2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:02,  1.51it/s]
  1%|          | 6/1000 [00:00<01:38, 10.09it/s]
  1%|          | 12/1000 [00:00<00:49, 19.90it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.55it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.72it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.45it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.80it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.06it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.48it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.13it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.42it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.38it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.86it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.84it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.13it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.53it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.72it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.98it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.11it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.98it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.94it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.99it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.96it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.98it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.05it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.08it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.96it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.85it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.82it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.74it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.70it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.65it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.45it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.35it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.41it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.64it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.55it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.62it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.71it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.80it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.84it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.59it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.79it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.91it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.13it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.12it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.05it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.20it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.20it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.10it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.92it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.12it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.21it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.95it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.05it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.14it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.02it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.71it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.78it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.90it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.05it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.13it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.13it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.16it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.18it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.28it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.26it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.13it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.23it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.13it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.12it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.02it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.07it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.06it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.20it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.15it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.21it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.22it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.11it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.09it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.05it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.99it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.13it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.01it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.13it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.04it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.15it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.12it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.21it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.97it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.12it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.07it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.15it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.19it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.13it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.15it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.20it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.08it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.18it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.16it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.14it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.98it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.04it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.04it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.15it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.07it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.18it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.14it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.18it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.22it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.01it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.01it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.06it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.03it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.05it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.16it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.29it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.32it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.20it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.20it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.15it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.04it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.15it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.10it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.13it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.09it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.19it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.24it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.23it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.25it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.29it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.37it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.38it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.33it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.77it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.86it/s]
+2022-12-21,10:53:48 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:02, 82.86it/s]
  1%|          | 512/50176 [00:05<09:18, 88.91it/s]
  2%|▏         | 768/50176 [00:08<08:37, 95.49it/s]
  2%|▏         | 1024/50176 [00:10<08:29, 96.41it/s]
  3%|▎         | 1280/50176 [00:13<08:37, 94.40it/s]
  3%|▎         | 1536/50176 [00:16<09:12, 88.07it/s]
  4%|▎         | 1792/50176 [00:19<08:50, 91.27it/s]
  4%|▍         | 2048/50176 [00:22<08:25, 95.19it/s]
  5%|▍         | 2304/50176 [00:24<08:07, 98.21it/s]
  5%|▌         | 2560/50176 [00:26<07:57, 99.65it/s]
  6%|▌         | 2816/50176 [00:29<08:18, 94.97it/s]
  6%|▌         | 3072/50176 [00:32<08:05, 97.01it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.24it/s]
  7%|▋         | 3584/50176 [00:37<07:43, 100.62it/s]
  8%|▊         | 3840/50176 [00:40<07:53, 97.89it/s] 
  8%|▊         | 4096/50176 [00:42<08:04, 95.11it/s]
  9%|▊         | 4352/50176 [00:45<07:50, 97.47it/s]
  9%|▉         | 4608/50176 [00:47<07:34, 100.22it/s]
 10%|▉         | 4864/50176 [00:50<07:32, 100.07it/s]
 10%|█         | 5120/50176 [00:52<07:21, 101.98it/s]
 11%|█         | 5376/50176 [00:55<07:23, 100.96it/s]
 11%|█         | 5632/50176 [00:58<07:32, 98.53it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.22it/s]
 12%|█▏        | 6144/50176 [01:03<07:56, 92.31it/s]
 13%|█▎        | 6400/50176 [01:06<07:41, 94.95it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.34it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.91it/s]
 14%|█▍        | 7168/50176 [01:13<07:16, 98.60it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.68it/s]
 15%|█▌        | 7680/50176 [01:19<07:03, 100.45it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.84it/s]
 16%|█▋        | 8192/50176 [01:23<06:48, 102.69it/s]
 17%|█▋        | 8448/50176 [01:26<06:44, 103.14it/s]
 17%|█▋        | 8704/50176 [01:29<06:54, 100.00it/s]
 18%|█▊        | 8960/50176 [01:31<06:54, 99.49it/s] 
 18%|█▊        | 9216/50176 [01:34<06:46, 100.67it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.85it/s]
 19%|█▉        | 9728/50176 [01:39<06:48, 99.09it/s] 
 20%|█▉        | 9984/50176 [01:42<06:51, 97.76it/s]
 20%|██        | 10240/50176 [01:44<06:30, 102.19it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.44it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.10it/s] 
 22%|██▏       | 11008/50176 [01:52<06:33, 99.49it/s]
 22%|██▏       | 11264/50176 [01:54<06:21, 102.01it/s]
 23%|██▎       | 11520/50176 [01:57<06:21, 101.28it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.80it/s]
 24%|██▍       | 12032/50176 [02:02<06:16, 101.45it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.93it/s]
 25%|██▌       | 12544/50176 [02:07<06:09, 101.75it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.69it/s]
 26%|██▌       | 13056/50176 [02:11<06:00, 102.85it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.09it/s]
 27%|██▋       | 13568/50176 [02:17<06:11, 98.48it/s] 
 28%|██▊       | 13824/50176 [02:20<06:19, 95.72it/s]
 28%|██▊       | 14080/50176 [02:22<06:19, 95.22it/s]
 29%|██▊       | 14336/50176 [02:25<06:21, 94.02it/s]
 29%|██▉       | 14592/50176 [02:28<06:07, 96.91it/s]
 30%|██▉       | 14848/50176 [02:31<06:20, 92.92it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.29it/s]
 31%|███       | 15360/50176 [02:36<05:58, 97.21it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.79it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.07it/s]
 32%|███▏      | 16128/50176 [02:43<05:43, 99.21it/s]
 33%|███▎      | 16384/50176 [02:46<05:38, 99.91it/s]
 33%|███▎      | 16640/50176 [02:49<05:48, 96.28it/s]
 34%|███▎      | 16896/50176 [02:51<05:41, 97.40it/s]
 34%|███▍      | 17152/50176 [02:54<05:30, 99.92it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 100.99it/s]
 35%|███▌      | 17664/50176 [02:59<05:17, 102.46it/s]
 36%|███▌      | 17920/50176 [03:01<05:11, 103.42it/s]
 36%|███▌      | 18176/50176 [03:04<05:09, 103.50it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.38it/s]
 37%|███▋      | 18688/50176 [03:08<05:03, 103.71it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.63it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.18it/s] 
 39%|███▉      | 19456/50176 [03:17<05:20, 95.96it/s]
 39%|███▉      | 19712/50176 [03:19<05:18, 95.72it/s]
 40%|███▉      | 19968/50176 [03:22<05:10, 97.41it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.32it/s]
 41%|████      | 20480/50176 [03:26<04:44, 104.49it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.09it/s]
 42%|████▏     | 20992/50176 [03:31<04:31, 107.30it/s]
 42%|████▏     | 21248/50176 [03:34<04:35, 104.98it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.37it/s]
 43%|████▎     | 21760/50176 [03:38<04:26, 106.46it/s]
 44%|████▍     | 22016/50176 [03:41<04:20, 108.00it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.25it/s]
 45%|████▍     | 22528/50176 [03:45<04:15, 108.09it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.20it/s]
 46%|████▌     | 23040/50176 [03:51<04:25, 102.02it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.55it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.86it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.37it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.52it/s]
 48%|████▊     | 24320/50176 [04:03<04:16, 100.92it/s]
 49%|████▉     | 24576/50176 [04:06<04:12, 101.20it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 104.96it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 106.29it/s]
 51%|█████     | 25344/50176 [04:13<03:51, 107.39it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.21it/s]
 52%|█████▏    | 25856/50176 [04:18<03:45, 107.66it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.16it/s]
 53%|█████▎    | 26368/50176 [04:22<03:44, 106.02it/s]
 53%|█████▎    | 26624/50176 [04:25<03:43, 105.52it/s]
 54%|█████▎    | 26880/50176 [04:27<03:41, 105.15it/s]
 54%|█████▍    | 27136/50176 [04:30<03:40, 104.67it/s]
 55%|█████▍    | 27392/50176 [04:32<03:33, 106.55it/s]
 55%|█████▌    | 27648/50176 [04:34<03:25, 109.67it/s]
 56%|█████▌    | 27904/50176 [04:37<03:38, 102.15it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.01it/s]
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.55it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 100.98it/s]
 58%|█████▊    | 28928/50176 [04:47<03:27, 102.18it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.27it/s]
 59%|█████▊    | 29440/50176 [04:53<03:30, 98.67it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.89it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.79it/s]
 60%|██████    | 30208/50176 [05:01<03:22, 98.62it/s]
 61%|██████    | 30464/50176 [05:03<03:07, 105.24it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 103.02it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.74it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.11it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.48it/s]
 63%|██████▎   | 31744/50176 [05:14<02:49, 108.54it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.24it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.81it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.20it/s]
 65%|██████▌   | 32768/50176 [05:24<02:46, 104.69it/s]
 66%|██████▌   | 33024/50176 [05:27<02:44, 104.04it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.65it/s]
 67%|██████▋   | 33536/50176 [05:32<02:41, 102.97it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.65it/s]
 68%|██████▊   | 34048/50176 [05:36<02:30, 107.32it/s]
 68%|██████▊   | 34304/50176 [05:38<02:22, 111.47it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.47it/s]
 69%|██████▉   | 34816/50176 [05:44<02:24, 105.99it/s]
 70%|██████▉   | 35072/50176 [05:46<02:22, 105.67it/s]
 70%|███████   | 35328/50176 [05:48<02:20, 106.03it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.77it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.15it/s]
 72%|███████▏  | 36096/50176 [05:56<02:12, 106.22it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.58it/s]
 73%|███████▎  | 36608/50176 [06:01<02:11, 102.89it/s]
 73%|███████▎  | 36864/50176 [06:03<02:12, 100.31it/s]
 74%|███████▍  | 37120/50176 [06:06<02:07, 102.69it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.46it/s]
 75%|███████▌  | 37632/50176 [06:10<01:58, 105.84it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.13it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.35it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.76it/s]
 77%|███████▋  | 38656/50176 [06:20<01:49, 105.63it/s]
 78%|███████▊  | 38912/50176 [06:23<01:47, 105.27it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.26it/s]
 79%|███████▊  | 39424/50176 [06:28<01:41, 105.98it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.45it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.84it/s]
 80%|████████  | 40192/50176 [06:35<01:34, 105.88it/s]
 81%|████████  | 40448/50176 [06:37<01:34, 102.69it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.23it/s]
 82%|████████▏ | 40960/50176 [06:42<01:27, 104.82it/s]
 82%|████████▏ | 41216/50176 [06:45<01:24, 105.60it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 104.00it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 106.69it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.96it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.91it/s]
 85%|████████▍ | 42496/50176 [06:57<01:10, 108.77it/s]
 85%|████████▌ | 42752/50176 [06:59<01:07, 109.58it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.19it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 107.43it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.29it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 103.26it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 105.85it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.91it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 103.91it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.79it/s]
 90%|████████▉ | 45056/50176 [07:21<00:47, 107.80it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.19it/s]
 91%|█████████ | 45568/50176 [07:26<00:43, 106.76it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 105.09it/s]
 92%|█████████▏| 46080/50176 [07:31<00:39, 104.90it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.51it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 104.83it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.32it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 101.15it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.17it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 100.77it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.71it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 102.57it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.60it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.95it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 106.19it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.91it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 105.56it/s]
 99%|█████████▉| 49664/50176 [08:05<00:04, 102.61it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.19it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 126.92it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.52it/s]
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6437	imagenet-zeroshot-val-top5: 0.8902
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt
new file mode 100644
index 0000000000000000000000000000000000000000..8ab230757d0f79d0eb24a83c67b6ba6d3936687c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88f14da2243c9aef3c7a2ba7b5d62b2597cff7e5dcaa5e4071231683c0b0705c
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..93cdc104785fa2d854a56b26d2390133c8339202
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_5.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:02,  1.51it/s]
  1%|          | 6/1000 [00:00<01:38, 10.05it/s]
  1%|          | 12/1000 [00:00<00:50, 19.40it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.70it/s]
  2%|▏         | 24/1000 [00:01<00:27, 34.92it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.77it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.23it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.61it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.10it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.92it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.25it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.24it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.91it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.37it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.88it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.17it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.55it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.67it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.74it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.83it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.87it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.86it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.91it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.00it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.04it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.89it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.75it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.73it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.72it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.61it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.59it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.55it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.37it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.27it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.36it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.57it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.57it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.48it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.55it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.64it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.74it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.77it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.57it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.58it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.68it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.79it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.04it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.97it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.97it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.05it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.88it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.94it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.05it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.14it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.93it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.95it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.65it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.86it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.57it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.66it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.81it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.88it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.98it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.00it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.01it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.05it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.95it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.00it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.01it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.97it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.13it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.04it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.02it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.02it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.95it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.01it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.01it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.05it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.18it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.00it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.93it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.97it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.05it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.93it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.89it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.04it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.03it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.97it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.10it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.01it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.00it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.13it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.12it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.20it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.98it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.12it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.07it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.15it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.22it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.22it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.10it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.19it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.17it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.14it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.05it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.04it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.14it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.23it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.05it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.16it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.12it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.16it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.19it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.99it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.99it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.05it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.02it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.02it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.13it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.28it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.29it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.20it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.25it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.20it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.18it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.16it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.02it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.13it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.12it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.09it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.12it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.11it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.25it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.25it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.27it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.17it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.28it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.33it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.06it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.02it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.88it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.76it/s]
+2022-12-21,11:03:09 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:39, 78.07it/s]
  1%|          | 512/50176 [00:05<09:15, 89.35it/s]
  2%|▏         | 768/50176 [00:08<08:26, 97.64it/s]
  2%|▏         | 1024/50176 [00:10<08:23, 97.71it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.54it/s]
  3%|▎         | 1536/50176 [00:16<08:53, 91.20it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.21it/s]
  4%|▍         | 2048/50176 [00:21<08:19, 96.27it/s]
  5%|▍         | 2304/50176 [00:24<08:08, 98.00it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.64it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.16it/s] 
  6%|▌         | 3072/50176 [00:31<07:53, 99.50it/s]
  7%|▋         | 3328/50176 [00:34<07:40, 101.81it/s]
  7%|▋         | 3584/50176 [00:36<07:30, 103.46it/s]
  8%|▊         | 3840/50176 [00:39<07:38, 101.12it/s]
  8%|▊         | 4096/50176 [00:42<07:54, 97.17it/s] 
  9%|▊         | 4352/50176 [00:44<07:49, 97.54it/s]
  9%|▉         | 4608/50176 [00:47<07:32, 100.70it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 102.01it/s]
 10%|█         | 5120/50176 [00:51<07:21, 102.14it/s]
 11%|█         | 5376/50176 [00:54<07:26, 100.40it/s]
 11%|█         | 5632/50176 [00:57<07:18, 101.69it/s]
 12%|█▏        | 5888/50176 [00:59<07:20, 100.47it/s]
 12%|█▏        | 6144/50176 [01:03<08:01, 91.53it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.95it/s]
 13%|█▎        | 6656/50176 [01:08<07:26, 97.39it/s]
 14%|█▍        | 6912/50176 [01:10<07:34, 95.15it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.18it/s]
 15%|█▍        | 7424/50176 [01:15<07:03, 100.97it/s]
 15%|█▌        | 7680/50176 [01:18<06:57, 101.77it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.14it/s]
 16%|█▋        | 8192/50176 [01:23<06:50, 102.32it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 103.00it/s]
 17%|█▋        | 8704/50176 [01:28<06:47, 101.83it/s]
 18%|█▊        | 8960/50176 [01:30<06:58, 98.48it/s] 
 18%|█▊        | 9216/50176 [01:33<06:39, 102.66it/s]
 19%|█▉        | 9472/50176 [01:35<06:34, 103.08it/s]
 19%|█▉        | 9728/50176 [01:38<06:43, 100.31it/s]
 20%|█▉        | 9984/50176 [01:40<06:41, 100.12it/s]
 20%|██        | 10240/50176 [01:43<06:27, 103.04it/s]
 21%|██        | 10496/50176 [01:45<06:27, 102.48it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.22it/s]
 22%|██▏       | 11008/50176 [01:50<06:26, 101.30it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.80it/s]
 23%|██▎       | 11520/50176 [01:55<06:12, 103.77it/s]
 23%|██▎       | 11776/50176 [01:58<06:10, 103.54it/s]
 24%|██▍       | 12032/50176 [02:00<06:14, 101.74it/s]
 24%|██▍       | 12288/50176 [02:03<06:08, 102.71it/s]
 25%|██▌       | 12544/50176 [02:05<06:14, 100.61it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 98.88it/s] 
 26%|██▌       | 13056/50176 [02:10<06:03, 102.12it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.80it/s]
 27%|██▋       | 13568/50176 [02:16<06:23, 95.50it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.55it/s]
 28%|██▊       | 14080/50176 [02:22<06:25, 93.73it/s]
 29%|██▊       | 14336/50176 [02:24<06:19, 94.36it/s]
 29%|██▉       | 14592/50176 [02:27<06:01, 98.34it/s]
 30%|██▉       | 14848/50176 [02:29<06:06, 96.39it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.47it/s]
 31%|███       | 15360/50176 [02:35<05:55, 97.89it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.74it/s]
 32%|███▏      | 15872/50176 [02:40<05:47, 98.80it/s]
 32%|███▏      | 16128/50176 [02:42<05:39, 100.32it/s]
 33%|███▎      | 16384/50176 [02:45<05:31, 101.79it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.81it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.90it/s]
 34%|███▍      | 17152/50176 [02:52<05:25, 101.52it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.31it/s]
 35%|███▌      | 17664/50176 [02:57<05:16, 102.70it/s]
 36%|███▌      | 17920/50176 [03:00<05:09, 104.13it/s]
 36%|███▌      | 18176/50176 [03:02<05:13, 102.08it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.90it/s]
 37%|███▋      | 18688/50176 [03:07<05:15, 99.70it/s] 
 38%|███▊      | 18944/50176 [03:10<05:16, 98.67it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.77it/s]
 39%|███▉      | 19456/50176 [03:16<05:18, 96.58it/s]
 39%|███▉      | 19712/50176 [03:18<05:19, 95.41it/s]
 40%|███▉      | 19968/50176 [03:21<05:13, 96.42it/s]
 40%|████      | 20224/50176 [03:23<04:53, 102.17it/s]
 41%|████      | 20480/50176 [03:26<04:49, 102.61it/s]
 41%|████▏     | 20736/50176 [03:28<04:43, 103.93it/s]
 42%|████▏     | 20992/50176 [03:30<04:36, 105.66it/s]
 42%|████▏     | 21248/50176 [03:33<04:40, 103.29it/s]
 43%|████▎     | 21504/50176 [03:35<04:35, 104.03it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.63it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.46it/s]
 44%|████▍     | 22272/50176 [03:42<04:19, 107.38it/s]
 45%|████▍     | 22528/50176 [03:45<04:18, 106.81it/s]
 45%|████▌     | 22784/50176 [03:47<04:23, 103.82it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.58it/s]
 46%|████▋     | 23296/50176 [03:53<04:29, 99.80it/s] 
 47%|████▋     | 23552/50176 [03:56<04:35, 96.81it/s]
 47%|████▋     | 23808/50176 [03:58<04:19, 101.56it/s]
 48%|████▊     | 24064/50176 [04:00<04:19, 100.49it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.03it/s]
 49%|████▉     | 24576/50176 [04:06<04:17, 99.38it/s] 
 49%|████▉     | 24832/50176 [04:08<04:08, 101.95it/s]
 50%|█████     | 25088/50176 [04:10<04:00, 104.21it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.45it/s]
 51%|█████     | 25600/50176 [04:15<03:57, 103.37it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.23it/s]
 52%|█████▏    | 26112/50176 [04:20<03:48, 105.21it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.37it/s]
 53%|█████▎    | 26624/50176 [04:26<03:54, 100.52it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.22it/s]
 54%|█████▍    | 27136/50176 [04:31<03:49, 100.24it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 101.97it/s]
 55%|█████▌    | 27648/50176 [04:35<03:33, 105.38it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.45it/s]
 56%|█████▌    | 28160/50176 [04:41<03:41, 99.18it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:38, 99.80it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.16it/s]
 58%|█████▊    | 28928/50176 [04:48<03:29, 101.57it/s]
 58%|█████▊    | 29184/50176 [04:51<03:23, 103.08it/s]
 59%|█████▊    | 29440/50176 [04:54<03:33, 97.00it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:30, 97.22it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.46it/s]
 60%|██████    | 30208/50176 [05:01<03:23, 98.21it/s]
 61%|██████    | 30464/50176 [05:04<03:12, 102.46it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.81it/s]
 62%|██████▏   | 30976/50176 [05:09<03:05, 103.30it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.13it/s]
 63%|██████▎   | 31488/50176 [05:13<02:55, 106.32it/s]
 63%|██████▎   | 31744/50176 [05:16<02:54, 105.35it/s]
 64%|██████▍   | 32000/50176 [05:18<02:52, 105.39it/s]
 64%|██████▍   | 32256/50176 [05:21<02:49, 105.48it/s]
 65%|██████▍   | 32512/50176 [05:24<02:56, 100.30it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.65it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 101.76it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.70it/s]
 67%|██████▋   | 33536/50176 [05:34<02:46, 100.02it/s]
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.12it/s]
 68%|██████▊   | 34048/50176 [05:38<02:33, 104.87it/s]
 68%|██████▊   | 34304/50176 [05:41<02:27, 107.25it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.84it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 106.06it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.71it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.44it/s]
 71%|███████   | 35584/50176 [05:53<02:16, 106.65it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.47it/s]
 72%|███████▏  | 36096/50176 [05:58<02:14, 104.35it/s]
 72%|███████▏  | 36352/50176 [06:00<02:13, 103.63it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.07it/s]
 73%|███████▎  | 36864/50176 [06:06<02:13, 99.94it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.40it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.19it/s]
 75%|███████▌  | 37632/50176 [06:12<01:56, 107.49it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.68it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 107.06it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.77it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.38it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.05it/s]
 78%|███████▊  | 39168/50176 [06:28<01:49, 100.45it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.85it/s]
 79%|███████▉  | 39680/50176 [06:32<01:42, 101.96it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.20it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.76it/s]
 81%|████████  | 40448/50176 [06:40<01:39, 98.22it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.35it/s]
 82%|████████▏ | 40960/50176 [06:45<01:32, 99.73it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.10it/s]
 83%|████████▎ | 41472/50176 [06:50<01:26, 100.81it/s]
 83%|████████▎ | 41728/50176 [06:53<01:22, 102.93it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.25it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.53it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.26it/s]
 85%|████████▌ | 42752/50176 [07:02<01:09, 107.04it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.98it/s]
 86%|████████▌ | 43264/50176 [07:07<01:05, 105.29it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.30it/s]
 87%|████████▋ | 43776/50176 [07:12<01:02, 102.27it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.84it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.36it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.39it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:25<00:48, 105.97it/s]
 90%|█████████ | 45312/50176 [07:27<00:46, 104.94it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 107.05it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.61it/s]
 92%|█████████▏| 46080/50176 [07:35<00:39, 103.11it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.65it/s]
 93%|█████████▎| 46592/50176 [07:39<00:33, 105.45it/s]
 93%|█████████▎| 46848/50176 [07:42<00:31, 106.76it/s]
 94%|█████████▍| 47104/50176 [07:44<00:30, 101.66it/s]
 94%|█████████▍| 47360/50176 [07:47<00:28, 99.05it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.23it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.88it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 105.67it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.98it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 102.58it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.73it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.29it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.28it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.87it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.79it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 124.55it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.59it/s]
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4672	imagenet-zeroshot-val-top5: 0.7495
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1a0ca6ac2f3fdd5c97af31785248c45023e7a116
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cbe21242ddfb102d7edab213154124bb056980b7f755f4ae7ea0f6f7c13b749
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..34ea887320bac540afe76e9040298352b4b7c046
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_50.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:33,  1.44it/s]
  1%|          | 7/1000 [00:00<01:29, 11.07it/s]
  1%|▏         | 13/1000 [00:00<00:49, 20.11it/s]
  2%|▏         | 19/1000 [00:01<00:34, 28.41it/s]
  2%|▎         | 25/1000 [00:01<00:27, 35.36it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.09it/s]
  4%|▎         | 37/1000 [00:01<00:21, 45.53it/s]
  4%|▍         | 43/1000 [00:01<00:19, 48.88it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.33it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.08it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.45it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.09it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.64it/s]
  8%|▊         | 85/1000 [00:02<00:16, 56.94it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.26it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.67it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.83it/s]
 11%|█         | 109/1000 [00:02<00:15, 58.07it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.26it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 58.18it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.18it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.19it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 58.17it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.21it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.25it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.21it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.10it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.90it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.87it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.85it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.85it/s]
 19%|█▉        | 193/1000 [00:04<00:13, 57.81it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.75it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.59it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.41it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.61it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.60it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.81it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.75it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.58it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.70it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.77it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.85it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.84it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.84it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.98it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.12it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.31it/s]
 30%|███       | 301/1000 [00:05<00:11, 58.29it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.23it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.41it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.44it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.27it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.14it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.27it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.32it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.28it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 58.20it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.22it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.28it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.37it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.89it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.95it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.03it/s]
 40%|████      | 403/1000 [00:07<00:10, 58.15it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.22it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.30it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.33it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.32it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.46it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.46it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.19it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.23it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.27it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.30it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.23it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.37it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.18it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.28it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.22it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.12it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.18it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.10it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.18it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.16it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.19it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.16it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.22it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.26it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.16it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.24it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.18it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.08it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.18it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.27it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.30it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.20it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.30it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.15it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.35it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.33it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.31it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.23it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.22it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.30it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.29it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.28it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.34it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.33it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.36it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.35it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.26it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.33it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.35it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 58.32it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.24it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.20it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.20it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.26it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.28it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.35it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.39it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.27it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.34it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.31it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.36it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.36it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.22it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.14it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.15it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.24it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.23it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.34it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.49it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.52it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.38it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.38it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.37it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.31it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.31it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.23it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.32it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.27it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.24it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.18it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.31it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.29it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.39it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.43it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.37it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.27it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.42it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.44it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.54it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.52it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.14it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 57.94it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 57.71it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.89it/s]
+2022-12-21,11:03:09 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:36, 78.47it/s]
  1%|          | 512/50176 [00:05<09:16, 89.27it/s]
  2%|▏         | 768/50176 [00:08<08:27, 97.42it/s]
  2%|▏         | 1024/50176 [00:10<08:21, 98.01it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.41it/s]
  3%|▎         | 1536/50176 [00:16<08:53, 91.22it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.13it/s]
  4%|▍         | 2048/50176 [00:21<08:19, 96.32it/s]
  5%|▍         | 2304/50176 [00:24<08:07, 98.13it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.64it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.09it/s] 
  6%|▌         | 3072/50176 [00:31<07:53, 99.53it/s]
  7%|▋         | 3328/50176 [00:34<07:39, 101.88it/s]
  7%|▋         | 3584/50176 [00:36<07:30, 103.50it/s]
  8%|▊         | 3840/50176 [00:39<07:39, 100.94it/s]
  8%|▊         | 4096/50176 [00:42<07:53, 97.26it/s] 
  9%|▊         | 4352/50176 [00:44<07:49, 97.60it/s]
  9%|▉         | 4608/50176 [00:47<07:32, 100.68it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.86it/s]
 10%|█         | 5120/50176 [00:51<07:20, 102.25it/s]
 11%|█         | 5376/50176 [00:54<07:26, 100.40it/s]
 11%|█         | 5632/50176 [00:57<07:19, 101.46it/s]
 12%|█▏        | 5888/50176 [00:59<07:20, 100.56it/s]
 12%|█▏        | 6144/50176 [01:03<08:00, 91.56it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.84it/s]
 13%|█▎        | 6656/50176 [01:08<07:26, 97.49it/s]
 14%|█▍        | 6912/50176 [01:10<07:35, 95.08it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.23it/s]
 15%|█▍        | 7424/50176 [01:15<07:03, 100.88it/s]
 15%|█▌        | 7680/50176 [01:18<06:56, 101.91it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.27it/s]
 16%|█▋        | 8192/50176 [01:23<06:50, 102.33it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 102.96it/s]
 17%|█▋        | 8704/50176 [01:28<06:47, 101.75it/s]
 18%|█▊        | 8960/50176 [01:30<06:59, 98.34it/s] 
 18%|█▊        | 9216/50176 [01:33<06:38, 102.81it/s]
 19%|█▉        | 9472/50176 [01:35<06:34, 103.23it/s]
 19%|█▉        | 9728/50176 [01:38<06:43, 100.18it/s]
 20%|█▉        | 9984/50176 [01:40<06:41, 100.05it/s]
 20%|██        | 10240/50176 [01:43<06:27, 102.94it/s]
 21%|██        | 10496/50176 [01:45<06:27, 102.35it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.25it/s]
 22%|██▏       | 11008/50176 [01:50<06:26, 101.27it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.85it/s]
 23%|██▎       | 11520/50176 [01:55<06:12, 103.67it/s]
 23%|██▎       | 11776/50176 [01:58<06:10, 103.66it/s]
 24%|██▍       | 12032/50176 [02:00<06:14, 101.72it/s]
 24%|██▍       | 12288/50176 [02:03<06:08, 102.76it/s]
 25%|██▌       | 12544/50176 [02:05<06:14, 100.56it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 98.89it/s] 
 26%|██▌       | 13056/50176 [02:10<06:03, 102.15it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.76it/s]
 27%|██▋       | 13568/50176 [02:16<06:22, 95.61it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.55it/s]
 28%|██▊       | 14080/50176 [02:22<06:25, 93.66it/s]
 29%|██▊       | 14336/50176 [02:24<06:20, 94.31it/s]
 29%|██▉       | 14592/50176 [02:27<06:02, 98.30it/s]
 30%|██▉       | 14848/50176 [02:29<06:06, 96.48it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.45it/s]
 31%|███       | 15360/50176 [02:35<05:55, 97.88it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.79it/s]
 32%|███▏      | 15872/50176 [02:40<05:46, 98.90it/s]
 32%|███▏      | 16128/50176 [02:42<05:40, 100.14it/s]
 33%|███▎      | 16384/50176 [02:45<05:32, 101.71it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.81it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.97it/s]
 34%|███▍      | 17152/50176 [02:52<05:25, 101.48it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.23it/s]
 35%|███▌      | 17664/50176 [02:57<05:16, 102.63it/s]
 36%|███▌      | 17920/50176 [03:00<05:09, 104.24it/s]
 36%|███▌      | 18176/50176 [03:02<05:13, 102.18it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.95it/s]
 37%|███▋      | 18688/50176 [03:07<05:15, 99.79it/s] 
 38%|███▊      | 18944/50176 [03:10<05:17, 98.47it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.84it/s]
 39%|███▉      | 19456/50176 [03:16<05:18, 96.54it/s]
 39%|███▉      | 19712/50176 [03:18<05:19, 95.47it/s]
 40%|███▉      | 19968/50176 [03:21<05:13, 96.48it/s]
 40%|████      | 20224/50176 [03:23<04:53, 102.15it/s]
 41%|████      | 20480/50176 [03:26<04:49, 102.58it/s]
 41%|████▏     | 20736/50176 [03:28<04:43, 104.00it/s]
 42%|████▏     | 20992/50176 [03:30<04:36, 105.50it/s]
 42%|████▏     | 21248/50176 [03:33<04:39, 103.42it/s]
 43%|████▎     | 21504/50176 [03:35<04:35, 104.05it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.55it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.42it/s]
 44%|████▍     | 22272/50176 [03:42<04:20, 107.27it/s]
 45%|████▍     | 22528/50176 [03:45<04:19, 106.73it/s]
 45%|████▌     | 22784/50176 [03:47<04:24, 103.74it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.61it/s]
 46%|████▋     | 23296/50176 [03:53<04:28, 99.98it/s] 
 47%|████▋     | 23552/50176 [03:56<04:34, 96.82it/s]
 47%|████▋     | 23808/50176 [03:58<04:19, 101.52it/s]
 48%|████▊     | 24064/50176 [04:00<04:19, 100.52it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.06it/s]
 49%|████▉     | 24576/50176 [04:06<04:17, 99.39it/s] 
 49%|████▉     | 24832/50176 [04:08<04:08, 101.94it/s]
 50%|█████     | 25088/50176 [04:10<04:00, 104.21it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.41it/s]
 51%|█████     | 25600/50176 [04:15<03:57, 103.35it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.24it/s]
 52%|█████▏    | 26112/50176 [04:20<03:48, 105.15it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.28it/s]
 53%|█████▎    | 26624/50176 [04:26<03:54, 100.61it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.22it/s]
 54%|█████▍    | 27136/50176 [04:31<03:49, 100.35it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 102.17it/s]
 55%|█████▌    | 27648/50176 [04:35<03:34, 105.19it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.52it/s]
 56%|█████▌    | 28160/50176 [04:41<03:42, 99.02it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:37, 99.92it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.11it/s]
 58%|█████▊    | 28928/50176 [04:48<03:29, 101.50it/s]
 58%|█████▊    | 29184/50176 [04:51<03:23, 103.06it/s]
 59%|█████▊    | 29440/50176 [04:54<03:33, 96.94it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:30, 97.18it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.49it/s]
 60%|██████    | 30208/50176 [05:01<03:23, 98.33it/s]
 61%|██████    | 30464/50176 [05:04<03:12, 102.40it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.85it/s]
 62%|██████▏   | 30976/50176 [05:09<03:05, 103.29it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.19it/s]
 63%|██████▎   | 31488/50176 [05:13<02:55, 106.35it/s]
 63%|██████▎   | 31744/50176 [05:16<02:54, 105.40it/s]
 64%|██████▍   | 32000/50176 [05:18<02:52, 105.41it/s]
 64%|██████▍   | 32256/50176 [05:21<02:49, 105.48it/s]
 65%|██████▍   | 32512/50176 [05:24<02:56, 100.14it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.78it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 101.81it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.77it/s]
 67%|██████▋   | 33536/50176 [05:34<02:46, 99.96it/s] 
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.18it/s]
 68%|██████▊   | 34048/50176 [05:38<02:33, 104.92it/s]
 68%|██████▊   | 34304/50176 [05:41<02:28, 107.22it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.92it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 106.02it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.62it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.52it/s]
 71%|███████   | 35584/50176 [05:53<02:16, 106.69it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.44it/s]
 72%|███████▏  | 36096/50176 [05:58<02:14, 104.41it/s]
 72%|███████▏  | 36352/50176 [06:00<02:13, 103.67it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.14it/s]
 73%|███████▎  | 36864/50176 [06:06<02:13, 99.86it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.31it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.25it/s]
 75%|███████▌  | 37632/50176 [06:12<01:56, 107.51it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.67it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 107.07it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.77it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.39it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.05it/s]
 78%|███████▊  | 39168/50176 [06:28<01:49, 100.51it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.82it/s]
 79%|███████▉  | 39680/50176 [06:32<01:42, 101.96it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.18it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.71it/s]
 81%|████████  | 40448/50176 [06:40<01:39, 98.22it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.35it/s]
 82%|████████▏ | 40960/50176 [06:45<01:32, 99.77it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.00it/s]
 83%|████████▎ | 41472/50176 [06:50<01:26, 100.88it/s]
 83%|████████▎ | 41728/50176 [06:53<01:22, 102.98it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.23it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.50it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.29it/s]
 85%|████████▌ | 42752/50176 [07:02<01:09, 107.12it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.90it/s]
 86%|████████▌ | 43264/50176 [07:07<01:05, 105.29it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.24it/s]
 87%|████████▋ | 43776/50176 [07:12<01:02, 102.35it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.85it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.38it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.37it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:25<00:48, 105.97it/s]
 90%|█████████ | 45312/50176 [07:27<00:46, 104.97it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 106.94it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.62it/s]
 92%|█████████▏| 46080/50176 [07:35<00:39, 103.15it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.64it/s]
 93%|█████████▎| 46592/50176 [07:39<00:33, 105.44it/s]
 93%|█████████▎| 46848/50176 [07:42<00:31, 106.78it/s]
 94%|█████████▍| 47104/50176 [07:44<00:30, 101.66it/s]
 94%|█████████▍| 47360/50176 [07:47<00:28, 99.05it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.23it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.87it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 105.65it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.96it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 102.47it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.87it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.39it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.09it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.98it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.66it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 124.42it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.59it/s]
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6466	imagenet-zeroshot-val-top5: 0.8905
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt
new file mode 100644
index 0000000000000000000000000000000000000000..ceaa8ec373b4e8349ecac373c532ceec0765488d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b0623baea96903177b4462262d80c89c689fd41fa9c9a1fa0a3a2704f04dfcc8
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5fdbaf8a6d7a81c037c701348bde382af5a5e4e1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_51.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:32,  1.44it/s]
  1%|          | 6/1000 [00:00<01:42,  9.65it/s]
  1%|          | 11/1000 [00:00<00:56, 17.48it/s]
  2%|▏         | 17/1000 [00:01<00:38, 25.76it/s]
  2%|▏         | 23/1000 [00:01<00:29, 32.94it/s]
  3%|▎         | 29/1000 [00:01<00:25, 38.70it/s]
  4%|▎         | 35/1000 [00:01<00:22, 43.53it/s]
  4%|▍         | 41/1000 [00:01<00:20, 47.46it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.17it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.37it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.71it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.90it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.61it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.22it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.79it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.25it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.39it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.65it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.97it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.11it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.13it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 58.08it/s]
 13%|█▎        | 131/1000 [00:02<00:14, 58.16it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 58.05it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 58.08it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.13it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.21it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 58.09it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.97it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.94it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.77it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.76it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.72it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.60it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.48it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.50it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.68it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.54it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.45it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 57.50it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.58it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.79it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.84it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.61it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.73it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.67it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.80it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 58.07it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 58.09it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.14it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.26it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.19it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 58.00it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.08it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.13it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.23it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 58.05it/s]
 36%|███▌      | 359/1000 [00:07<00:16, 37.99it/s]
 36%|███▋      | 365/1000 [00:07<00:14, 42.37it/s]
 37%|███▋      | 371/1000 [00:07<00:13, 46.16it/s]
 38%|███▊      | 377/1000 [00:07<00:12, 49.11it/s]
 38%|███▊      | 383/1000 [00:07<00:12, 51.33it/s]
 39%|███▉      | 389/1000 [00:07<00:11, 53.12it/s]
 40%|███▉      | 395/1000 [00:07<00:11, 54.55it/s]
 40%|████      | 401/1000 [00:07<00:10, 55.65it/s]
 41%|████      | 407/1000 [00:07<00:10, 56.46it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 57.00it/s]
 42%|████▏     | 419/1000 [00:08<00:10, 57.39it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.58it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.90it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.03it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.92it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.01it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.12it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.12it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 58.27it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.19it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.18it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.88it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.07it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 58.27it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 58.18it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.31it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.36it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.23it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.20it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.17it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 58.11it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.26it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 58.19it/s]
 59%|█████▉    | 593/1000 [00:11<00:06, 58.15it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.26it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.17it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.27it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.29it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.35it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.22it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 58.09it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 58.30it/s]
 65%|██████▌   | 653/1000 [00:12<00:05, 58.22it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.32it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.30it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.33it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.39it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.25it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.35it/s]
 71%|███████   | 707/1000 [00:13<00:05, 58.33it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.34it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.38it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.31it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.28it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.33it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.28it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.26it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.41it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 58.23it/s]
 77%|███████▋  | 767/1000 [00:14<00:03, 58.28it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.30it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.35it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.38it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.18it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.20it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.21it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 58.02it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 58.08it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.30it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.41it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.29it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.16it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.26it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.24it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.29it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 58.32it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 58.14it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.24it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.31it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.27it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.28it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.22it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.34it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 58.43it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 58.41it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.39it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.43it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.36it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.42it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.50it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.53it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.55it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.49it/s]
100%|█████████▉| 995/1000 [00:18<00:00, 58.36it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.17it/s]
+2022-12-21,11:03:09 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:47, 70.57it/s]
  1%|          | 512/50176 [00:06<09:46, 84.62it/s]
  2%|▏         | 768/50176 [00:08<08:41, 94.65it/s]
  2%|▏         | 1024/50176 [00:11<08:31, 96.01it/s]
  3%|▎         | 1280/50176 [00:13<08:22, 97.36it/s]
  3%|▎         | 1536/50176 [00:16<08:57, 90.53it/s]
  4%|▎         | 1792/50176 [00:19<08:42, 92.65it/s]
  4%|▍         | 2048/50176 [00:22<08:21, 96.05it/s]
  5%|▍         | 2304/50176 [00:24<08:10, 97.65it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.57it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.08it/s] 
  6%|▌         | 3072/50176 [00:32<07:53, 99.40it/s]
  7%|▋         | 3328/50176 [00:34<07:40, 101.83it/s]
  7%|▋         | 3584/50176 [00:36<07:30, 103.35it/s]
  8%|▊         | 3840/50176 [00:39<07:38, 100.99it/s]
  8%|▊         | 4096/50176 [00:42<07:52, 97.45it/s] 
  9%|▊         | 4352/50176 [00:45<07:49, 97.62it/s]
  9%|▉         | 4608/50176 [00:47<07:33, 100.55it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.86it/s]
 10%|█         | 5120/50176 [00:52<07:21, 102.16it/s]
 11%|█         | 5376/50176 [00:55<07:26, 100.30it/s]
 11%|█         | 5632/50176 [00:57<07:18, 101.57it/s]
 12%|█▏        | 5888/50176 [01:00<07:21, 100.37it/s]
 12%|█▏        | 6144/50176 [01:03<08:00, 91.67it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.86it/s]
 13%|█▎        | 6656/50176 [01:08<07:26, 97.54it/s]
 14%|█▍        | 6912/50176 [01:11<07:34, 95.11it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.26it/s]
 15%|█▍        | 7424/50176 [01:16<07:04, 100.78it/s]
 15%|█▌        | 7680/50176 [01:18<06:57, 101.88it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.10it/s]
 16%|█▋        | 8192/50176 [01:23<06:50, 102.33it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 102.96it/s]
 17%|█▋        | 8704/50176 [01:28<06:46, 101.96it/s]
 18%|█▊        | 8960/50176 [01:31<06:58, 98.40it/s] 
 18%|█▊        | 9216/50176 [01:33<06:38, 102.87it/s]
 19%|█▉        | 9472/50176 [01:35<06:35, 102.99it/s]
 19%|█▉        | 9728/50176 [01:38<06:43, 100.30it/s]
 20%|█▉        | 9984/50176 [01:41<06:42, 99.96it/s] 
 20%|██        | 10240/50176 [01:43<06:27, 103.13it/s]
 21%|██        | 10496/50176 [01:46<06:27, 102.36it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.23it/s]
 22%|██▏       | 11008/50176 [01:51<06:26, 101.30it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.82it/s]
 23%|██▎       | 11520/50176 [01:56<06:12, 103.84it/s]
 23%|██▎       | 11776/50176 [01:58<06:10, 103.69it/s]
 24%|██▍       | 12032/50176 [02:01<06:14, 101.72it/s]
 24%|██▍       | 12288/50176 [02:03<06:09, 102.66it/s]
 25%|██▌       | 12544/50176 [02:06<06:14, 100.48it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 98.93it/s] 
 26%|██▌       | 13056/50176 [02:11<06:03, 102.09it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.70it/s]
 27%|██▋       | 13568/50176 [02:16<06:22, 95.59it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.59it/s]
 28%|██▊       | 14080/50176 [02:22<06:25, 93.71it/s]
 29%|██▊       | 14336/50176 [02:25<06:19, 94.40it/s]
 29%|██▉       | 14592/50176 [02:27<06:02, 98.24it/s]
 30%|██▉       | 14848/50176 [02:30<06:06, 96.36it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.56it/s]
 31%|███       | 15360/50176 [02:35<05:55, 98.00it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.70it/s]
 32%|███▏      | 15872/50176 [02:40<05:47, 98.73it/s]
 32%|███▏      | 16128/50176 [02:42<05:39, 100.21it/s]
 33%|███▎      | 16384/50176 [02:45<05:32, 101.77it/s]
 33%|███▎      | 16640/50176 [02:48<05:42, 97.87it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.91it/s]
 34%|███▍      | 17152/50176 [02:53<05:24, 101.63it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.20it/s]
 35%|███▌      | 17664/50176 [02:58<05:16, 102.63it/s]
 36%|███▌      | 17920/50176 [03:00<05:09, 104.12it/s]
 36%|███▌      | 18176/50176 [03:03<05:13, 102.09it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.92it/s]
 37%|███▋      | 18688/50176 [03:08<05:15, 99.69it/s] 
 38%|███▊      | 18944/50176 [03:11<05:16, 98.58it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.92it/s]
 39%|███▉      | 19456/50176 [03:16<05:18, 96.56it/s]
 39%|███▉      | 19712/50176 [03:19<05:19, 95.37it/s]
 40%|███▉      | 19968/50176 [03:21<05:12, 96.57it/s]
 40%|████      | 20224/50176 [03:23<04:52, 102.28it/s]
 41%|████      | 20480/50176 [03:26<04:49, 102.64it/s]
 41%|████▏     | 20736/50176 [03:28<04:43, 103.96it/s]
 42%|████▏     | 20992/50176 [03:31<04:36, 105.47it/s]
 42%|████▏     | 21248/50176 [03:33<04:40, 103.18it/s]
 43%|████▎     | 21504/50176 [03:36<04:35, 103.97it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.68it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.56it/s]
 44%|████▍     | 22272/50176 [03:43<04:20, 107.29it/s]
 45%|████▍     | 22528/50176 [03:45<04:19, 106.70it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.74it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.62it/s]
 46%|████▋     | 23296/50176 [03:53<04:28, 99.95it/s] 
 47%|████▋     | 23552/50176 [03:56<04:34, 96.87it/s]
 47%|████▋     | 23808/50176 [03:58<04:19, 101.60it/s]
 48%|████▊     | 24064/50176 [04:01<04:19, 100.51it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.00it/s]
 49%|████▉     | 24576/50176 [04:06<04:17, 99.35it/s] 
 49%|████▉     | 24832/50176 [04:08<04:08, 101.85it/s]
 50%|█████     | 25088/50176 [04:11<04:00, 104.30it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.35it/s]
 51%|█████     | 25600/50176 [04:16<03:57, 103.39it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.22it/s]
 52%|█████▏    | 26112/50176 [04:21<03:48, 105.12it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.32it/s]
 53%|█████▎    | 26624/50176 [04:26<03:53, 100.66it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.23it/s]
 54%|█████▍    | 27136/50176 [04:31<03:50, 100.15it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 101.96it/s]
 55%|█████▌    | 27648/50176 [04:36<03:33, 105.30it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.62it/s]
 56%|█████▌    | 28160/50176 [04:41<03:42, 99.08it/s] 
 57%|█████▋    | 28416/50176 [04:44<03:37, 99.94it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.26it/s]
 58%|█████▊    | 28928/50176 [04:49<03:29, 101.53it/s]
 58%|█████▊    | 29184/50176 [04:51<03:23, 103.06it/s]
 59%|█████▊    | 29440/50176 [04:54<03:33, 96.92it/s] 
 59%|█████▉    | 29696/50176 [04:57<03:30, 97.15it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.56it/s]
 60%|██████    | 30208/50176 [05:02<03:23, 98.26it/s]
 61%|██████    | 30464/50176 [05:04<03:12, 102.33it/s]
 61%|██████    | 30720/50176 [05:07<03:10, 101.94it/s]
 62%|██████▏   | 30976/50176 [05:09<03:05, 103.37it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.12it/s]
 63%|██████▎   | 31488/50176 [05:14<02:55, 106.43it/s]
 63%|██████▎   | 31744/50176 [05:16<02:55, 105.23it/s]
 64%|██████▍   | 32000/50176 [05:19<02:52, 105.30it/s]
 64%|██████▍   | 32256/50176 [05:21<02:49, 105.42it/s]
 65%|██████▍   | 32512/50176 [05:24<02:56, 100.15it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.67it/s]
 66%|██████▌   | 33024/50176 [05:29<02:48, 101.83it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.61it/s]
 67%|██████▋   | 33536/50176 [05:34<02:46, 99.97it/s] 
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.08it/s]
 68%|██████▊   | 34048/50176 [05:39<02:33, 104.94it/s]
 68%|██████▊   | 34304/50176 [05:41<02:27, 107.35it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.93it/s]
 69%|██████▉   | 34816/50176 [05:46<02:24, 105.94it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.82it/s]
 70%|███████   | 35328/50176 [05:51<02:19, 106.47it/s]
 71%|███████   | 35584/50176 [05:53<02:16, 106.63it/s]
 71%|███████▏  | 35840/50176 [05:56<02:17, 104.50it/s]
 72%|███████▏  | 36096/50176 [05:58<02:14, 104.43it/s]
 72%|███████▏  | 36352/50176 [06:01<02:13, 103.56it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.07it/s]
 73%|███████▎  | 36864/50176 [06:06<02:13, 99.86it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.29it/s]
 74%|███████▍  | 37376/50176 [06:11<02:01, 105.40it/s]
 75%|███████▌  | 37632/50176 [06:13<01:56, 107.46it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.76it/s]
 76%|███████▌  | 38144/50176 [06:18<01:52, 107.13it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.74it/s]
 77%|███████▋  | 38656/50176 [06:23<01:50, 104.43it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 103.98it/s]
 78%|███████▊  | 39168/50176 [06:28<01:49, 100.55it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.66it/s]
 79%|███████▉  | 39680/50176 [06:33<01:42, 101.94it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.34it/s]
 80%|████████  | 40192/50176 [06:38<01:37, 102.60it/s]
 81%|████████  | 40448/50176 [06:41<01:39, 98.25it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.43it/s]
 82%|████████▏ | 40960/50176 [06:46<01:32, 99.67it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.11it/s]
 83%|████████▎ | 41472/50176 [06:51<01:26, 100.83it/s]
 83%|████████▎ | 41728/50176 [06:53<01:21, 103.04it/s]
 84%|████████▎ | 41984/50176 [06:56<01:19, 103.17it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.52it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.30it/s]
 85%|████████▌ | 42752/50176 [07:03<01:09, 107.00it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.90it/s]
 86%|████████▌ | 43264/50176 [07:08<01:05, 105.36it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.28it/s]
 87%|████████▋ | 43776/50176 [07:13<01:02, 102.26it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.98it/s]
 88%|████████▊ | 44288/50176 [07:18<00:56, 104.23it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.34it/s]
 89%|████████▉ | 44800/50176 [07:23<00:52, 102.07it/s]
 90%|████████▉ | 45056/50176 [07:25<00:48, 105.94it/s]
 90%|█████████ | 45312/50176 [07:28<00:46, 104.91it/s]
 91%|█████████ | 45568/50176 [07:30<00:43, 107.02it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.62it/s]
 92%|█████████▏| 46080/50176 [07:35<00:39, 103.17it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.53it/s]
 93%|█████████▎| 46592/50176 [07:40<00:33, 105.57it/s]
 93%|█████████▎| 46848/50176 [07:42<00:31, 106.72it/s]
 94%|█████████▍| 47104/50176 [07:45<00:30, 101.66it/s]
 94%|█████████▍| 47360/50176 [07:48<00:28, 99.12it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.26it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.93it/s]
 96%|█████████▌| 48128/50176 [07:55<00:19, 105.62it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.94it/s]
 97%|█████████▋| 48640/50176 [08:00<00:14, 102.47it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.79it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.30it/s]
 98%|█████████▊| 49408/50176 [08:08<00:07, 101.20it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.89it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.70it/s] 
100%|██████████| 50176/50176 [08:14<00:00, 124.66it/s]
100%|██████████| 50176/50176 [08:14<00:00, 101.52it/s]
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6518	imagenet-zeroshot-val-top5: 0.8922
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt
new file mode 100644
index 0000000000000000000000000000000000000000..86aa04a904d8a6769e52d70fc0494bdb644c55c1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3fbb049794f64a124df09148e1879adf22a0ca0f5b0bbe31027c8669f5fcc14f
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f865dae6e364d14e15a290f0c0a7501deaa69e12
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_52.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:48,  1.41it/s]
  1%|          | 6/1000 [00:00<01:44,  9.52it/s]
  1%|          | 12/1000 [00:00<00:53, 18.63it/s]
  2%|▏         | 18/1000 [00:01<00:36, 27.06it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.21it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.04it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.53it/s]
  4%|▍         | 42/1000 [00:01<00:19, 47.95it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.51it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.70it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.71it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.40it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.82it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.31it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.61it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.98it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.16it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.42it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.52it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.51it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.45it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.45it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.43it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.47it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.53it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.51it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.38it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.25it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.26it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.21it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.13it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.12it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.08it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.89it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.78it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.89it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.84it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.05it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.03it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.95it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 56.87it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 56.97it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 57.06it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.14it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 56.97it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.21it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.33it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.53it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.52it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.50it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.61it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.58it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.50it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.32it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.38it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.47it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.57it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.36it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.37it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.23it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.37it/s]
 38%|███▊      | 378/1000 [00:07<00:11, 53.80it/s]
 38%|███▊      | 384/1000 [00:07<00:11, 54.52it/s]
 39%|███▉      | 390/1000 [00:07<00:11, 55.34it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 55.97it/s]
 40%|████      | 402/1000 [00:07<00:10, 56.51it/s]
 41%|████      | 408/1000 [00:07<00:10, 56.86it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.05it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.20it/s]
 43%|████▎     | 426/1000 [00:08<00:10, 57.32it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.48it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.49it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.32it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.38it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.46it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.46it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.44it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.59it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.52it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.50it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.50it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.42it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.45it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.37it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.45it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.28it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.42it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.40it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.53it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.58it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.48it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.50it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.55it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.45it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.41it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.55it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.54it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.47it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.57it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.49it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.46it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.58it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.59it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.67it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.49it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.42it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.57it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.52it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.63it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.65it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.60it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.64it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.65it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.68it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.55it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.32it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.40it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.46it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.45it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.42it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.46it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.50it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.52it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.59it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.68it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.53it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.65it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.60it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.63it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.67it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.47it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.47it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.50it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.49it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.48it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.51it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.58it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.72it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.75it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.67it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.72it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.66it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.67it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.62it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.67it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.64it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.54it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.66it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.65it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.61it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.61it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.59it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.57it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.66it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.71it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.69it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.70it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.73it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.67it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.67it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.51it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.56it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.61it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.36it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.03it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.07it/s]
+2022-12-21,11:03:10 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<09:51, 84.42it/s]
  1%|          | 512/50176 [00:05<08:58, 92.20it/s]
  2%|▏         | 768/50176 [00:07<08:17, 99.26it/s]
  2%|▏         | 1024/50176 [00:10<08:15, 99.21it/s]
  3%|▎         | 1280/50176 [00:13<08:13, 99.12it/s]
  3%|▎         | 1536/50176 [00:16<08:48, 91.99it/s]
  4%|▎         | 1792/50176 [00:18<08:36, 93.74it/s]
  4%|▍         | 2048/50176 [00:21<08:17, 96.76it/s]
  5%|▍         | 2304/50176 [00:23<08:07, 98.26it/s]
  5%|▌         | 2560/50176 [00:26<07:51, 100.97it/s]
  6%|▌         | 2816/50176 [00:29<08:00, 98.49it/s] 
  6%|▌         | 3072/50176 [00:31<07:53, 99.57it/s]
  7%|▋         | 3328/50176 [00:33<07:39, 101.96it/s]
  7%|▋         | 3584/50176 [00:36<07:29, 103.56it/s]
  8%|▊         | 3840/50176 [00:38<07:38, 100.98it/s]
  8%|▊         | 4096/50176 [00:41<07:53, 97.22it/s] 
  9%|▊         | 4352/50176 [00:44<07:49, 97.63it/s]
  9%|▉         | 4608/50176 [00:46<07:32, 100.81it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.90it/s]
 10%|█         | 5120/50176 [00:51<07:21, 102.13it/s]
 11%|█         | 5376/50176 [00:54<07:26, 100.29it/s]
 11%|█         | 5632/50176 [00:56<07:18, 101.56it/s]
 12%|█▏        | 5888/50176 [00:59<07:20, 100.52it/s]
 12%|█▏        | 6144/50176 [01:02<08:01, 91.51it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.82it/s]
 13%|█▎        | 6656/50176 [01:07<07:26, 97.46it/s]
 14%|█▍        | 6912/50176 [01:10<07:34, 95.23it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.18it/s]
 15%|█▍        | 7424/50176 [01:15<07:03, 100.86it/s]
 15%|█▌        | 7680/50176 [01:17<06:56, 101.99it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.17it/s]
 16%|█▋        | 8192/50176 [01:22<06:50, 102.29it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 102.85it/s]
 17%|█▋        | 8704/50176 [01:27<06:47, 101.76it/s]
 18%|█▊        | 8960/50176 [01:30<06:58, 98.39it/s] 
 18%|█▊        | 9216/50176 [01:32<06:38, 102.84it/s]
 19%|█▉        | 9472/50176 [01:35<06:34, 103.21it/s]
 19%|█▉        | 9728/50176 [01:38<06:42, 100.38it/s]
 20%|█▉        | 9984/50176 [01:40<06:41, 100.09it/s]
 20%|██        | 10240/50176 [01:42<06:27, 102.95it/s]
 21%|██        | 10496/50176 [01:45<06:27, 102.39it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.16it/s]
 22%|██▏       | 11008/50176 [01:50<06:26, 101.27it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.79it/s]
 23%|██▎       | 11520/50176 [01:55<06:12, 103.73it/s]
 23%|██▎       | 11776/50176 [01:57<06:11, 103.50it/s]
 24%|██▍       | 12032/50176 [02:00<06:15, 101.68it/s]
 24%|██▍       | 12288/50176 [02:02<06:08, 102.71it/s]
 25%|██▌       | 12544/50176 [02:05<06:14, 100.50it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 99.01it/s] 
 26%|██▌       | 13056/50176 [02:10<06:03, 102.07it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.84it/s]
 27%|██▋       | 13568/50176 [02:16<06:23, 95.55it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.55it/s]
 28%|██▊       | 14080/50176 [02:21<06:24, 93.79it/s]
 29%|██▊       | 14336/50176 [02:24<06:19, 94.32it/s]
 29%|██▉       | 14592/50176 [02:26<06:01, 98.36it/s]
 30%|██▉       | 14848/50176 [02:29<06:06, 96.35it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.51it/s]
 31%|███       | 15360/50176 [02:34<05:55, 97.83it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.76it/s]
 32%|███▏      | 15872/50176 [02:39<05:47, 98.83it/s]
 32%|███▏      | 16128/50176 [02:42<05:39, 100.23it/s]
 33%|███▎      | 16384/50176 [02:44<05:31, 101.83it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.91it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.89it/s]
 34%|███▍      | 17152/50176 [02:52<05:25, 101.48it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.29it/s]
 35%|███▌      | 17664/50176 [02:57<05:16, 102.64it/s]
 36%|███▌      | 17920/50176 [02:59<05:09, 104.11it/s]
 36%|███▌      | 18176/50176 [03:02<05:13, 102.10it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.91it/s]
 37%|███▋      | 18688/50176 [03:07<05:15, 99.73it/s] 
 38%|███▊      | 18944/50176 [03:10<05:16, 98.61it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.84it/s]
 39%|███▉      | 19456/50176 [03:15<05:17, 96.64it/s]
 39%|███▉      | 19712/50176 [03:18<05:19, 95.35it/s]
 40%|███▉      | 19968/50176 [03:21<05:13, 96.45it/s]
 40%|████      | 20224/50176 [03:23<04:52, 102.27it/s]
 41%|████      | 20480/50176 [03:25<04:49, 102.58it/s]
 41%|████▏     | 20736/50176 [03:28<04:42, 104.03it/s]
 42%|████▏     | 20992/50176 [03:30<04:36, 105.56it/s]
 42%|████▏     | 21248/50176 [03:33<04:40, 103.27it/s]
 43%|████▎     | 21504/50176 [03:35<04:35, 103.95it/s]
 43%|████▎     | 21760/50176 [03:38<04:33, 103.72it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.42it/s]
 44%|████▍     | 22272/50176 [03:42<04:19, 107.35it/s]
 45%|████▍     | 22528/50176 [03:45<04:18, 106.78it/s]
 45%|████▌     | 22784/50176 [03:47<04:23, 103.81it/s]
 46%|████▌     | 23040/50176 [03:50<04:26, 101.67it/s]
 46%|████▋     | 23296/50176 [03:53<04:28, 100.01it/s]
 47%|████▋     | 23552/50176 [03:55<04:35, 96.77it/s] 
 47%|████▋     | 23808/50176 [03:58<04:19, 101.47it/s]
 48%|████▊     | 24064/50176 [04:00<04:19, 100.49it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.12it/s]
 49%|████▉     | 24576/50176 [04:05<04:17, 99.43it/s] 
 49%|████▉     | 24832/50176 [04:08<04:09, 101.77it/s]
 50%|█████     | 25088/50176 [04:10<04:00, 104.18it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.33it/s]
 51%|█████     | 25600/50176 [04:15<03:57, 103.48it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.32it/s]
 52%|█████▏    | 26112/50176 [04:20<03:48, 105.10it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.29it/s]
 53%|█████▎    | 26624/50176 [04:25<03:54, 100.58it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.29it/s]
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.21it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 102.09it/s]
 55%|█████▌    | 27648/50176 [04:35<03:34, 105.26it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.53it/s]
 56%|█████▌    | 28160/50176 [04:40<03:42, 99.10it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:37, 99.90it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.10it/s]
 58%|█████▊    | 28928/50176 [04:48<03:29, 101.57it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.14it/s]
 59%|█████▊    | 29440/50176 [04:53<03:33, 96.98it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:30, 97.20it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.49it/s]
 60%|██████    | 30208/50176 [05:01<03:23, 98.16it/s]
 61%|██████    | 30464/50176 [05:03<03:12, 102.41it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.81it/s]
 62%|██████▏   | 30976/50176 [05:08<03:05, 103.30it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.13it/s]
 63%|██████▎   | 31488/50176 [05:13<02:55, 106.36it/s]
 63%|██████▎   | 31744/50176 [05:16<02:55, 105.28it/s]
 64%|██████▍   | 32000/50176 [05:18<02:52, 105.36it/s]
 64%|██████▍   | 32256/50176 [05:20<02:49, 105.45it/s]
 65%|██████▍   | 32512/50176 [05:23<02:56, 100.28it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.76it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 101.83it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.70it/s]
 67%|██████▋   | 33536/50176 [05:33<02:46, 100.10it/s]
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.08it/s]
 68%|██████▊   | 34048/50176 [05:38<02:33, 105.08it/s]
 68%|██████▊   | 34304/50176 [05:40<02:28, 107.18it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.87it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 105.99it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.66it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.49it/s]
 71%|███████   | 35584/50176 [05:52<02:16, 106.68it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.52it/s]
 72%|███████▏  | 36096/50176 [05:57<02:14, 104.38it/s]
 72%|███████▏  | 36352/50176 [06:00<02:13, 103.58it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.10it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.96it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.37it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.23it/s]
 75%|███████▌  | 37632/50176 [06:12<01:56, 107.51it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.64it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 107.07it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.84it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.38it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.02it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.64it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.70it/s]
 79%|███████▉  | 39680/50176 [06:32<01:43, 101.85it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.31it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.69it/s]
 81%|████████  | 40448/50176 [06:40<01:38, 98.31it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.33it/s]
 82%|████████▏ | 40960/50176 [06:45<01:32, 99.69it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.00it/s]
 83%|████████▎ | 41472/50176 [06:50<01:26, 100.84it/s]
 83%|████████▎ | 41728/50176 [06:53<01:22, 102.99it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.21it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.63it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.34it/s]
 85%|████████▌ | 42752/50176 [07:02<01:09, 107.01it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.93it/s]
 86%|████████▌ | 43264/50176 [07:07<01:05, 105.36it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.27it/s]
 87%|████████▋ | 43776/50176 [07:12<01:02, 102.26it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.81it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.28it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.37it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:24<00:48, 105.97it/s]
 90%|█████████ | 45312/50176 [07:27<00:46, 104.93it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 107.05it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.61it/s]
 92%|█████████▏| 46080/50176 [07:34<00:39, 103.14it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.58it/s]
 93%|█████████▎| 46592/50176 [07:39<00:33, 105.57it/s]
 93%|█████████▎| 46848/50176 [07:41<00:31, 106.69it/s]
 94%|█████████▍| 47104/50176 [07:44<00:30, 101.65it/s]
 94%|█████████▍| 47360/50176 [07:47<00:28, 99.14it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.14it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.98it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 105.61it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.99it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 102.46it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.79it/s]
 98%|█████████▊| 49152/50176 [08:04<00:10, 100.30it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.26it/s]
 99%|█████████▉| 49664/50176 [08:09<00:05, 100.90it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.72it/s] 
100%|██████████| 50176/50176 [08:13<00:00, 124.36it/s]
100%|██████████| 50176/50176 [08:13<00:00, 101.65it/s]
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6474	imagenet-zeroshot-val-top5: 0.8909
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c7625b575faa44da9cbebae9c13f40198f653cf2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4464928b5caa78db2566d3822dc553ccf331590e2c278f227e863a190057ae33
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..34180aeed76f5ee8c6578728b602b6c1c6b6e67c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt).
+2022-12-21,11:11:47 | INFO | Model:
+2022-12-21,11:11:47 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:47 | INFO | Params:
+2022-12-21,11:11:47 | INFO |   batch_size: 256
+2022-12-21,11:11:47 | INFO |   beta1: 0.9
+2022-12-21,11:11:47 | INFO |   beta2: 0.98
+2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/checkpoints
+2022-12-21,11:11:47 | INFO |   copy_codebase: False
+2022-12-21,11:11:47 | INFO |   csv_caption_key: title
+2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:47 | INFO |   csv_separator: 	
+2022-12-21,11:11:47 | INFO |   dataset_resampled: False
+2022-12-21,11:11:47 | INFO |   dataset_type: auto
+2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:47 | INFO |   debug: False
+2022-12-21,11:11:47 | INFO |   device: cuda:0
+2022-12-21,11:11:47 | INFO |   dist_backend: nccl
+2022-12-21,11:11:47 | INFO |   dist_url: env://
+2022-12-21,11:11:47 | INFO |   distributed: False
+2022-12-21,11:11:47 | INFO |   epochs: 32
+2022-12-21,11:11:47 | INFO |   eps: 1e-06
+2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:47 | INFO |   gather_with_grad: False
+2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:47 | INFO |   horovod: False
+2022-12-21,11:11:47 | INFO |   image_mean: None
+2022-12-21,11:11:47 | INFO |   image_std: None
+2022-12-21,11:11:47 | INFO |   imagenet_v2: None
+2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:47 | INFO |   local_loss: False
+2022-12-21,11:11:47 | INFO |   local_rank: 0
+2022-12-21,11:11:47 | INFO |   lock_image: False
+2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:47 | INFO |   log_level: 20
+2022-12-21,11:11:47 | INFO |   log_local: False
+2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/out.log
+2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:47 | INFO |   lr: 0.0005
+2022-12-21,11:11:47 | INFO |   model: ViT-B-16
+2022-12-21,11:11:47 | INFO |   name: epoch_53.pt
+2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt
+2022-12-21,11:11:47 | INFO |   pretrained_image: False
+2022-12-21,11:11:47 | INFO |   rank: 0
+2022-12-21,11:11:47 | INFO |   report_to: 
+2022-12-21,11:11:47 | INFO |   resume: None
+2022-12-21,11:11:47 | INFO |   save_frequency: 1
+2022-12-21,11:11:47 | INFO |   save_most_recent: False
+2022-12-21,11:11:47 | INFO |   seed: 0
+2022-12-21,11:11:47 | INFO |   skip_scheduler: False
+2022-12-21,11:11:47 | INFO |   tensorboard: False
+2022-12-21,11:11:47 | INFO |   tensorboard_path: 
+2022-12-21,11:11:47 | INFO |   torchscript: False
+2022-12-21,11:11:47 | INFO |   trace: False
+2022-12-21,11:11:47 | INFO |   train_data: None
+2022-12-21,11:11:47 | INFO |   train_num_samples: None
+2022-12-21,11:11:47 | INFO |   use_bn_sync: False
+2022-12-21,11:11:47 | INFO |   val_data: None
+2022-12-21,11:11:47 | INFO |   val_frequency: 1
+2022-12-21,11:11:47 | INFO |   val_num_samples: None
+2022-12-21,11:11:47 | INFO |   wandb: False
+2022-12-21,11:11:47 | INFO |   wandb_notes: 
+2022-12-21,11:11:47 | INFO |   warmup: 10000
+2022-12-21,11:11:47 | INFO |   wd: 0.2
+2022-12-21,11:11:47 | INFO |   workers: 1
+2022-12-21,11:11:47 | INFO |   world_size: 1
+2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:47 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:07,  1.64it/s]
  1%|          | 7/1000 [00:00<01:20, 12.27it/s]
  1%|▏         | 13/1000 [00:00<00:44, 21.95it/s]
  2%|▏         | 19/1000 [00:00<00:32, 30.29it/s]
  2%|▎         | 25/1000 [00:01<00:26, 37.11it/s]
  3%|▎         | 31/1000 [00:01<00:22, 42.58it/s]
  4%|▎         | 37/1000 [00:01<00:20, 46.72it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.76it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.96it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.42it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.75it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.60it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.12it/s]
  8%|▊         | 79/1000 [00:01<00:16, 56.65it/s]
  8%|▊         | 85/1000 [00:02<00:16, 57.11it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.35it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.70it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.84it/s]
 11%|█         | 109/1000 [00:02<00:15, 58.05it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.20it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.07it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.09it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 58.04it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.08it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.08it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.06it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.83it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.84it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.80it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.78it/s]
 19%|█▉        | 193/1000 [00:03<00:13, 57.73it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.65it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.53it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.65it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.64it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.47it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.61it/s]
 25%|██▌       | 253/1000 [00:04<00:12, 57.70it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.81it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.78it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.63it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.77it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.90it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.02it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.19it/s]
 30%|███       | 301/1000 [00:05<00:12, 58.18it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.14it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.31it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.33it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.15it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 57.67it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 57.89it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 55.81it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 56.44it/s]
 36%|███▌      | 361/1000 [00:06<00:11, 56.97it/s]
 37%|███▋      | 367/1000 [00:06<00:11, 57.36it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 57.70it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 57.78it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.82it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.60it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 57.73it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.86it/s]
 41%|████      | 409/1000 [00:07<00:10, 57.98it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.13it/s]
 43%|████▎     | 427/1000 [00:07<00:09, 58.16it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.29it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.29it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.13it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.16it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.19it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.11it/s]
 48%|████▊     | 475/1000 [00:08<00:09, 58.27it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.09it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.18it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 57.87it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 57.62it/s]
 50%|█████     | 505/1000 [00:09<00:08, 57.79it/s]
 51%|█████     | 511/1000 [00:09<00:08, 57.77it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 57.97it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.06it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.14it/s]
 54%|█████▍    | 541/1000 [00:09<00:08, 57.37it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 57.57it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 57.66it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 57.82it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 57.59it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 57.62it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 57.82it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 57.99it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 57.98it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 57.96it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.10it/s]
 61%|██████    | 607/1000 [00:11<00:06, 57.98it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.32it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.28it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.25it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 57.69it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 57.79it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 57.99it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.06it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 57.64it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 57.78it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 57.82it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 57.83it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 57.96it/s]
 70%|███████   | 703/1000 [00:12<00:05, 57.87it/s]
 71%|███████   | 709/1000 [00:12<00:05, 57.97it/s]
 72%|███████▏  | 715/1000 [00:12<00:04, 58.03it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.08it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 57.91it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 57.97it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.08it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.15it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.30it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.17it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.24it/s]
 78%|███████▊  | 775/1000 [00:13<00:03, 58.21it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.23it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.24it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.11it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.08it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.12it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.09it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.21it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.37it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.42it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.30it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.35it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.34it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.34it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.25it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.30it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.26it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.18it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.26it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.24it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.17it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.13it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.23it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.18it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.27it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.31it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.21it/s]
 95%|█████████▍| 949/1000 [00:16<00:00, 58.00it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.19it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.12it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.27it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 58.06it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.98it/s]
+2022-12-21,11:12:05 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:04<14:36, 56.97it/s]
  1%|          | 512/50176 [00:07<11:03, 74.86it/s]
  2%|▏         | 768/50176 [00:09<09:47, 84.07it/s]
  2%|▏         | 1024/50176 [00:12<09:17, 88.18it/s]
  3%|▎         | 1280/50176 [00:15<09:13, 88.34it/s]
  3%|▎         | 1536/50176 [00:18<09:30, 85.31it/s]
  4%|▎         | 1792/50176 [00:21<09:16, 86.95it/s]
  4%|▍         | 2048/50176 [00:23<08:48, 91.09it/s]
  5%|▍         | 2304/50176 [00:26<08:48, 90.59it/s]
  5%|▌         | 2560/50176 [00:29<08:23, 94.63it/s]
  6%|▌         | 2816/50176 [00:32<08:32, 92.47it/s]
  6%|▌         | 3072/50176 [00:34<08:23, 93.60it/s]
  7%|▋         | 3328/50176 [00:37<08:10, 95.42it/s]
  7%|▋         | 3584/50176 [00:39<07:57, 97.53it/s]
  8%|▊         | 3840/50176 [00:42<08:06, 95.16it/s]
  8%|▊         | 4096/50176 [00:45<08:25, 91.14it/s]
  9%|▊         | 4352/50176 [00:48<08:06, 94.12it/s]
  9%|▉         | 4608/50176 [00:50<07:58, 95.23it/s]
 10%|▉         | 4864/50176 [00:53<07:45, 97.38it/s]
 10%|█         | 5120/50176 [00:55<07:35, 98.91it/s]
 11%|█         | 5376/50176 [00:58<07:40, 97.29it/s]
 11%|█         | 5632/50176 [01:01<07:53, 94.13it/s]
 12%|█▏        | 5888/50176 [01:04<07:42, 95.82it/s]
 12%|█▏        | 6144/50176 [01:07<08:05, 90.73it/s]
 13%|█▎        | 6400/50176 [01:09<07:51, 92.89it/s]
 13%|█▎        | 6656/50176 [01:12<07:33, 95.98it/s]
 14%|█▍        | 6912/50176 [01:15<07:36, 94.75it/s]
 14%|█▍        | 7168/50176 [01:17<07:23, 96.87it/s]
 15%|█▍        | 7424/50176 [01:20<07:16, 97.95it/s]
 15%|█▌        | 7680/50176 [01:22<07:10, 98.81it/s]
 16%|█▌        | 7936/50176 [01:25<07:03, 99.78it/s]
 16%|█▋        | 8192/50176 [01:27<07:06, 98.54it/s]
 17%|█▋        | 8448/50176 [01:30<06:54, 100.61it/s]
 17%|█▋        | 8704/50176 [01:32<06:58, 99.19it/s] 
 18%|█▊        | 8960/50176 [01:35<07:03, 97.31it/s]
 18%|█▊        | 9216/50176 [01:38<06:49, 99.93it/s]
 19%|█▉        | 9472/50176 [01:40<06:46, 100.04it/s]
 19%|█▉        | 9728/50176 [01:43<06:51, 98.38it/s] 
 20%|█▉        | 9984/50176 [01:48<08:44, 76.69it/s]
 20%|██        | 10240/50176 [01:52<09:32, 69.74it/s]
 21%|██        | 10496/50176 [01:55<08:29, 77.85it/s]
 21%|██▏       | 10752/50176 [01:58<08:04, 81.37it/s]
 22%|██▏       | 11008/50176 [02:00<07:28, 87.35it/s]
 22%|██▏       | 11264/50176 [02:03<07:07, 91.10it/s]
 23%|██▎       | 11520/50176 [02:05<06:50, 94.25it/s]
 23%|██▎       | 11776/50176 [02:07<06:33, 97.57it/s]
 24%|██▍       | 12032/50176 [02:10<06:32, 97.12it/s]
 24%|██▍       | 12288/50176 [02:12<06:19, 99.84it/s]
 25%|██▌       | 12544/50176 [02:15<06:11, 101.27it/s]
 26%|██▌       | 12800/50176 [02:17<06:10, 100.80it/s]
 26%|██▌       | 13056/50176 [02:20<06:06, 101.41it/s]
 27%|██▋       | 13312/50176 [02:22<06:03, 101.46it/s]
 27%|██▋       | 13568/50176 [02:25<06:12, 98.23it/s] 
 28%|██▊       | 13824/50176 [02:28<06:24, 94.44it/s]
 28%|██▊       | 14080/50176 [02:31<06:22, 94.40it/s]
 29%|██▊       | 14336/50176 [02:34<06:23, 93.48it/s]
 29%|██▉       | 14592/50176 [02:36<06:08, 96.56it/s]
 30%|██▉       | 14848/50176 [02:39<06:31, 90.25it/s]
 30%|███       | 15104/50176 [02:42<06:17, 92.90it/s]
 31%|███       | 15360/50176 [02:45<06:08, 94.38it/s]
 31%|███       | 15616/50176 [02:47<05:55, 97.16it/s]
 32%|███▏      | 15872/50176 [02:52<07:27, 76.65it/s]
 32%|███▏      | 16128/50176 [02:55<06:49, 83.05it/s]
 33%|███▎      | 16384/50176 [02:57<06:22, 88.28it/s]
 33%|███▎      | 16640/50176 [03:00<06:23, 87.40it/s]
 34%|███▎      | 16896/50176 [03:03<06:05, 90.96it/s]
 34%|███▍      | 17152/50176 [03:05<05:50, 94.12it/s]
 35%|███▍      | 17408/50176 [03:08<05:45, 94.83it/s]
 35%|███▌      | 17664/50176 [03:10<05:30, 98.44it/s]
 36%|███▌      | 17920/50176 [03:13<05:24, 99.46it/s]
 36%|███▌      | 18176/50176 [03:15<05:22, 99.29it/s]
 37%|███▋      | 18432/50176 [03:18<05:14, 100.81it/s]
 37%|███▋      | 18688/50176 [03:21<05:26, 96.52it/s] 
 38%|███▊      | 18944/50176 [03:23<05:24, 96.17it/s]
 38%|███▊      | 19200/50176 [03:26<05:27, 94.56it/s]
 39%|███▉      | 19456/50176 [03:29<05:25, 94.26it/s]
 39%|███▉      | 19712/50176 [03:34<06:46, 74.86it/s]
 40%|███▉      | 19968/50176 [03:39<07:29, 67.17it/s]
 40%|████      | 20224/50176 [03:41<06:29, 76.87it/s]
 41%|████      | 20480/50176 [03:46<07:22, 67.13it/s]
 41%|████▏     | 20736/50176 [03:48<06:32, 75.08it/s]
 42%|████▏     | 20992/50176 [03:51<05:50, 83.31it/s]
 42%|████▏     | 21248/50176 [03:53<05:34, 86.42it/s]
 43%|████▎     | 21504/50176 [03:58<06:36, 72.27it/s]
 43%|████▎     | 21760/50176 [04:01<05:56, 79.72it/s]
 44%|████▍     | 22016/50176 [04:03<05:21, 87.51it/s]
 44%|████▍     | 22272/50176 [04:05<05:01, 92.49it/s]
 45%|████▍     | 22528/50176 [04:08<04:44, 97.29it/s]
 45%|████▌     | 22784/50176 [04:10<04:41, 97.25it/s]
 46%|████▌     | 23040/50176 [04:13<04:41, 96.41it/s]
 46%|████▋     | 23296/50176 [04:16<04:41, 95.56it/s]
 47%|████▋     | 23552/50176 [04:18<04:38, 95.50it/s]
 47%|████▋     | 23808/50176 [04:21<04:23, 100.11it/s]
 48%|████▊     | 24064/50176 [04:23<04:15, 102.04it/s]
 48%|████▊     | 24320/50176 [04:25<04:14, 101.51it/s]
 49%|████▉     | 24576/50176 [04:28<04:14, 100.71it/s]
 49%|████▉     | 24832/50176 [04:31<04:08, 101.90it/s]
 50%|█████     | 25088/50176 [04:33<04:01, 104.01it/s]
 51%|█████     | 25344/50176 [04:35<03:53, 106.30it/s]
 51%|█████     | 25600/50176 [04:38<03:55, 104.26it/s]
 52%|█████▏    | 25856/50176 [04:40<03:51, 105.02it/s]
 52%|█████▏    | 26112/50176 [04:42<03:46, 106.17it/s]
 53%|█████▎    | 26368/50176 [04:45<03:45, 105.67it/s]
 53%|█████▎    | 26624/50176 [04:47<03:43, 105.45it/s]
 54%|█████▎    | 26880/50176 [04:50<03:42, 104.83it/s]
 54%|█████▍    | 27136/50176 [04:52<03:43, 103.30it/s]
 55%|█████▍    | 27392/50176 [04:55<03:32, 106.97it/s]
 55%|█████▌    | 27648/50176 [04:57<03:27, 108.71it/s]
 56%|█████▌    | 27904/50176 [05:00<03:35, 103.22it/s]
 56%|█████▌    | 28160/50176 [05:02<03:37, 101.09it/s]
 57%|█████▋    | 28416/50176 [05:05<03:34, 101.33it/s]
 57%|█████▋    | 28672/50176 [05:07<03:31, 101.58it/s]
 58%|█████▊    | 28928/50176 [05:10<03:22, 104.86it/s]
 58%|█████▊    | 29184/50176 [05:12<03:22, 103.71it/s]
 59%|█████▊    | 29440/50176 [05:15<03:32, 97.48it/s] 
 59%|█████▉    | 29696/50176 [05:18<03:35, 95.13it/s]
 60%|█████▉    | 29952/50176 [05:20<03:28, 97.17it/s]
 60%|██████    | 30208/50176 [05:23<03:21, 99.16it/s]
 61%|██████    | 30464/50176 [05:25<03:09, 104.27it/s]
 61%|██████    | 30720/50176 [05:28<03:08, 103.46it/s]
 62%|██████▏   | 30976/50176 [05:30<03:00, 106.18it/s]
 62%|██████▏   | 31232/50176 [05:32<02:58, 105.83it/s]
 63%|██████▎   | 31488/50176 [05:35<02:53, 107.62it/s]
 63%|██████▎   | 31744/50176 [05:37<02:49, 108.51it/s]
 64%|██████▍   | 32000/50176 [05:39<02:48, 108.14it/s]
 64%|██████▍   | 32256/50176 [05:42<02:48, 106.48it/s]
 65%|██████▍   | 32512/50176 [05:44<02:53, 102.00it/s]
 65%|██████▌   | 32768/50176 [05:47<02:44, 105.53it/s]
 66%|██████▌   | 33024/50176 [05:49<02:42, 105.69it/s]
 66%|██████▋   | 33280/50176 [05:51<02:36, 107.90it/s]
 67%|██████▋   | 33536/50176 [05:54<02:37, 105.62it/s]
 67%|██████▋   | 33792/50176 [05:56<02:34, 105.97it/s]
 68%|██████▊   | 34048/50176 [05:59<02:29, 108.20it/s]
 68%|██████▊   | 34304/50176 [06:01<02:24, 109.46it/s]
 69%|██████▉   | 34560/50176 [06:03<02:26, 106.57it/s]
 69%|██████▉   | 34816/50176 [06:06<02:25, 105.41it/s]
 70%|██████▉   | 35072/50176 [06:08<02:25, 103.70it/s]
 70%|███████   | 35328/50176 [06:11<02:19, 106.59it/s]
 71%|███████   | 35584/50176 [06:13<02:16, 106.64it/s]
 71%|███████▏  | 35840/50176 [06:16<02:17, 103.94it/s]
 72%|███████▏  | 36096/50176 [06:18<02:12, 106.58it/s]
 72%|███████▏  | 36352/50176 [06:20<02:10, 105.54it/s]
 73%|███████▎  | 36608/50176 [06:23<02:10, 103.80it/s]
 73%|███████▎  | 36864/50176 [06:26<02:12, 100.78it/s]
 74%|███████▍  | 37120/50176 [06:28<02:08, 101.72it/s]
 74%|███████▍  | 37376/50176 [06:30<02:00, 105.95it/s]
 75%|███████▌  | 37632/50176 [06:33<01:57, 106.44it/s]
 76%|███████▌  | 37888/50176 [06:35<01:55, 106.46it/s]
 76%|███████▌  | 38144/50176 [06:38<01:52, 107.16it/s]
 77%|███████▋  | 38400/50176 [06:40<01:51, 105.55it/s]
 77%|███████▋  | 38656/50176 [06:43<01:50, 104.48it/s]
 78%|███████▊  | 38912/50176 [06:45<01:47, 104.30it/s]
 78%|███████▊  | 39168/50176 [06:48<01:47, 102.64it/s]
 79%|███████▊  | 39424/50176 [06:50<01:42, 104.70it/s]
 79%|███████▉  | 39680/50176 [06:52<01:40, 104.63it/s]
 80%|███████▉  | 39936/50176 [06:55<01:35, 107.37it/s]
 80%|████████  | 40192/50176 [06:57<01:33, 106.49it/s]
 81%|████████  | 40448/50176 [07:00<01:34, 102.44it/s]
 81%|████████  | 40704/50176 [07:03<01:36, 98.23it/s] 
 82%|████████▏ | 40960/50176 [07:05<01:29, 102.80it/s]
 82%|████████▏ | 41216/50176 [07:07<01:25, 104.31it/s]
 83%|████████▎ | 41472/50176 [07:10<01:22, 105.09it/s]
 83%|████████▎ | 41728/50176 [07:12<01:19, 106.36it/s]
 84%|████████▎ | 41984/50176 [07:14<01:16, 106.51it/s]
 84%|████████▍ | 42240/50176 [07:17<01:13, 108.17it/s]
 85%|████████▍ | 42496/50176 [07:19<01:10, 109.34it/s]
 85%|████████▌ | 42752/50176 [07:24<01:29, 82.52it/s] 
 86%|████████▌ | 43008/50176 [07:26<01:21, 88.29it/s]
 86%|████████▌ | 43264/50176 [07:29<01:14, 93.30it/s]
 87%|████████▋ | 43520/50176 [07:36<01:45, 63.23it/s]
 87%|████████▋ | 43776/50176 [07:40<01:46, 59.86it/s]
 88%|████████▊ | 44032/50176 [07:43<01:28, 69.57it/s]
 88%|████████▊ | 44288/50176 [07:45<01:16, 77.47it/s]
 89%|████████▉ | 44544/50176 [07:48<01:08, 82.46it/s]
 89%|████████▉ | 44800/50176 [07:50<01:01, 87.90it/s]
 90%|████████▉ | 45056/50176 [07:53<00:54, 93.97it/s]
 90%|█████████ | 45312/50176 [07:55<00:50, 95.43it/s]
 91%|█████████ | 45568/50176 [07:57<00:45, 101.35it/s]
 91%|█████████▏| 45824/50176 [08:00<00:43, 100.94it/s]
 92%|█████████▏| 46080/50176 [08:03<00:40, 100.08it/s]
 92%|█████████▏| 46336/50176 [08:05<00:38, 100.74it/s]
 93%|█████████▎| 46592/50176 [08:07<00:34, 103.87it/s]
 93%|█████████▎| 46848/50176 [08:10<00:32, 103.70it/s]
 94%|█████████▍| 47104/50176 [08:13<00:30, 99.11it/s] 
 94%|█████████▍| 47360/50176 [08:15<00:28, 99.06it/s]
 95%|█████████▍| 47616/50176 [08:18<00:25, 98.79it/s]
 95%|█████████▌| 47872/50176 [08:20<00:22, 101.25it/s]
 96%|█████████▌| 48128/50176 [08:23<00:19, 103.61it/s]
 96%|█████████▋| 48384/50176 [08:25<00:17, 103.34it/s]
 97%|█████████▋| 48640/50176 [08:27<00:14, 104.21it/s]
 97%|█████████▋| 48896/50176 [08:30<00:12, 104.13it/s]
 98%|█████████▊| 49152/50176 [08:32<00:09, 102.64it/s]
 98%|█████████▊| 49408/50176 [08:35<00:07, 102.13it/s]
 99%|█████████▉| 49664/50176 [08:38<00:05, 101.28it/s]
 99%|█████████▉| 49920/50176 [08:41<00:02, 95.67it/s] 
100%|██████████| 50176/50176 [08:41<00:00, 125.23it/s]
100%|██████████| 50176/50176 [08:41<00:00, 96.17it/s] 
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6545	imagenet-zeroshot-val-top5: 0.8941
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt
new file mode 100644
index 0000000000000000000000000000000000000000..1cc4157d1712a0a32d6e4970673cd23189161165
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dada855f041d99847d7bed0145ea46640ba57d90f6939aad88bc7b6d82388b07
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f1615afe3d233d7c001f59b309ace4f6811f2a55
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt).
+2022-12-21,11:11:45 | INFO | Model:
+2022-12-21,11:11:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:45 | INFO | Params:
+2022-12-21,11:11:45 | INFO |   batch_size: 256
+2022-12-21,11:11:45 | INFO |   beta1: 0.9
+2022-12-21,11:11:45 | INFO |   beta2: 0.98
+2022-12-21,11:11:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/checkpoints
+2022-12-21,11:11:45 | INFO |   copy_codebase: False
+2022-12-21,11:11:45 | INFO |   csv_caption_key: title
+2022-12-21,11:11:45 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:45 | INFO |   csv_separator: 	
+2022-12-21,11:11:45 | INFO |   dataset_resampled: False
+2022-12-21,11:11:45 | INFO |   dataset_type: auto
+2022-12-21,11:11:45 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:45 | INFO |   debug: False
+2022-12-21,11:11:45 | INFO |   device: cuda:0
+2022-12-21,11:11:45 | INFO |   dist_backend: nccl
+2022-12-21,11:11:45 | INFO |   dist_url: env://
+2022-12-21,11:11:45 | INFO |   distributed: False
+2022-12-21,11:11:45 | INFO |   epochs: 32
+2022-12-21,11:11:45 | INFO |   eps: 1e-06
+2022-12-21,11:11:45 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:45 | INFO |   gather_with_grad: False
+2022-12-21,11:11:45 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:45 | INFO |   horovod: False
+2022-12-21,11:11:45 | INFO |   image_mean: None
+2022-12-21,11:11:45 | INFO |   image_std: None
+2022-12-21,11:11:45 | INFO |   imagenet_v2: None
+2022-12-21,11:11:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:45 | INFO |   local_loss: False
+2022-12-21,11:11:45 | INFO |   local_rank: 0
+2022-12-21,11:11:45 | INFO |   lock_image: False
+2022-12-21,11:11:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:45 | INFO |   log_level: 20
+2022-12-21,11:11:45 | INFO |   log_local: False
+2022-12-21,11:11:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/out.log
+2022-12-21,11:11:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:45 | INFO |   lr: 0.0005
+2022-12-21,11:11:45 | INFO |   model: ViT-B-16
+2022-12-21,11:11:45 | INFO |   name: epoch_54.pt
+2022-12-21,11:11:45 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:45 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:45 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt
+2022-12-21,11:11:45 | INFO |   pretrained_image: False
+2022-12-21,11:11:45 | INFO |   rank: 0
+2022-12-21,11:11:45 | INFO |   report_to: 
+2022-12-21,11:11:45 | INFO |   resume: None
+2022-12-21,11:11:45 | INFO |   save_frequency: 1
+2022-12-21,11:11:45 | INFO |   save_most_recent: False
+2022-12-21,11:11:45 | INFO |   seed: 0
+2022-12-21,11:11:45 | INFO |   skip_scheduler: False
+2022-12-21,11:11:45 | INFO |   tensorboard: False
+2022-12-21,11:11:45 | INFO |   tensorboard_path: 
+2022-12-21,11:11:45 | INFO |   torchscript: False
+2022-12-21,11:11:45 | INFO |   trace: False
+2022-12-21,11:11:45 | INFO |   train_data: None
+2022-12-21,11:11:45 | INFO |   train_num_samples: None
+2022-12-21,11:11:45 | INFO |   use_bn_sync: False
+2022-12-21,11:11:45 | INFO |   val_data: None
+2022-12-21,11:11:45 | INFO |   val_frequency: 1
+2022-12-21,11:11:45 | INFO |   val_num_samples: None
+2022-12-21,11:11:45 | INFO |   wandb: False
+2022-12-21,11:11:45 | INFO |   wandb_notes: 
+2022-12-21,11:11:45 | INFO |   warmup: 10000
+2022-12-21,11:11:45 | INFO |   wd: 0.2
+2022-12-21,11:11:45 | INFO |   workers: 1
+2022-12-21,11:11:45 | INFO |   world_size: 1
+2022-12-21,11:11:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:46 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:46 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:23,  1.24it/s]
  1%|          | 6/1000 [00:00<01:56,  8.52it/s]
  1%|          | 11/1000 [00:01<01:02, 15.72it/s]
  2%|▏         | 17/1000 [00:01<00:41, 23.89it/s]
  2%|▏         | 23/1000 [00:01<00:31, 30.98it/s]
  3%|▎         | 29/1000 [00:01<00:26, 37.04it/s]
  4%|▎         | 35/1000 [00:01<00:22, 42.14it/s]
  4%|▍         | 41/1000 [00:01<00:20, 45.82it/s]
  5%|▍         | 47/1000 [00:01<00:19, 48.10it/s]
  5%|▌         | 53/1000 [00:01<00:18, 49.95it/s]
  6%|▌         | 59/1000 [00:01<00:18, 51.19it/s]
  6%|▋         | 65/1000 [00:02<00:17, 52.22it/s]
  7%|▋         | 71/1000 [00:02<00:17, 52.90it/s]
  8%|▊         | 77/1000 [00:02<00:17, 54.08it/s]
  8%|▊         | 83/1000 [00:02<00:16, 55.04it/s]
  9%|▉         | 89/1000 [00:02<00:16, 55.84it/s]
 10%|▉         | 95/1000 [00:02<00:16, 56.33it/s]
 10%|█         | 101/1000 [00:02<00:15, 56.71it/s]
 11%|█         | 107/1000 [00:02<00:15, 56.93it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.21it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 57.32it/s]
 12%|█▎        | 125/1000 [00:03<00:15, 57.34it/s]
 13%|█▎        | 131/1000 [00:03<00:15, 57.47it/s]
 14%|█▎        | 137/1000 [00:03<00:15, 57.42it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.37it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.48it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.57it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.47it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.37it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.38it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.17it/s]
 18%|█▊        | 185/1000 [00:04<00:14, 57.06it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.09it/s]
 20%|█▉        | 197/1000 [00:04<00:14, 57.09it/s]
 20%|██        | 203/1000 [00:04<00:13, 56.94it/s]
 21%|██        | 209/1000 [00:04<00:13, 56.79it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 56.89it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.03it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.12it/s]
 24%|██▍       | 239/1000 [00:05<00:13, 56.99it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.05it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.13it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.32it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.35it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.10it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.19it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.34it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.43it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.66it/s]
 30%|██▉       | 299/1000 [00:06<00:12, 57.66it/s]
 30%|███       | 305/1000 [00:06<00:12, 57.68it/s]
 31%|███       | 311/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 57.77it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 57.73it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.53it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.59it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 57.61it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 57.73it/s]
 35%|███▌      | 353/1000 [00:07<00:11, 57.52it/s]
 36%|███▌      | 359/1000 [00:07<00:11, 57.66it/s]
 36%|███▋      | 365/1000 [00:07<00:11, 57.57it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 57.72it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.58it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.36it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.28it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.35it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.50it/s]
 41%|████      | 407/1000 [00:07<00:10, 57.61it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 57.68it/s]
 42%|████▏     | 419/1000 [00:08<00:10, 57.74it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.68it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.84it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 57.86it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.66it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 57.70it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 57.74it/s]
 47%|████▋     | 467/1000 [00:09<00:09, 57.72it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 57.86it/s]
 48%|████▊     | 479/1000 [00:09<00:09, 57.77it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 57.74it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.71it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.62it/s]
 50%|█████     | 503/1000 [00:09<00:08, 57.74it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.62it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.65it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 57.70it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 57.78it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 57.68it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 57.78it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 57.82it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.73it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.71it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 57.80it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.69it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.62it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 57.77it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 57.71it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.64it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 57.75it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.65it/s]
 61%|██████    | 611/1000 [00:11<00:06, 57.71it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 57.78it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 57.79it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 57.84it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.71it/s]
 64%|██████▍   | 641/1000 [00:12<00:06, 57.59it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 57.79it/s]
 65%|██████▌   | 653/1000 [00:12<00:06, 57.72it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 57.82it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 57.80it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 57.78it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 57.81it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 57.78it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.82it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.71it/s]
 70%|███████   | 701/1000 [00:13<00:05, 57.80it/s]
 71%|███████   | 707/1000 [00:13<00:05, 57.78it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 57.75it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 57.78it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 57.73it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 57.71it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 57.78it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 57.74it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 57.81it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 57.95it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 57.77it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 57.83it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 57.83it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 57.86it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 57.87it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.66it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.66it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.68it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.66it/s]
 82%|████████▏ | 815/1000 [00:15<00:03, 57.66it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 57.73it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 57.72it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 57.89it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 57.97it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 57.82it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 57.84it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 57.77it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 57.81it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 57.75it/s]
 88%|████████▊ | 875/1000 [00:16<00:02, 57.79it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 57.81it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.64it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 57.74it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 57.75it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 57.72it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 57.70it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 57.76it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.68it/s]
 93%|█████████▎| 929/1000 [00:17<00:01, 57.81it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 57.88it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 57.86it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 57.83it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 57.87it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 57.83it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 57.88it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 57.96it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.00it/s]
 99%|█████████▉| 989/1000 [00:18<00:00, 57.93it/s]
100%|█████████▉| 995/1000 [00:18<00:00, 57.79it/s]
100%|██████████| 1000/1000 [00:18<00:00, 54.85it/s]
+2022-12-21,11:12:04 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:05<17:57, 46.35it/s]
  1%|          | 512/50176 [00:08<12:29, 66.24it/s]
  2%|▏         | 768/50176 [00:10<10:32, 78.15it/s]
  2%|▏         | 1024/50176 [00:13<09:44, 84.10it/s]
  3%|▎         | 1280/50176 [00:16<09:30, 85.75it/s]
  3%|▎         | 1536/50176 [00:19<09:42, 83.52it/s]
  4%|▎         | 1792/50176 [00:22<09:24, 85.76it/s]
  4%|▍         | 2048/50176 [00:24<08:52, 90.33it/s]
  5%|▍         | 2304/50176 [00:27<08:52, 89.86it/s]
  5%|▌         | 2560/50176 [00:30<08:25, 94.19it/s]
  6%|▌         | 2816/50176 [00:33<08:33, 92.19it/s]
  6%|▌         | 3072/50176 [00:35<08:25, 93.25it/s]
  7%|▋         | 3328/50176 [00:38<08:10, 95.41it/s]
  7%|▋         | 3584/50176 [00:40<07:59, 97.26it/s]
  8%|▊         | 3840/50176 [00:43<08:08, 94.85it/s]
  8%|▊         | 4096/50176 [00:46<08:24, 91.34it/s]
  9%|▊         | 4352/50176 [00:49<08:05, 94.31it/s]
  9%|▉         | 4608/50176 [00:51<07:59, 95.09it/s]
 10%|▉         | 4864/50176 [00:54<07:44, 97.46it/s]
 10%|█         | 5120/50176 [00:56<07:34, 99.05it/s]
 11%|█         | 5376/50176 [00:59<07:41, 97.11it/s]
 11%|█         | 5632/50176 [01:02<07:53, 94.15it/s]
 12%|█▏        | 5888/50176 [01:05<07:42, 95.73it/s]
 12%|█▏        | 6144/50176 [01:08<08:06, 90.58it/s]
 13%|█▎        | 6400/50176 [01:10<07:50, 93.09it/s]
 13%|█▎        | 6656/50176 [01:13<07:34, 95.83it/s]
 14%|█▍        | 6912/50176 [01:16<07:37, 94.66it/s]
 14%|█▍        | 7168/50176 [01:18<07:23, 96.92it/s]
 15%|█▍        | 7424/50176 [01:21<07:15, 98.14it/s]
 15%|█▌        | 7680/50176 [01:23<07:10, 98.81it/s]
 16%|█▌        | 7936/50176 [01:26<07:03, 99.68it/s]
 16%|█▋        | 8192/50176 [01:28<07:06, 98.35it/s]
 17%|█▋        | 8448/50176 [01:31<06:55, 100.53it/s]
 17%|█▋        | 8704/50176 [01:33<06:58, 99.20it/s] 
 18%|█▊        | 8960/50176 [01:36<07:02, 97.50it/s]
 18%|█▊        | 9216/50176 [01:39<06:50, 99.84it/s]
 19%|█▉        | 9472/50176 [01:41<06:45, 100.31it/s]
 19%|█▉        | 9728/50176 [01:44<06:51, 98.20it/s] 
 20%|█▉        | 9984/50176 [01:49<08:44, 76.70it/s]
 20%|██        | 10240/50176 [01:53<09:32, 69.79it/s]
 21%|██        | 10496/50176 [01:56<08:29, 77.81it/s]
 21%|██▏       | 10752/50176 [01:59<08:04, 81.35it/s]
 22%|██▏       | 11008/50176 [02:01<07:28, 87.32it/s]
 22%|██▏       | 11264/50176 [02:04<07:07, 91.05it/s]
 23%|██▎       | 11520/50176 [02:06<06:49, 94.38it/s]
 23%|██▎       | 11776/50176 [02:08<06:33, 97.65it/s]
 24%|██▍       | 12032/50176 [02:11<06:32, 97.22it/s]
 24%|██▍       | 12288/50176 [02:14<06:19, 99.73it/s]
 25%|██▌       | 12544/50176 [02:16<06:12, 101.12it/s]
 26%|██▌       | 12800/50176 [02:19<06:10, 100.89it/s]
 26%|██▌       | 13056/50176 [02:21<06:06, 101.23it/s]
 27%|██▋       | 13312/50176 [02:24<06:03, 101.52it/s]
 27%|██▋       | 13568/50176 [02:26<06:11, 98.41it/s] 
 28%|██▊       | 13824/50176 [02:29<06:25, 94.35it/s]
 28%|██▊       | 14080/50176 [02:32<06:22, 94.32it/s]
 29%|██▊       | 14336/50176 [02:35<06:23, 93.45it/s]
 29%|██▉       | 14592/50176 [02:37<06:08, 96.55it/s]
 30%|██▉       | 14848/50176 [02:41<06:30, 90.47it/s]
 30%|███       | 15104/50176 [02:43<06:17, 92.82it/s]
 31%|███       | 15360/50176 [02:46<06:09, 94.32it/s]
 31%|███       | 15616/50176 [02:48<05:54, 97.35it/s]
 32%|███▏      | 15872/50176 [02:53<07:28, 76.56it/s]
 32%|███▏      | 16128/50176 [02:56<06:49, 83.05it/s]
 33%|███▎      | 16384/50176 [02:58<06:21, 88.48it/s]
 33%|███▎      | 16640/50176 [03:01<06:23, 87.34it/s]
 34%|███▎      | 16896/50176 [03:04<06:05, 91.06it/s]
 34%|███▍      | 17152/50176 [03:06<05:50, 94.20it/s]
 35%|███▍      | 17408/50176 [03:09<05:45, 94.76it/s]
 35%|███▌      | 17664/50176 [03:11<05:30, 98.35it/s]
 36%|███▌      | 17920/50176 [03:14<05:24, 99.48it/s]
 36%|███▌      | 18176/50176 [03:16<05:22, 99.34it/s]
 37%|███▋      | 18432/50176 [03:19<05:15, 100.61it/s]
 37%|███▋      | 18688/50176 [03:22<05:26, 96.55it/s] 
 38%|███▊      | 18944/50176 [03:24<05:24, 96.24it/s]
 38%|███▊      | 19200/50176 [03:27<05:27, 94.67it/s]
 39%|███▉      | 19456/50176 [03:30<05:26, 94.13it/s]
 39%|███▉      | 19712/50176 [03:35<06:47, 74.82it/s]
 40%|███▉      | 19968/50176 [03:40<07:29, 67.19it/s]
 40%|████      | 20224/50176 [03:42<06:29, 76.96it/s]
 41%|████      | 20480/50176 [03:47<07:22, 67.04it/s]
 41%|████▏     | 20736/50176 [03:49<06:32, 75.04it/s]
 42%|████▏     | 20992/50176 [03:52<05:50, 83.35it/s]
 42%|████▏     | 21248/50176 [03:54<05:35, 86.22it/s]
 43%|████▎     | 21504/50176 [03:59<06:36, 72.39it/s]
 43%|████▎     | 21760/50176 [04:02<05:56, 79.69it/s]
 44%|████▍     | 22016/50176 [04:04<05:21, 87.71it/s]
 44%|████▍     | 22272/50176 [04:06<05:01, 92.50it/s]
 45%|████▍     | 22528/50176 [04:09<04:44, 97.34it/s]
 45%|████▌     | 22784/50176 [04:11<04:41, 97.29it/s]
 46%|████▌     | 23040/50176 [04:14<04:41, 96.42it/s]
 46%|████▋     | 23296/50176 [04:17<04:41, 95.54it/s]
 47%|████▋     | 23552/50176 [04:19<04:38, 95.56it/s]
 47%|████▋     | 23808/50176 [04:22<04:23, 100.16it/s]
 48%|████▊     | 24064/50176 [04:24<04:15, 102.00it/s]
 48%|████▊     | 24320/50176 [04:27<04:15, 101.33it/s]
 49%|████▉     | 24576/50176 [04:29<04:14, 100.62it/s]
 49%|████▉     | 24832/50176 [04:32<04:08, 101.90it/s]
 50%|█████     | 25088/50176 [04:34<04:01, 104.09it/s]
 51%|█████     | 25344/50176 [04:36<03:53, 106.23it/s]
 51%|█████     | 25600/50176 [04:39<03:55, 104.45it/s]
 52%|█████▏    | 25856/50176 [04:41<03:51, 105.12it/s]
 52%|█████▏    | 26112/50176 [04:44<03:46, 106.14it/s]
 53%|█████▎    | 26368/50176 [04:46<03:45, 105.55it/s]
 53%|█████▎    | 26624/50176 [04:48<03:43, 105.51it/s]
 54%|█████▎    | 26880/50176 [04:51<03:41, 104.96it/s]
 54%|█████▍    | 27136/50176 [04:53<03:43, 103.19it/s]
 55%|█████▍    | 27392/50176 [04:56<03:33, 106.87it/s]
 55%|█████▌    | 27648/50176 [04:58<03:27, 108.76it/s]
 56%|█████▌    | 27904/50176 [05:01<03:35, 103.21it/s]
 56%|█████▌    | 28160/50176 [05:03<03:37, 101.03it/s]
 57%|█████▋    | 28416/50176 [05:06<03:34, 101.23it/s]
 57%|█████▋    | 28672/50176 [05:08<03:31, 101.53it/s]
 58%|█████▊    | 28928/50176 [05:11<03:22, 104.90it/s]
 58%|█████▊    | 29184/50176 [05:13<03:22, 103.56it/s]
 59%|█████▊    | 29440/50176 [05:16<03:32, 97.58it/s] 
 59%|█████▉    | 29696/50176 [05:19<03:35, 95.00it/s]
 60%|█████▉    | 29952/50176 [05:21<03:27, 97.28it/s]
 60%|██████    | 30208/50176 [05:24<03:21, 99.20it/s]
 61%|██████    | 30464/50176 [05:26<03:08, 104.39it/s]
 61%|██████    | 30720/50176 [05:29<03:07, 103.51it/s]
 62%|██████▏   | 30976/50176 [05:31<03:01, 106.03it/s]
 62%|██████▏   | 31232/50176 [05:33<02:59, 105.69it/s]
 63%|██████▎   | 31488/50176 [05:36<02:53, 107.85it/s]
 63%|██████▎   | 31744/50176 [05:38<02:49, 108.49it/s]
 64%|██████▍   | 32000/50176 [05:40<02:47, 108.31it/s]
 64%|██████▍   | 32256/50176 [05:43<02:48, 106.44it/s]
 65%|██████▍   | 32512/50176 [05:46<02:53, 102.07it/s]
 65%|██████▌   | 32768/50176 [05:48<02:44, 105.55it/s]
 66%|██████▌   | 33024/50176 [05:50<02:42, 105.70it/s]
 66%|██████▋   | 33280/50176 [05:52<02:36, 108.16it/s]
 67%|██████▋   | 33536/50176 [05:55<02:37, 105.49it/s]
 67%|██████▋   | 33792/50176 [05:57<02:34, 105.89it/s]
 68%|██████▊   | 34048/50176 [06:00<02:29, 108.18it/s]
 68%|██████▊   | 34304/50176 [06:02<02:24, 109.57it/s]
 69%|██████▉   | 34560/50176 [06:04<02:26, 106.50it/s]
 69%|██████▉   | 34816/50176 [06:07<02:25, 105.45it/s]
 70%|██████▉   | 35072/50176 [06:09<02:25, 103.71it/s]
 70%|███████   | 35328/50176 [06:12<02:19, 106.76it/s]
 71%|███████   | 35584/50176 [06:14<02:16, 106.52it/s]
 71%|███████▏  | 35840/50176 [06:17<02:17, 103.90it/s]
 72%|███████▏  | 36096/50176 [06:19<02:12, 106.50it/s]
 72%|███████▏  | 36352/50176 [06:22<02:11, 105.35it/s]
 73%|███████▎  | 36608/50176 [06:24<02:10, 103.85it/s]
 73%|███████▎  | 36864/50176 [06:27<02:12, 100.62it/s]
 74%|███████▍  | 37120/50176 [06:29<02:08, 101.72it/s]
 74%|███████▍  | 37376/50176 [06:31<02:00, 105.85it/s]
 75%|███████▌  | 37632/50176 [06:34<01:57, 106.54it/s]
 76%|███████▌  | 37888/50176 [06:36<01:55, 106.68it/s]
 76%|███████▌  | 38144/50176 [06:39<01:52, 107.11it/s]
 77%|███████▋  | 38400/50176 [06:41<01:51, 105.47it/s]
 77%|███████▋  | 38656/50176 [06:44<01:50, 104.46it/s]
 78%|███████▊  | 38912/50176 [06:46<01:47, 104.33it/s]
 78%|███████▊  | 39168/50176 [06:49<01:47, 102.65it/s]
 79%|███████▊  | 39424/50176 [06:51<01:42, 104.65it/s]
 79%|███████▉  | 39680/50176 [06:53<01:40, 104.59it/s]
 80%|███████▉  | 39936/50176 [06:56<01:35, 107.29it/s]
 80%|████████  | 40192/50176 [06:58<01:33, 106.51it/s]
 81%|████████  | 40448/50176 [07:01<01:34, 102.57it/s]
 81%|████████  | 40704/50176 [07:04<01:36, 98.17it/s] 
 82%|████████▏ | 40960/50176 [07:06<01:29, 102.83it/s]
 82%|████████▏ | 41216/50176 [07:08<01:25, 104.37it/s]
 83%|████████▎ | 41472/50176 [07:11<01:22, 105.07it/s]
 83%|████████▎ | 41728/50176 [07:13<01:19, 106.24it/s]
 84%|████████▎ | 41984/50176 [07:15<01:16, 106.44it/s]
 84%|████████▍ | 42240/50176 [07:18<01:13, 108.25it/s]
 85%|████████▍ | 42496/50176 [07:20<01:10, 109.42it/s]
 85%|████████▌ | 42752/50176 [07:25<01:29, 82.59it/s] 
 86%|████████▌ | 43008/50176 [07:27<01:21, 88.36it/s]
 86%|████████▌ | 43264/50176 [07:30<01:14, 93.28it/s]
 87%|████████▋ | 43520/50176 [07:37<01:45, 63.23it/s]
 87%|████████▋ | 43776/50176 [07:42<01:46, 59.85it/s]
 88%|████████▊ | 44032/50176 [07:44<01:28, 69.55it/s]
 88%|████████▊ | 44288/50176 [07:46<01:16, 77.39it/s]
 89%|████████▉ | 44544/50176 [07:49<01:08, 82.40it/s]
 89%|████████▉ | 44800/50176 [07:51<01:01, 87.93it/s]
 90%|████████▉ | 45056/50176 [07:54<00:54, 93.98it/s]
 90%|█████████ | 45312/50176 [07:56<00:50, 95.40it/s]
 91%|█████████ | 45568/50176 [07:58<00:45, 101.30it/s]
 91%|█████████▏| 45824/50176 [08:01<00:43, 101.05it/s]
 92%|█████████▏| 46080/50176 [08:04<00:40, 100.20it/s]
 92%|█████████▏| 46336/50176 [08:06<00:38, 100.63it/s]
 93%|█████████▎| 46592/50176 [08:08<00:34, 103.89it/s]
 93%|█████████▎| 46848/50176 [08:11<00:32, 103.75it/s]
 94%|█████████▍| 47104/50176 [08:14<00:30, 99.20it/s] 
 94%|█████████▍| 47360/50176 [08:16<00:28, 99.06it/s]
 95%|█████████▍| 47616/50176 [08:19<00:25, 98.80it/s]
 95%|█████████▌| 47872/50176 [08:21<00:22, 101.20it/s]
 96%|█████████▌| 48128/50176 [08:24<00:19, 103.58it/s]
 96%|█████████▋| 48384/50176 [08:26<00:17, 103.38it/s]
 97%|█████████▋| 48640/50176 [08:28<00:14, 104.19it/s]
 97%|█████████▋| 48896/50176 [08:31<00:12, 104.29it/s]
 98%|█████████▊| 49152/50176 [08:34<00:09, 102.61it/s]
 98%|█████████▊| 49408/50176 [08:36<00:07, 102.14it/s]
 99%|█████████▉| 49664/50176 [08:39<00:05, 101.22it/s]
 99%|█████████▉| 49920/50176 [08:42<00:02, 95.69it/s] 
100%|██████████| 50176/50176 [08:42<00:00, 125.13it/s]
100%|██████████| 50176/50176 [08:42<00:00, 95.98it/s] 
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6589	imagenet-zeroshot-val-top5: 0.8959
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_55.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_55.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6585cd98d34b5cebebdc45735f40f473ec70091c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_55.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:45943e2558adce0aa2282baea7a2523301b96a191558f1e73d87df5b5c8f356d
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_55.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_55.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..79e518815bd0af5fadf8842fff2d0bbc0f5d91b9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_55.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_55.pt).
+2022-12-21,11:11:47 | INFO | Model:
+2022-12-21,11:11:47 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:47 | INFO | Params:
+2022-12-21,11:11:47 | INFO |   batch_size: 256
+2022-12-21,11:11:47 | INFO |   beta1: 0.9
+2022-12-21,11:11:47 | INFO |   beta2: 0.98
+2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_55.pt/checkpoints
+2022-12-21,11:11:47 | INFO |   copy_codebase: False
+2022-12-21,11:11:47 | INFO |   csv_caption_key: title
+2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:47 | INFO |   csv_separator: 	
+2022-12-21,11:11:47 | INFO |   dataset_resampled: False
+2022-12-21,11:11:47 | INFO |   dataset_type: auto
+2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:47 | INFO |   debug: False
+2022-12-21,11:11:47 | INFO |   device: cuda:0
+2022-12-21,11:11:47 | INFO |   dist_backend: nccl
+2022-12-21,11:11:47 | INFO |   dist_url: env://
+2022-12-21,11:11:47 | INFO |   distributed: False
+2022-12-21,11:11:47 | INFO |   epochs: 32
+2022-12-21,11:11:47 | INFO |   eps: 1e-06
+2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:47 | INFO |   gather_with_grad: False
+2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:47 | INFO |   horovod: False
+2022-12-21,11:11:47 | INFO |   image_mean: None
+2022-12-21,11:11:47 | INFO |   image_std: None
+2022-12-21,11:11:47 | INFO |   imagenet_v2: None
+2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:47 | INFO |   local_loss: False
+2022-12-21,11:11:47 | INFO |   local_rank: 0
+2022-12-21,11:11:47 | INFO |   lock_image: False
+2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:47 | INFO |   log_level: 20
+2022-12-21,11:11:47 | INFO |   log_local: False
+2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_55.pt/out.log
+2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:47 | INFO |   lr: 0.0005
+2022-12-21,11:11:47 | INFO |   model: ViT-B-16
+2022-12-21,11:11:47 | INFO |   name: epoch_55.pt
+2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_55.pt
+2022-12-21,11:11:47 | INFO |   pretrained_image: False
+2022-12-21,11:11:47 | INFO |   rank: 0
+2022-12-21,11:11:47 | INFO |   report_to: 
+2022-12-21,11:11:47 | INFO |   resume: None
+2022-12-21,11:11:47 | INFO |   save_frequency: 1
+2022-12-21,11:11:47 | INFO |   save_most_recent: False
+2022-12-21,11:11:47 | INFO |   seed: 0
+2022-12-21,11:11:47 | INFO |   skip_scheduler: False
+2022-12-21,11:11:47 | INFO |   tensorboard: False
+2022-12-21,11:11:47 | INFO |   tensorboard_path: 
+2022-12-21,11:11:47 | INFO |   torchscript: False
+2022-12-21,11:11:47 | INFO |   trace: False
+2022-12-21,11:11:47 | INFO |   train_data: None
+2022-12-21,11:11:47 | INFO |   train_num_samples: None
+2022-12-21,11:11:47 | INFO |   use_bn_sync: False
+2022-12-21,11:11:47 | INFO |   val_data: None
+2022-12-21,11:11:47 | INFO |   val_frequency: 1
+2022-12-21,11:11:47 | INFO |   val_num_samples: None
+2022-12-21,11:11:47 | INFO |   wandb: False
+2022-12-21,11:11:47 | INFO |   wandb_notes: 
+2022-12-21,11:11:47 | INFO |   warmup: 10000
+2022-12-21,11:11:47 | INFO |   wd: 0.2
+2022-12-21,11:11:47 | INFO |   workers: 1
+2022-12-21,11:11:47 | INFO |   world_size: 1
+2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:47 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:56,  1.52it/s]
  1%|          | 6/1000 [00:00<01:37, 10.16it/s]
  1%|          | 12/1000 [00:00<00:50, 19.57it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.91it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.10it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.95it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.36it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.68it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.23it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.03it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.40it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.43it/s]
  7%|▋         | 72/1000 [00:01<00:16, 56.11it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.56it/s]
  8%|▊         | 84/1000 [00:02<00:16, 57.08it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.36it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.75it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.94it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.19it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.22it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.20it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.10it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.13it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.11it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.18it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.28it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.25it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.14it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 58.01it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 58.04it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.97it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.90it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.88it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.79it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.61it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.42it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.55it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.50it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.72it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.73it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.65it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.74it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.83it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.93it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.99it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.82it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.87it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 58.07it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.17it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.37it/s]
 30%|███       | 300/1000 [00:05<00:11, 58.34it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.31it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.46it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.43it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.29it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.26it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.38it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.50it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.54it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.28it/s]
 36%|███▌      | 360/1000 [00:06<00:10, 58.27it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.33it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.41it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.31it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.97it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 58.04it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 58.15it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.28it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.38it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.38it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.39it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.41it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.52it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.50it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.16it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.26it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.34it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.34it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.30it/s]
 47%|████▋     | 474/1000 [00:08<00:08, 58.46it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.39it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.37it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.37it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.22it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.20it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.22it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.28it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.38it/s]
 53%|█████▎    | 534/1000 [00:09<00:07, 58.33it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.44it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.41it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.33it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.36it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.42it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.30it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.25it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.38it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.35it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.27it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.38it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.30it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.29it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.42it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.40it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.49it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.28it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.24it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.41it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.34it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.45it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.50it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.44it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.50it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.26it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.37it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.30it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.42it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.38it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.38it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.42it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.38it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.40it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.42it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.39it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.49it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.56it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.39it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.51it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.43it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.48it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.51it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.30it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.30it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.36it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.34it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.32it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.35it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.43it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.55it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.60it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.51it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.55it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.48it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.49it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.44it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.50it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.50it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.38it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.48it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.48it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.45it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.48it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.46it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.44it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.53it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.56it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.40it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.45it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.49it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.45it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.54it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.62it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.44it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.48it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.49it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.35it/s]
100%|██████████| 1000/1000 [00:17<00:00, 56.05it/s]
+2022-12-21,11:12:05 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:04<14:23, 57.78it/s]
  1%|          | 512/50176 [00:07<10:58, 75.37it/s]
  2%|▏         | 768/50176 [00:09<09:43, 84.67it/s]
  2%|▏         | 1024/50176 [00:12<09:14, 88.63it/s]
  3%|▎         | 1280/50176 [00:15<09:12, 88.48it/s]
  3%|▎         | 1536/50176 [00:18<09:30, 85.30it/s]
  4%|▎         | 1792/50176 [00:21<09:15, 87.16it/s]
  4%|▍         | 2048/50176 [00:23<08:48, 91.03it/s]
  5%|▍         | 2304/50176 [00:26<08:48, 90.65it/s]
  5%|▌         | 2560/50176 [00:29<08:22, 94.81it/s]
  6%|▌         | 2816/50176 [00:32<08:32, 92.49it/s]
  6%|▌         | 3072/50176 [00:34<08:23, 93.52it/s]
  7%|▋         | 3328/50176 [00:37<08:10, 95.50it/s]
  7%|▋         | 3584/50176 [00:39<07:58, 97.43it/s]
  8%|▊         | 3840/50176 [00:42<08:08, 94.91it/s]
  8%|▊         | 4096/50176 [00:45<08:24, 91.38it/s]
  9%|▊         | 4352/50176 [00:48<08:06, 94.29it/s]
  9%|▉         | 4608/50176 [00:50<07:58, 95.16it/s]
 10%|▉         | 4864/50176 [00:53<07:44, 97.51it/s]
 10%|█         | 5120/50176 [00:55<07:34, 99.05it/s]
 11%|█         | 5376/50176 [00:58<07:41, 97.15it/s]
 11%|█         | 5632/50176 [01:01<07:52, 94.18it/s]
 12%|█▏        | 5888/50176 [01:03<07:42, 95.80it/s]
 12%|█▏        | 6144/50176 [01:07<08:05, 90.63it/s]
 13%|█▎        | 6400/50176 [01:09<07:50, 92.95it/s]
 13%|█▎        | 6656/50176 [01:12<07:34, 95.80it/s]
 14%|█▍        | 6912/50176 [01:15<07:36, 94.71it/s]
 14%|█▍        | 7168/50176 [01:17<07:23, 96.93it/s]
 15%|█▍        | 7424/50176 [01:20<07:16, 97.99it/s]
 15%|█▌        | 7680/50176 [01:22<07:09, 98.90it/s]
 16%|█▌        | 7936/50176 [01:25<07:04, 99.62it/s]
 16%|█▋        | 8192/50176 [01:27<07:06, 98.42it/s]
 17%|█▋        | 8448/50176 [01:30<06:55, 100.52it/s]
 17%|█▋        | 8704/50176 [01:32<06:57, 99.32it/s] 
 18%|█▊        | 8960/50176 [01:35<07:03, 97.34it/s]
 18%|█▊        | 9216/50176 [01:38<06:50, 99.90it/s]
 19%|█▉        | 9472/50176 [01:40<06:46, 100.21it/s]
 19%|█▉        | 9728/50176 [01:43<06:51, 98.31it/s] 
 20%|█▉        | 9984/50176 [01:48<08:43, 76.78it/s]
 20%|██        | 10240/50176 [01:52<09:32, 69.70it/s]
 21%|██        | 10496/50176 [01:55<08:29, 77.81it/s]
 21%|██▏       | 10752/50176 [01:57<08:04, 81.37it/s]
 22%|██▏       | 11008/50176 [02:00<07:28, 87.30it/s]
 22%|██▏       | 11264/50176 [02:02<07:06, 91.19it/s]
 23%|██▎       | 11520/50176 [02:05<06:50, 94.21it/s]
 23%|██▎       | 11776/50176 [02:07<06:33, 97.53it/s]
 24%|██▍       | 12032/50176 [02:10<06:32, 97.22it/s]
 24%|██▍       | 12288/50176 [02:12<06:19, 99.95it/s]
 25%|██▌       | 12544/50176 [02:15<06:11, 101.16it/s]
 26%|██▌       | 12800/50176 [02:17<06:10, 100.77it/s]
 26%|██▌       | 13056/50176 [02:20<06:06, 101.31it/s]
 27%|██▋       | 13312/50176 [02:22<06:02, 101.58it/s]
 27%|██▋       | 13568/50176 [02:25<06:11, 98.44it/s] 
 28%|██▊       | 13824/50176 [02:28<06:25, 94.29it/s]
 28%|██▊       | 14080/50176 [02:31<06:21, 94.53it/s]
 29%|██▊       | 14336/50176 [02:34<06:23, 93.44it/s]
 29%|██▉       | 14592/50176 [02:36<06:08, 96.61it/s]
 30%|██▉       | 14848/50176 [02:39<06:31, 90.33it/s]
 30%|███       | 15104/50176 [02:42<06:17, 92.83it/s]
 31%|███       | 15360/50176 [02:45<06:08, 94.39it/s]
 31%|███       | 15616/50176 [02:47<05:55, 97.29it/s]
 32%|███▏      | 15872/50176 [02:52<07:27, 76.60it/s]
 32%|███▏      | 16128/50176 [02:54<06:49, 83.15it/s]
 33%|███▎      | 16384/50176 [02:57<06:22, 88.25it/s]
 33%|███▎      | 16640/50176 [03:00<06:24, 87.31it/s]
 34%|███▎      | 16896/50176 [03:03<06:05, 90.96it/s]
 34%|███▍      | 17152/50176 [03:05<05:50, 94.14it/s]
 35%|███▍      | 17408/50176 [03:08<05:45, 94.73it/s]
 35%|███▌      | 17664/50176 [03:10<05:30, 98.25it/s]
 36%|███▌      | 17920/50176 [03:13<05:24, 99.53it/s]
 36%|███▌      | 18176/50176 [03:15<05:22, 99.25it/s]
 37%|███▋      | 18432/50176 [03:18<05:15, 100.75it/s]
 37%|███▋      | 18688/50176 [03:20<05:25, 96.73it/s] 
 38%|███▊      | 18944/50176 [03:23<05:24, 96.15it/s]
 38%|███▊      | 19200/50176 [03:26<05:27, 94.60it/s]
 39%|███▉      | 19456/50176 [03:29<05:26, 94.18it/s]
 39%|███▉      | 19712/50176 [03:34<06:46, 74.90it/s]
 40%|███▉      | 19968/50176 [03:39<07:29, 67.13it/s]
 40%|████      | 20224/50176 [03:41<06:29, 76.92it/s]
 41%|████      | 20480/50176 [03:46<07:22, 67.07it/s]
 41%|████▏     | 20736/50176 [03:48<06:31, 75.13it/s]
 42%|████▏     | 20992/50176 [03:50<05:50, 83.22it/s]
 42%|████▏     | 21248/50176 [03:53<05:35, 86.30it/s]
 43%|████▎     | 21504/50176 [03:58<06:36, 72.37it/s]
 43%|████▎     | 21760/50176 [04:00<05:56, 79.76it/s]
 44%|████▍     | 22016/50176 [04:03<05:21, 87.54it/s]
 44%|████▍     | 22272/50176 [04:05<05:01, 92.58it/s]
 45%|████▍     | 22528/50176 [04:07<04:44, 97.25it/s]
 45%|████▌     | 22784/50176 [04:10<04:41, 97.24it/s]
 46%|████▌     | 23040/50176 [04:13<04:41, 96.33it/s]
 46%|████▋     | 23296/50176 [04:16<04:41, 95.56it/s]
 47%|████▋     | 23552/50176 [04:18<04:38, 95.56it/s]
 47%|████▋     | 23808/50176 [04:20<04:23, 99.96it/s]
 48%|████▊     | 24064/50176 [04:23<04:16, 101.99it/s]
 48%|████▊     | 24320/50176 [04:25<04:14, 101.44it/s]
 49%|████▉     | 24576/50176 [04:28<04:14, 100.67it/s]
 49%|████▉     | 24832/50176 [04:30<04:08, 101.84it/s]
 50%|█████     | 25088/50176 [04:33<04:01, 104.03it/s]
 51%|█████     | 25344/50176 [04:35<03:54, 106.09it/s]
 51%|█████     | 25600/50176 [04:38<03:55, 104.30it/s]
 52%|█████▏    | 25856/50176 [04:40<03:51, 105.11it/s]
 52%|█████▏    | 26112/50176 [04:42<03:46, 106.25it/s]
 53%|█████▎    | 26368/50176 [04:45<03:45, 105.65it/s]
 53%|█████▎    | 26624/50176 [04:47<03:43, 105.40it/s]
 54%|█████▎    | 26880/50176 [04:50<03:42, 104.82it/s]
 54%|█████▍    | 27136/50176 [04:52<03:42, 103.33it/s]
 55%|█████▍    | 27392/50176 [04:55<03:32, 107.07it/s]
 55%|█████▌    | 27648/50176 [04:57<03:27, 108.67it/s]
 56%|█████▌    | 27904/50176 [05:00<03:35, 103.28it/s]
 56%|█████▌    | 28160/50176 [05:02<03:38, 100.89it/s]
 57%|█████▋    | 28416/50176 [05:05<03:34, 101.31it/s]
 57%|█████▋    | 28672/50176 [05:07<03:31, 101.63it/s]
 58%|█████▊    | 28928/50176 [05:09<03:22, 104.74it/s]
 58%|█████▊    | 29184/50176 [05:12<03:22, 103.71it/s]
 59%|█████▊    | 29440/50176 [05:15<03:32, 97.61it/s] 
 59%|█████▉    | 29696/50176 [05:18<03:35, 95.06it/s]
 60%|█████▉    | 29952/50176 [05:20<03:27, 97.25it/s]
 60%|██████    | 30208/50176 [05:23<03:21, 99.14it/s]
 61%|██████    | 30464/50176 [05:25<03:08, 104.39it/s]
 61%|██████    | 30720/50176 [05:27<03:08, 103.37it/s]
 62%|██████▏   | 30976/50176 [05:30<03:00, 106.09it/s]
 62%|██████▏   | 31232/50176 [05:32<02:59, 105.77it/s]
 63%|██████▎   | 31488/50176 [05:34<02:53, 107.68it/s]
 63%|██████▎   | 31744/50176 [05:37<02:49, 108.47it/s]
 64%|██████▍   | 32000/50176 [05:39<02:48, 108.18it/s]
 64%|██████▍   | 32256/50176 [05:42<02:48, 106.44it/s]
 65%|██████▍   | 32512/50176 [05:44<02:53, 102.03it/s]
 65%|██████▌   | 32768/50176 [05:47<02:44, 105.58it/s]
 66%|██████▌   | 33024/50176 [05:49<02:42, 105.63it/s]
 66%|██████▋   | 33280/50176 [05:51<02:36, 107.92it/s]
 67%|██████▋   | 33536/50176 [05:54<02:37, 105.44it/s]
 67%|██████▋   | 33792/50176 [05:56<02:34, 105.98it/s]
 68%|██████▊   | 34048/50176 [05:59<02:28, 108.33it/s]
 68%|██████▊   | 34304/50176 [06:01<02:24, 109.54it/s]
 69%|██████▉   | 34560/50176 [06:03<02:26, 106.68it/s]
 69%|██████▉   | 34816/50176 [06:06<02:25, 105.26it/s]
 70%|██████▉   | 35072/50176 [06:08<02:25, 103.77it/s]
 70%|███████   | 35328/50176 [06:11<02:19, 106.61it/s]
 71%|███████   | 35584/50176 [06:13<02:16, 106.56it/s]
 71%|███████▏  | 35840/50176 [06:16<02:17, 103.94it/s]
 72%|███████▏  | 36096/50176 [06:18<02:12, 106.47it/s]
 72%|███████▏  | 36352/50176 [06:20<02:11, 105.42it/s]
 73%|███████▎  | 36608/50176 [06:23<02:10, 103.72it/s]
 73%|███████▎  | 36864/50176 [06:26<02:12, 100.76it/s]
 74%|███████▍  | 37120/50176 [06:28<02:08, 101.78it/s]
 74%|███████▍  | 37376/50176 [06:30<02:00, 105.98it/s]
 75%|███████▌  | 37632/50176 [06:33<01:57, 106.45it/s]
 76%|███████▌  | 37888/50176 [06:35<01:55, 106.53it/s]
 76%|███████▌  | 38144/50176 [06:37<01:52, 107.11it/s]
 77%|███████▋  | 38400/50176 [06:40<01:51, 105.53it/s]
 77%|███████▋  | 38656/50176 [06:42<01:50, 104.43it/s]
 78%|███████▊  | 38912/50176 [06:45<01:47, 104.41it/s]
 78%|███████▊  | 39168/50176 [06:48<01:47, 102.78it/s]
 79%|███████▊  | 39424/50176 [06:50<01:42, 104.69it/s]
 79%|███████▉  | 39680/50176 [06:52<01:40, 104.67it/s]
 80%|███████▉  | 39936/50176 [06:55<01:35, 107.29it/s]
 80%|████████  | 40192/50176 [06:57<01:33, 106.51it/s]
 81%|████████  | 40448/50176 [07:00<01:35, 102.36it/s]
 81%|████████  | 40704/50176 [07:03<01:36, 98.23it/s] 
 82%|████████▏ | 40960/50176 [07:05<01:29, 102.80it/s]
 82%|████████▏ | 41216/50176 [07:07<01:25, 104.33it/s]
 83%|████████▎ | 41472/50176 [07:10<01:22, 105.04it/s]
 83%|████████▎ | 41728/50176 [07:12<01:19, 106.34it/s]
 84%|████████▎ | 41984/50176 [07:14<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:17<01:13, 108.09it/s]
 85%|████████▍ | 42496/50176 [07:19<01:10, 109.37it/s]
 85%|████████▌ | 42752/50176 [07:24<01:30, 82.47it/s] 
 86%|████████▌ | 43008/50176 [07:26<01:21, 88.32it/s]
 86%|████████▌ | 43264/50176 [07:29<01:14, 93.26it/s]
 87%|████████▋ | 43520/50176 [07:36<01:45, 63.25it/s]
 87%|████████▋ | 43776/50176 [07:40<01:46, 59.85it/s]
 88%|████████▊ | 44032/50176 [07:43<01:28, 69.56it/s]
 88%|████████▊ | 44288/50176 [07:45<01:16, 77.46it/s]
 89%|████████▉ | 44544/50176 [07:48<01:08, 82.54it/s]
 89%|████████▉ | 44800/50176 [07:50<01:01, 87.97it/s]
 90%|████████▉ | 45056/50176 [07:53<00:54, 93.92it/s]
 90%|█████████ | 45312/50176 [07:55<00:50, 95.40it/s]
 91%|█████████ | 45568/50176 [07:57<00:45, 101.44it/s]
 91%|█████████▏| 45824/50176 [08:00<00:43, 101.02it/s]
 92%|█████████▏| 46080/50176 [08:02<00:40, 100.05it/s]
 92%|█████████▏| 46336/50176 [08:05<00:38, 100.75it/s]
 93%|█████████▎| 46592/50176 [08:07<00:34, 103.93it/s]
 93%|█████████▎| 46848/50176 [08:10<00:32, 103.48it/s]
 94%|█████████▍| 47104/50176 [08:13<00:31, 99.07it/s] 
 94%|█████████▍| 47360/50176 [08:15<00:28, 99.04it/s]
 95%|█████████▍| 47616/50176 [08:18<00:25, 98.76it/s]
 95%|█████████▌| 47872/50176 [08:20<00:22, 101.25it/s]
 96%|█████████▌| 48128/50176 [08:22<00:19, 103.51it/s]
 96%|█████████▋| 48384/50176 [08:25<00:17, 103.35it/s]
 97%|█████████▋| 48640/50176 [08:27<00:14, 104.17it/s]
 97%|█████████▋| 48896/50176 [08:30<00:12, 104.12it/s]
 98%|█████████▊| 49152/50176 [08:32<00:09, 102.69it/s]
 98%|█████████▊| 49408/50176 [08:35<00:07, 102.16it/s]
 99%|█████████▉| 49664/50176 [08:38<00:05, 101.35it/s]
 99%|█████████▉| 49920/50176 [08:41<00:02, 95.70it/s] 
100%|██████████| 50176/50176 [08:41<00:00, 124.94it/s]
100%|██████████| 50176/50176 [08:41<00:00, 96.18it/s] 
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6622	imagenet-zeroshot-val-top5: 0.8979
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_56.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_56.pt
new file mode 100644
index 0000000000000000000000000000000000000000..37b809b850da733bb4e1abf24b7ee00ac725ef41
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_56.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa76c09a72717f3eddf1dc9a349a94aa86a2abe402ae0bb3a57bc99985436bb5
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_56.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_56.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e9acc3a79b8d48fbd92f81631ae45c17e3fa4550
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_56.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_56.pt).
+2022-12-21,11:11:47 | INFO | Model:
+2022-12-21,11:11:47 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:47 | INFO | Params:
+2022-12-21,11:11:47 | INFO |   batch_size: 256
+2022-12-21,11:11:47 | INFO |   beta1: 0.9
+2022-12-21,11:11:47 | INFO |   beta2: 0.98
+2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_56.pt/checkpoints
+2022-12-21,11:11:47 | INFO |   copy_codebase: False
+2022-12-21,11:11:47 | INFO |   csv_caption_key: title
+2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:47 | INFO |   csv_separator: 	
+2022-12-21,11:11:47 | INFO |   dataset_resampled: False
+2022-12-21,11:11:47 | INFO |   dataset_type: auto
+2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:47 | INFO |   debug: False
+2022-12-21,11:11:47 | INFO |   device: cuda:0
+2022-12-21,11:11:47 | INFO |   dist_backend: nccl
+2022-12-21,11:11:47 | INFO |   dist_url: env://
+2022-12-21,11:11:47 | INFO |   distributed: False
+2022-12-21,11:11:47 | INFO |   epochs: 32
+2022-12-21,11:11:47 | INFO |   eps: 1e-06
+2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:47 | INFO |   gather_with_grad: False
+2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:47 | INFO |   horovod: False
+2022-12-21,11:11:47 | INFO |   image_mean: None
+2022-12-21,11:11:47 | INFO |   image_std: None
+2022-12-21,11:11:47 | INFO |   imagenet_v2: None
+2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:47 | INFO |   local_loss: False
+2022-12-21,11:11:47 | INFO |   local_rank: 0
+2022-12-21,11:11:47 | INFO |   lock_image: False
+2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:47 | INFO |   log_level: 20
+2022-12-21,11:11:47 | INFO |   log_local: False
+2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_56.pt/out.log
+2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:47 | INFO |   lr: 0.0005
+2022-12-21,11:11:47 | INFO |   model: ViT-B-16
+2022-12-21,11:11:47 | INFO |   name: epoch_56.pt
+2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_56.pt
+2022-12-21,11:11:47 | INFO |   pretrained_image: False
+2022-12-21,11:11:47 | INFO |   rank: 0
+2022-12-21,11:11:47 | INFO |   report_to: 
+2022-12-21,11:11:47 | INFO |   resume: None
+2022-12-21,11:11:47 | INFO |   save_frequency: 1
+2022-12-21,11:11:47 | INFO |   save_most_recent: False
+2022-12-21,11:11:47 | INFO |   seed: 0
+2022-12-21,11:11:47 | INFO |   skip_scheduler: False
+2022-12-21,11:11:47 | INFO |   tensorboard: False
+2022-12-21,11:11:47 | INFO |   tensorboard_path: 
+2022-12-21,11:11:47 | INFO |   torchscript: False
+2022-12-21,11:11:47 | INFO |   trace: False
+2022-12-21,11:11:47 | INFO |   train_data: None
+2022-12-21,11:11:47 | INFO |   train_num_samples: None
+2022-12-21,11:11:47 | INFO |   use_bn_sync: False
+2022-12-21,11:11:47 | INFO |   val_data: None
+2022-12-21,11:11:47 | INFO |   val_frequency: 1
+2022-12-21,11:11:47 | INFO |   val_num_samples: None
+2022-12-21,11:11:47 | INFO |   wandb: False
+2022-12-21,11:11:47 | INFO |   wandb_notes: 
+2022-12-21,11:11:47 | INFO |   warmup: 10000
+2022-12-21,11:11:47 | INFO |   wd: 0.2
+2022-12-21,11:11:47 | INFO |   workers: 1
+2022-12-21,11:11:47 | INFO |   world_size: 1
+2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:47 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:02,  1.28it/s]
  1%|          | 6/1000 [00:00<01:53,  8.77it/s]
  1%|          | 12/1000 [00:01<00:56, 17.44it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.51it/s]
  2%|▏         | 24/1000 [00:01<00:29, 32.65it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.30it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.18it/s]
  4%|▍         | 42/1000 [00:01<00:20, 46.98it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.89it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.67it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.10it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.42it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.34it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.95it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.60it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.99it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.44it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.66it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.95it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.09it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.07it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.83it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.96it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.02it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.17it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.07it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.93it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.92it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.87it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.81it/s]
 19%|█▉        | 192/1000 [00:04<00:13, 57.79it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.72it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.53it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.56it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.53it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.73it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.71it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.61it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.68it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.78it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.89it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.93it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.74it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.77it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.95it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.04it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.26it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.23it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.20it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.34it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.33it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.24it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.05it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.12it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.22it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.31it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.07it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.99it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.06it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.08it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.09it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.77it/s]
 39%|███▉      | 390/1000 [00:07<00:12, 50.60it/s]
 40%|███▉      | 396/1000 [00:07<00:11, 52.67it/s]
 40%|████      | 402/1000 [00:07<00:11, 54.28it/s]
 41%|████      | 408/1000 [00:07<00:10, 55.43it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 56.21it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 56.79it/s]
 43%|████▎     | 426/1000 [00:08<00:10, 57.21it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.62it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.81it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.73it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.89it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.00it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.04it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.20it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.14it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.15it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.06it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.10it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.03it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.10it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.25it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.19it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.30it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.30it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.20it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.20it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.25it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.14it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.08it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.21it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.18it/s]
 59%|█████▉    | 594/1000 [00:11<00:06, 58.09it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.20it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.13it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.12it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.25it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.25it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.32it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.15it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.11it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.25it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.20it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.27it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.31it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.24it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.29it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.29it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.16it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.25it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.23it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.22it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.25it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.20it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.22it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.19it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.19it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.28it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.36it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.20it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.31it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.24it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.08it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.14it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.97it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.99it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.07it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.06it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.06it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 58.11it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.22it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.35it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.36it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.27it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.34it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.30it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.30it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.25it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.29it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.28it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.14it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.25it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.26it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.23it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.27it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.23it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.29it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.34it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.38it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.35it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.43it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.50it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.52it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.49it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.19it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 58.09it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.32it/s]
+2022-12-21,11:12:05 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:04<13:36, 61.17it/s]
  1%|          | 512/50176 [00:06<10:40, 77.51it/s]
  2%|▏         | 768/50176 [00:09<09:33, 86.17it/s]
  2%|▏         | 1024/50176 [00:12<09:09, 89.44it/s]
  3%|▎         | 1280/50176 [00:15<09:08, 89.09it/s]
  3%|▎         | 1536/50176 [00:18<09:27, 85.68it/s]
  4%|▎         | 1792/50176 [00:21<09:14, 87.29it/s]
  4%|▍         | 2048/50176 [00:23<08:45, 91.53it/s]
  5%|▍         | 2304/50176 [00:26<08:47, 90.69it/s]
  5%|▌         | 2560/50176 [00:28<08:22, 94.77it/s]
  6%|▌         | 2816/50176 [00:31<08:31, 92.57it/s]
  6%|▌         | 3072/50176 [00:34<08:23, 93.64it/s]
  7%|▋         | 3328/50176 [00:37<08:10, 95.51it/s]
  7%|▋         | 3584/50176 [00:39<07:57, 97.49it/s]
  8%|▊         | 3840/50176 [00:42<08:07, 94.99it/s]
  8%|▊         | 4096/50176 [00:45<08:24, 91.32it/s]
  9%|▊         | 4352/50176 [00:47<08:05, 94.33it/s]
  9%|▉         | 4608/50176 [00:50<07:59, 95.11it/s]
 10%|▉         | 4864/50176 [00:53<07:44, 97.51it/s]
 10%|█         | 5120/50176 [00:55<07:35, 98.96it/s]
 11%|█         | 5376/50176 [00:58<07:40, 97.22it/s]
 11%|█         | 5632/50176 [01:01<07:53, 94.10it/s]
 12%|█▏        | 5888/50176 [01:03<07:41, 95.87it/s]
 12%|█▏        | 6144/50176 [01:06<08:05, 90.64it/s]
 13%|█▎        | 6400/50176 [01:09<07:50, 92.94it/s]
 13%|█▎        | 6656/50176 [01:11<07:33, 95.91it/s]
 14%|█▍        | 6912/50176 [01:14<07:36, 94.70it/s]
 14%|█▍        | 7168/50176 [01:17<07:23, 96.89it/s]
 15%|█▍        | 7424/50176 [01:19<07:16, 97.99it/s]
 15%|█▌        | 7680/50176 [01:22<07:10, 98.80it/s]
 16%|█▌        | 7936/50176 [01:24<07:03, 99.71it/s]
 16%|█▋        | 8192/50176 [01:27<07:05, 98.58it/s]
 17%|█▋        | 8448/50176 [01:29<06:54, 100.58it/s]
 17%|█▋        | 8704/50176 [01:32<06:58, 99.18it/s] 
 18%|█▊        | 8960/50176 [01:35<07:03, 97.38it/s]
 18%|█▊        | 9216/50176 [01:37<06:50, 99.88it/s]
 19%|█▉        | 9472/50176 [01:40<06:46, 100.17it/s]
 19%|█▉        | 9728/50176 [01:43<06:51, 98.29it/s] 
 20%|█▉        | 9984/50176 [01:48<08:43, 76.74it/s]
 20%|██        | 10240/50176 [01:52<09:32, 69.75it/s]
 21%|██        | 10496/50176 [01:54<08:29, 77.81it/s]
 21%|██▏       | 10752/50176 [01:57<08:04, 81.34it/s]
 22%|██▏       | 11008/50176 [02:00<07:28, 87.30it/s]
 22%|██▏       | 11264/50176 [02:02<07:07, 91.10it/s]
 23%|██▎       | 11520/50176 [02:05<06:50, 94.25it/s]
 23%|██▎       | 11776/50176 [02:07<06:33, 97.56it/s]
 24%|██▍       | 12032/50176 [02:10<06:32, 97.18it/s]
 24%|██▍       | 12288/50176 [02:12<06:19, 99.73it/s]
 25%|██▌       | 12544/50176 [02:15<06:11, 101.25it/s]
 26%|██▌       | 12800/50176 [02:17<06:10, 100.84it/s]
 26%|██▌       | 13056/50176 [02:20<06:06, 101.37it/s]
 27%|██▋       | 13312/50176 [02:22<06:03, 101.48it/s]
 27%|██▋       | 13568/50176 [02:25<06:11, 98.43it/s] 
 28%|██▊       | 13824/50176 [02:28<06:25, 94.38it/s]
 28%|██▊       | 14080/50176 [02:31<06:22, 94.33it/s]
 29%|██▊       | 14336/50176 [02:33<06:23, 93.45it/s]
 29%|██▉       | 14592/50176 [02:36<06:08, 96.58it/s]
 30%|██▉       | 14848/50176 [02:39<06:30, 90.35it/s]
 30%|███       | 15104/50176 [02:42<06:17, 92.83it/s]
 31%|███       | 15360/50176 [02:44<06:08, 94.45it/s]
 31%|███       | 15616/50176 [02:47<05:55, 97.24it/s]
 32%|███▏      | 15872/50176 [02:52<07:27, 76.66it/s]
 32%|███▏      | 16128/50176 [02:54<06:50, 83.02it/s]
 33%|███▎      | 16384/50176 [02:57<06:22, 88.34it/s]
 33%|███▎      | 16640/50176 [03:00<06:23, 87.36it/s]
 34%|███▎      | 16896/50176 [03:02<06:05, 91.07it/s]
 34%|███▍      | 17152/50176 [03:05<05:50, 94.13it/s]
 35%|███▍      | 17408/50176 [03:07<05:45, 94.73it/s]
 35%|███▌      | 17664/50176 [03:10<05:30, 98.37it/s]
 36%|███▌      | 17920/50176 [03:12<05:24, 99.47it/s]
 36%|███▌      | 18176/50176 [03:15<05:22, 99.26it/s]
 37%|███▋      | 18432/50176 [03:17<05:15, 100.75it/s]
 37%|███▋      | 18688/50176 [03:20<05:25, 96.64it/s] 
 38%|███▊      | 18944/50176 [03:23<05:24, 96.15it/s]
 38%|███▊      | 19200/50176 [03:26<05:27, 94.61it/s]
 39%|███▉      | 19456/50176 [03:29<05:25, 94.25it/s]
 39%|███▉      | 19712/50176 [03:34<06:46, 74.89it/s]
 40%|███▉      | 19968/50176 [03:38<07:29, 67.13it/s]
 40%|████      | 20224/50176 [03:40<06:29, 76.90it/s]
 41%|████      | 20480/50176 [03:45<07:22, 67.09it/s]
 41%|████▏     | 20736/50176 [03:48<06:31, 75.12it/s]
 42%|████▏     | 20992/50176 [03:50<05:50, 83.25it/s]
 42%|████▏     | 21248/50176 [03:53<05:34, 86.35it/s]
 43%|████▎     | 21504/50176 [03:58<06:36, 72.38it/s]
 43%|████▎     | 21760/50176 [04:00<05:56, 79.68it/s]
 44%|████▍     | 22016/50176 [04:02<05:21, 87.57it/s]
 44%|████▍     | 22272/50176 [04:05<05:01, 92.67it/s]
 45%|████▍     | 22528/50176 [04:07<04:44, 97.23it/s]
 45%|████▌     | 22784/50176 [04:10<04:41, 97.23it/s]
 46%|████▌     | 23040/50176 [04:13<04:40, 96.89it/s]
 46%|████▋     | 23296/50176 [04:15<04:42, 95.26it/s]
 47%|████▋     | 23552/50176 [04:18<04:37, 95.90it/s]
 47%|████▋     | 23808/50176 [04:20<04:22, 100.45it/s]
 48%|████▊     | 24064/50176 [04:23<04:15, 102.28it/s]
 48%|████▊     | 24320/50176 [04:25<04:15, 101.38it/s]
 49%|████▉     | 24576/50176 [04:28<04:13, 100.84it/s]
 49%|████▉     | 24832/50176 [04:30<04:08, 101.89it/s]
 50%|█████     | 25088/50176 [04:33<04:00, 104.24it/s]
 51%|█████     | 25344/50176 [04:35<03:54, 106.08it/s]
 51%|█████     | 25600/50176 [04:37<03:55, 104.35it/s]
 52%|█████▏    | 25856/50176 [04:40<03:51, 105.13it/s]
 52%|█████▏    | 26112/50176 [04:42<03:46, 106.17it/s]
 53%|█████▎    | 26368/50176 [04:45<03:45, 105.64it/s]
 53%|█████▎    | 26624/50176 [04:47<03:43, 105.59it/s]
 54%|█████▎    | 26880/50176 [04:49<03:42, 104.84it/s]
 54%|█████▍    | 27136/50176 [04:52<03:42, 103.37it/s]
 55%|█████▍    | 27392/50176 [04:54<03:32, 106.97it/s]
 55%|█████▌    | 27648/50176 [04:56<03:26, 108.83it/s]
 56%|█████▌    | 27904/50176 [04:59<03:35, 103.26it/s]
 56%|█████▌    | 28160/50176 [05:02<03:37, 101.02it/s]
 57%|█████▋    | 28416/50176 [05:04<03:35, 101.19it/s]
 57%|█████▋    | 28672/50176 [05:07<03:31, 101.52it/s]
 58%|█████▊    | 28928/50176 [05:09<03:22, 104.81it/s]
 58%|█████▊    | 29184/50176 [05:12<03:22, 103.66it/s]
 59%|█████▊    | 29440/50176 [05:15<03:32, 97.53it/s] 
 59%|█████▉    | 29696/50176 [05:18<03:35, 95.06it/s]
 60%|█████▉    | 29952/50176 [05:20<03:27, 97.26it/s]
 60%|██████    | 30208/50176 [05:23<03:21, 99.29it/s]
 61%|██████    | 30464/50176 [05:25<03:09, 104.28it/s]
 61%|██████    | 30720/50176 [05:27<03:08, 103.40it/s]
 62%|██████▏   | 30976/50176 [05:29<03:01, 106.07it/s]
 62%|██████▏   | 31232/50176 [05:32<02:59, 105.69it/s]
 63%|██████▎   | 31488/50176 [05:34<02:53, 107.73it/s]
 63%|██████▎   | 31744/50176 [05:37<02:49, 108.57it/s]
 64%|██████▍   | 32000/50176 [05:39<02:48, 108.15it/s]
 64%|██████▍   | 32256/50176 [05:41<02:48, 106.35it/s]
 65%|██████▍   | 32512/50176 [05:44<02:53, 102.04it/s]
 65%|██████▌   | 32768/50176 [05:46<02:44, 105.64it/s]
 66%|██████▌   | 33024/50176 [05:49<02:42, 105.70it/s]
 66%|██████▋   | 33280/50176 [05:51<02:36, 107.90it/s]
 67%|██████▋   | 33536/50176 [05:54<02:37, 105.61it/s]
 67%|██████▋   | 33792/50176 [05:56<02:34, 106.05it/s]
 68%|██████▊   | 34048/50176 [05:58<02:28, 108.36it/s]
 68%|██████▊   | 34304/50176 [06:01<02:24, 109.49it/s]
 69%|██████▉   | 34560/50176 [06:03<02:26, 106.56it/s]
 69%|██████▉   | 34816/50176 [06:06<02:26, 105.19it/s]
 70%|██████▉   | 35072/50176 [06:08<02:25, 103.77it/s]
 70%|███████   | 35328/50176 [06:10<02:19, 106.68it/s]
 71%|███████   | 35584/50176 [06:13<02:16, 106.61it/s]
 71%|███████▏  | 35840/50176 [06:15<02:17, 103.90it/s]
 72%|███████▏  | 36096/50176 [06:18<02:12, 106.41it/s]
 72%|███████▏  | 36352/50176 [06:20<02:11, 105.40it/s]
 73%|███████▎  | 36608/50176 [06:23<02:10, 103.67it/s]
 73%|███████▎  | 36864/50176 [06:25<02:12, 100.66it/s]
 74%|███████▍  | 37120/50176 [06:28<02:08, 101.75it/s]
 74%|███████▍  | 37376/50176 [06:30<02:00, 105.89it/s]
 75%|███████▌  | 37632/50176 [06:32<01:57, 106.46it/s]
 76%|███████▌  | 37888/50176 [06:35<01:55, 106.58it/s]
 76%|███████▌  | 38144/50176 [06:37<01:52, 107.23it/s]
 77%|███████▋  | 38400/50176 [06:40<01:51, 105.53it/s]
 77%|███████▋  | 38656/50176 [06:42<01:50, 104.51it/s]
 78%|███████▊  | 38912/50176 [06:45<01:47, 104.54it/s]
 78%|███████▊  | 39168/50176 [06:47<01:47, 102.68it/s]
 79%|███████▊  | 39424/50176 [06:50<01:42, 104.67it/s]
 79%|███████▉  | 39680/50176 [06:52<01:40, 104.53it/s]
 80%|███████▉  | 39936/50176 [06:54<01:35, 107.26it/s]
 80%|████████  | 40192/50176 [06:57<01:33, 106.44it/s]
 81%|████████  | 40448/50176 [06:59<01:35, 102.39it/s]
 81%|████████  | 40704/50176 [07:02<01:36, 98.20it/s] 
 82%|████████▏ | 40960/50176 [07:05<01:29, 102.82it/s]
 82%|████████▏ | 41216/50176 [07:07<01:25, 104.28it/s]
 83%|████████▎ | 41472/50176 [07:09<01:22, 105.06it/s]
 83%|████████▎ | 41728/50176 [07:12<01:19, 106.28it/s]
 84%|████████▎ | 41984/50176 [07:14<01:16, 106.53it/s]
 84%|████████▍ | 42240/50176 [07:16<01:13, 108.22it/s]
 85%|████████▍ | 42496/50176 [07:19<01:10, 109.46it/s]
 85%|████████▌ | 42752/50176 [07:23<01:30, 82.46it/s] 
 86%|████████▌ | 43008/50176 [07:26<01:21, 88.37it/s]
 86%|████████▌ | 43264/50176 [07:28<01:14, 93.22it/s]
 87%|████████▋ | 43520/50176 [07:35<01:45, 63.26it/s]
 87%|████████▋ | 43776/50176 [07:40<01:46, 59.84it/s]
 88%|████████▊ | 44032/50176 [07:42<01:28, 69.56it/s]
 88%|████████▊ | 44288/50176 [07:45<01:16, 77.38it/s]
 89%|████████▉ | 44544/50176 [07:48<01:08, 82.53it/s]
 89%|████████▉ | 44800/50176 [07:50<01:01, 87.99it/s]
 90%|████████▉ | 45056/50176 [07:52<00:54, 93.95it/s]
 90%|█████████ | 45312/50176 [07:55<00:50, 95.63it/s]
 91%|█████████ | 45568/50176 [07:57<00:45, 101.32it/s]
 91%|█████████▏| 45824/50176 [08:00<00:43, 101.02it/s]
 92%|█████████▏| 46080/50176 [08:02<00:40, 99.97it/s] 
 92%|█████████▏| 46336/50176 [08:05<00:38, 100.66it/s]
 93%|█████████▎| 46592/50176 [08:07<00:34, 104.02it/s]
 93%|█████████▎| 46848/50176 [08:09<00:32, 103.44it/s]
 94%|█████████▍| 47104/50176 [08:12<00:30, 99.14it/s] 
 94%|█████████▍| 47360/50176 [08:15<00:28, 99.06it/s]
 95%|█████████▍| 47616/50176 [08:17<00:25, 98.71it/s]
 95%|█████████▌| 47872/50176 [08:20<00:22, 101.36it/s]
 96%|█████████▌| 48128/50176 [08:22<00:19, 103.35it/s]
 96%|█████████▋| 48384/50176 [08:25<00:17, 103.50it/s]
 97%|█████████▋| 48640/50176 [08:27<00:14, 104.05it/s]
 97%|█████████▋| 48896/50176 [08:30<00:12, 103.98it/s]
 98%|█████████▊| 49152/50176 [08:32<00:09, 102.80it/s]
 98%|█████████▊| 49408/50176 [08:35<00:07, 102.16it/s]
 99%|█████████▉| 49664/50176 [08:37<00:05, 101.33it/s]
 99%|█████████▉| 49920/50176 [08:40<00:02, 95.71it/s] 
100%|██████████| 50176/50176 [08:41<00:00, 124.94it/s]
100%|██████████| 50176/50176 [08:41<00:00, 96.23it/s] 
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6658	imagenet-zeroshot-val-top5: 0.9001
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_57.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_57.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f7b9600e013c19f7138efd57293740c6de68c173
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_57.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcc59a6a1a1310736cf2e7f0ec94a3677d1284da62c393ded08a26f4b18acc21
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_57.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_57.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..10c08c3c498fd4d103396f3e9c42ea7634e0fd09
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_57.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_57.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_57.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_57.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_57.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_57.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:12 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:12 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:10,  1.37it/s]
  1%|          | 6/1000 [00:00<01:47,  9.26it/s]
  1%|          | 12/1000 [00:00<00:53, 18.38it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.38it/s]
  2%|▏         | 24/1000 [00:01<00:29, 33.60it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.57it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.21it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.75it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.40it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.23it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.65it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.67it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.41it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.94it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.48it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.80it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.19it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.38it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.63it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.73it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.72it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.65it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.61it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.61it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.66it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.74it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.75it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.60it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.43it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.45it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.43it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.35it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.30it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.24it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.05it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.96it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.07it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.04it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.26it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.27it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.18it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.25it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.34it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.45it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.50it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.31it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.29it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.47it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.59it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.77it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.67it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.66it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.81it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.81it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.71it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.55it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.64it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.76it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.85it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.63it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.62it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.68it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.77it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.67it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.36it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.37it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.48it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.64it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.73it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.72it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.73it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.78it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.91it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.90it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.70it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.81it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.78it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.72it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.85it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.72it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.71it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.70it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.62it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.66it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.58it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.66it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.71it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.80it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.75it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.86it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.86it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.76it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.75it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.79it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.67it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.58it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.72it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.70it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.61it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.74it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.64it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.65it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.76it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.77it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.84it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.68it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.62it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.76it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.71it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.79it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.79it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.72it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.78it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.79it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.81it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.68it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.76it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.77it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.74it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.77it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.72it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.70it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.71it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.72it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.80it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.87it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.50it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.65it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.63it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.70it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.76it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.58it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.57it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.64it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.62it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.59it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.59it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.70it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.85it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.88it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.79it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.84it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.81it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.84it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.75it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.82it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.83it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.69it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.78it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.79it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.75it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.77it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.74it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.73it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.82it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.88it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.87it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.89it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.91it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.86it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.91it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.98it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.01it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.70it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.34it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 57.08it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.25it/s]
+2022-12-21,11:21:30 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:37, 71.57it/s]
  1%|          | 512/50176 [00:06<09:41, 85.47it/s]
  2%|▏         | 768/50176 [00:08<08:39, 95.10it/s]
  2%|▏         | 1024/50176 [00:11<08:39, 94.56it/s]
  3%|▎         | 1280/50176 [00:13<08:40, 93.86it/s]
  3%|▎         | 1536/50176 [00:17<09:00, 89.95it/s]
  4%|▎         | 1792/50176 [00:19<08:50, 91.13it/s]
  4%|▍         | 2048/50176 [00:24<10:45, 74.57it/s]
  5%|▍         | 2304/50176 [00:27<09:59, 79.88it/s]
  5%|▌         | 2560/50176 [00:29<09:07, 86.92it/s]
  6%|▌         | 2816/50176 [00:32<09:02, 87.35it/s]
  6%|▌         | 3072/50176 [00:35<08:38, 90.81it/s]
  7%|▋         | 3328/50176 [00:37<08:18, 94.00it/s]
  7%|▋         | 3584/50176 [00:40<08:01, 96.79it/s]
  8%|▊         | 3840/50176 [00:42<08:08, 94.84it/s]
  8%|▊         | 4096/50176 [00:45<08:22, 91.75it/s]
  9%|▊         | 4352/50176 [00:48<08:02, 94.96it/s]
  9%|▉         | 4608/50176 [00:50<07:52, 96.41it/s]
 10%|▉         | 4864/50176 [00:53<07:45, 97.44it/s]
 10%|█         | 5120/50176 [00:55<07:35, 98.92it/s]
 11%|█         | 5376/50176 [00:58<07:34, 98.66it/s]
 11%|█         | 5632/50176 [01:01<07:41, 96.54it/s]
 12%|█▏        | 5888/50176 [01:03<07:35, 97.29it/s]
 12%|█▏        | 6144/50176 [01:07<07:53, 92.96it/s]
 13%|█▎        | 6400/50176 [01:09<07:47, 93.65it/s]
 13%|█▎        | 6656/50176 [01:12<07:32, 96.22it/s]
 14%|█▍        | 6912/50176 [01:14<07:36, 94.84it/s]
 14%|█▍        | 7168/50176 [01:17<07:27, 96.07it/s]
 15%|█▍        | 7424/50176 [01:19<07:13, 98.66it/s]
 15%|█▌        | 7680/50176 [01:22<07:02, 100.50it/s]
 16%|█▌        | 7936/50176 [01:24<06:48, 103.39it/s]
 16%|█▋        | 8192/50176 [01:27<06:48, 102.69it/s]
 17%|█▋        | 8448/50176 [01:29<06:44, 103.14it/s]
 17%|█▋        | 8704/50176 [01:32<06:44, 102.40it/s]
 18%|█▊        | 8960/50176 [01:34<06:48, 100.84it/s]
 18%|█▊        | 9216/50176 [01:37<06:36, 103.31it/s]
 19%|█▉        | 9472/50176 [01:39<06:29, 104.47it/s]
 19%|█▉        | 9728/50176 [01:42<06:40, 100.92it/s]
 20%|█▉        | 9984/50176 [01:44<06:40, 100.30it/s]
 20%|██        | 10240/50176 [01:47<06:23, 104.14it/s]
 21%|██        | 10496/50176 [01:49<06:20, 104.22it/s]
 21%|██▏       | 10752/50176 [01:52<06:27, 101.83it/s]
 22%|██▏       | 11008/50176 [01:54<06:22, 102.31it/s]
 22%|██▏       | 11264/50176 [01:57<06:20, 102.19it/s]
 23%|██▎       | 11520/50176 [01:59<06:11, 104.14it/s]
 23%|██▎       | 11776/50176 [02:02<06:09, 103.93it/s]
 24%|██▍       | 12032/50176 [02:04<06:13, 102.16it/s]
 24%|██▍       | 12288/50176 [02:07<06:08, 102.84it/s]
 25%|██▌       | 12544/50176 [02:09<06:02, 103.87it/s]
 26%|██▌       | 12800/50176 [02:12<06:02, 103.24it/s]
 26%|██▌       | 13056/50176 [02:14<05:51, 105.50it/s]
 27%|██▋       | 13312/50176 [02:16<05:50, 105.25it/s]
 27%|██▋       | 13568/50176 [02:19<06:07, 99.59it/s] 
 28%|██▊       | 13824/50176 [02:22<06:19, 95.80it/s]
 28%|██▊       | 14080/50176 [02:25<06:15, 96.21it/s]
 29%|██▊       | 14336/50176 [02:27<06:15, 95.40it/s]
 29%|██▉       | 14592/50176 [02:30<06:03, 98.02it/s]
 30%|██▉       | 14848/50176 [02:33<06:18, 93.45it/s]
 30%|███       | 15104/50176 [02:36<06:13, 93.82it/s]
 31%|███       | 15360/50176 [02:38<05:57, 97.40it/s]
 31%|███       | 15616/50176 [02:40<05:44, 100.21it/s]
 32%|███▏      | 15872/50176 [02:43<05:47, 98.73it/s] 
 32%|███▏      | 16128/50176 [02:46<05:43, 99.10it/s]
 33%|███▎      | 16384/50176 [02:48<05:32, 101.58it/s]
 33%|███▎      | 16640/50176 [02:51<05:39, 98.91it/s] 
 34%|███▎      | 16896/50176 [02:53<05:34, 99.56it/s]
 34%|███▍      | 17152/50176 [02:56<05:25, 101.45it/s]
 35%|███▍      | 17408/50176 [02:58<05:26, 100.24it/s]
 35%|███▌      | 17664/50176 [03:01<05:15, 103.10it/s]
 36%|███▌      | 17920/50176 [03:03<05:11, 103.62it/s]
 36%|███▌      | 18176/50176 [03:06<05:05, 104.58it/s]
 37%|███▋      | 18432/50176 [03:08<05:03, 104.67it/s]
 37%|███▋      | 18688/50176 [03:11<05:12, 100.70it/s]
 38%|███▊      | 18944/50176 [03:13<05:11, 100.39it/s]
 38%|███▊      | 19200/50176 [03:16<05:20, 96.72it/s] 
 39%|███▉      | 19456/50176 [03:19<05:22, 95.23it/s]
 39%|███▉      | 19712/50176 [03:22<05:20, 95.00it/s]
 40%|███▉      | 19968/50176 [03:24<05:15, 95.69it/s]
 40%|████      | 20224/50176 [03:27<04:56, 101.04it/s]
 41%|████      | 20480/50176 [03:29<04:54, 100.91it/s]
 41%|████▏     | 20736/50176 [03:31<04:46, 102.66it/s]
 42%|████▏     | 20992/50176 [03:34<04:34, 106.46it/s]
 42%|████▏     | 21248/50176 [03:36<04:33, 105.62it/s]
 43%|████▎     | 21504/50176 [03:38<04:29, 106.37it/s]
 43%|████▎     | 21760/50176 [03:41<04:29, 105.49it/s]
 44%|████▍     | 22016/50176 [03:43<04:22, 107.37it/s]
 44%|████▍     | 22272/50176 [03:46<04:17, 108.29it/s]
 45%|████▍     | 22528/50176 [03:48<04:11, 109.91it/s]
 45%|████▌     | 22784/50176 [03:50<04:20, 105.34it/s]
 46%|████▌     | 23040/50176 [03:53<04:22, 103.32it/s]
 46%|████▋     | 23296/50176 [03:56<04:28, 100.00it/s]
 47%|████▋     | 23552/50176 [03:59<04:32, 97.68it/s] 
 47%|████▋     | 23808/50176 [04:01<04:18, 102.03it/s]
 48%|████▊     | 24064/50176 [04:03<04:15, 102.30it/s]
 48%|████▊     | 24320/50176 [04:06<04:16, 100.65it/s]
 49%|████▉     | 24576/50176 [04:08<04:11, 101.79it/s]
 49%|████▉     | 24832/50176 [04:11<04:05, 103.12it/s]
 50%|█████     | 25088/50176 [04:13<04:00, 104.50it/s]
 51%|█████     | 25344/50176 [04:16<03:54, 105.73it/s]
 51%|█████     | 25600/50176 [04:18<03:52, 105.51it/s]
 52%|█████▏    | 25856/50176 [04:20<03:46, 107.24it/s]
 52%|█████▏    | 26112/50176 [04:23<03:47, 105.60it/s]
 53%|█████▎    | 26368/50176 [04:25<03:48, 104.41it/s]
 53%|█████▎    | 26624/50176 [04:28<03:42, 106.05it/s]
 54%|█████▎    | 26880/50176 [04:30<03:41, 105.20it/s]
 54%|█████▍    | 27136/50176 [04:33<03:38, 105.51it/s]
 55%|█████▍    | 27392/50176 [04:35<03:31, 107.90it/s]
 55%|█████▌    | 27648/50176 [04:37<03:24, 110.30it/s]
 56%|█████▌    | 27904/50176 [04:40<03:32, 104.82it/s]
 56%|█████▌    | 28160/50176 [04:42<03:35, 102.16it/s]
 57%|█████▋    | 28416/50176 [04:45<03:30, 103.27it/s]
 57%|█████▋    | 28672/50176 [04:47<03:32, 101.04it/s]
 58%|█████▊    | 28928/50176 [04:50<03:22, 105.06it/s]
 58%|█████▊    | 29184/50176 [04:52<03:20, 104.56it/s]
 59%|█████▊    | 29440/50176 [04:55<03:29, 98.93it/s] 
 59%|█████▉    | 29696/50176 [04:58<03:28, 98.42it/s]
 60%|█████▉    | 29952/50176 [05:00<03:23, 99.50it/s]
 60%|██████    | 30208/50176 [05:03<03:17, 101.28it/s]
 61%|██████    | 30464/50176 [05:05<03:06, 105.49it/s]
 61%|██████    | 30720/50176 [05:07<03:03, 105.91it/s]
 62%|██████▏   | 30976/50176 [05:09<02:57, 108.08it/s]
 62%|██████▏   | 31232/50176 [05:12<02:54, 108.40it/s]
 63%|██████▎   | 31488/50176 [05:14<02:49, 110.52it/s]
 63%|██████▎   | 31744/50176 [05:16<02:46, 110.39it/s]
 64%|██████▍   | 32000/50176 [05:19<02:47, 108.60it/s]
 64%|██████▍   | 32256/50176 [05:21<02:46, 107.58it/s]
 65%|██████▍   | 32512/50176 [05:24<02:54, 100.99it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.42it/s]
 66%|██████▌   | 33024/50176 [05:29<02:44, 104.02it/s]
 66%|██████▋   | 33280/50176 [05:31<02:38, 106.58it/s]
 67%|██████▋   | 33536/50176 [05:34<02:38, 104.78it/s]
 67%|██████▋   | 33792/50176 [05:36<02:33, 106.77it/s]
 68%|██████▊   | 34048/50176 [05:38<02:28, 108.27it/s]
 68%|██████▊   | 34304/50176 [05:40<02:21, 111.80it/s]
 69%|██████▉   | 34560/50176 [05:43<02:26, 106.80it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 106.17it/s]
 70%|██████▉   | 35072/50176 [05:48<02:22, 106.05it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.36it/s]
 71%|███████   | 35584/50176 [05:53<02:14, 108.43it/s]
 71%|███████▏  | 35840/50176 [05:55<02:16, 104.96it/s]
 72%|███████▏  | 36096/50176 [05:58<02:13, 105.84it/s]
 72%|███████▏  | 36352/50176 [06:00<02:10, 105.76it/s]
 73%|███████▎  | 36608/50176 [06:02<02:07, 106.71it/s]
 73%|███████▎  | 36864/50176 [06:05<02:06, 105.54it/s]
 74%|███████▍  | 37120/50176 [06:07<02:02, 106.22it/s]
 74%|███████▍  | 37376/50176 [06:09<01:56, 110.09it/s]
 75%|███████▌  | 37632/50176 [06:12<01:53, 110.91it/s]
 76%|███████▌  | 37888/50176 [06:14<01:51, 110.25it/s]
 76%|███████▌  | 38144/50176 [06:16<01:47, 111.76it/s]
 77%|███████▋  | 38400/50176 [06:19<01:48, 108.71it/s]
 77%|███████▋  | 38656/50176 [06:21<01:48, 106.35it/s]
 78%|███████▊  | 38912/50176 [06:24<01:46, 105.93it/s]
 78%|███████▊  | 39168/50176 [06:26<01:44, 105.34it/s]
 79%|███████▊  | 39424/50176 [06:28<01:39, 108.04it/s]
 79%|███████▉  | 39680/50176 [06:31<01:38, 106.38it/s]
 80%|███████▉  | 39936/50176 [06:33<01:34, 108.45it/s]
 80%|████████  | 40192/50176 [06:35<01:32, 108.49it/s]
 81%|████████  | 40448/50176 [06:38<01:33, 104.52it/s]
 81%|████████  | 40704/50176 [06:41<01:33, 101.67it/s]
 82%|████████▏ | 40960/50176 [06:43<01:27, 105.18it/s]
 82%|████████▏ | 41216/50176 [06:45<01:24, 106.54it/s]
 83%|████████▎ | 41472/50176 [06:48<01:21, 106.92it/s]
 83%|████████▎ | 41728/50176 [06:50<01:17, 109.28it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 107.41it/s]
 84%|████████▍ | 42240/50176 [06:55<01:14, 107.07it/s]
 85%|████████▍ | 42496/50176 [06:57<01:09, 111.05it/s]
 85%|████████▌ | 42752/50176 [06:59<01:08, 108.60it/s]
 86%|████████▌ | 43008/50176 [07:02<01:06, 107.46it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 107.06it/s]
 87%|████████▋ | 43520/50176 [07:07<01:03, 104.69it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 104.22it/s]
 88%|████████▊ | 44032/50176 [07:12<00:58, 105.04it/s]
 88%|████████▊ | 44288/50176 [07:14<00:54, 107.34it/s]
 89%|████████▉ | 44544/50176 [07:17<00:54, 103.92it/s]
 89%|████████▉ | 44800/50176 [07:19<00:52, 102.02it/s]
 90%|████████▉ | 45056/50176 [07:22<00:53, 96.35it/s] 
 90%|█████████ | 45312/50176 [07:25<00:52, 93.43it/s]
 91%|█████████ | 45568/50176 [07:27<00:45, 100.54it/s]
 91%|█████████▏| 45824/50176 [07:30<00:43, 100.46it/s]
 92%|█████████▏| 46080/50176 [07:32<00:40, 100.92it/s]
 92%|█████████▏| 46336/50176 [07:35<00:37, 102.53it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 105.73it/s]
 93%|█████████▎| 46848/50176 [07:39<00:31, 104.88it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.48it/s]
 94%|█████████▍| 47360/50176 [07:45<00:27, 102.00it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 101.69it/s]
 95%|█████████▌| 47872/50176 [07:50<00:22, 103.47it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 104.52it/s]
 96%|█████████▋| 48384/50176 [07:54<00:17, 105.08it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 105.79it/s]
 97%|█████████▋| 48896/50176 [07:59<00:12, 105.88it/s]
 98%|█████████▊| 49152/50176 [08:02<00:09, 103.10it/s]
 98%|█████████▊| 49408/50176 [08:04<00:07, 102.46it/s]
 99%|█████████▉| 49664/50176 [08:07<00:05, 101.09it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 95.35it/s] 
100%|██████████| 50176/50176 [08:10<00:00, 124.89it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.19it/s]
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6676	imagenet-zeroshot-val-top5: 0.9022
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_58.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_58.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d63fd3d81e1e2bcf6e6c6a8ed2567842d4b5f08b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_58.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d4b2e290864da3eca7c4974a693c0e31bf6574b209d0f2e873719ec047f622f
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_58.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_58.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3eccb30d884bcaab14927b9fed19de293d6acd83
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_58.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_58.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_58.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_58.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_58.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_58.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:12 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:12 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:56,  1.52it/s]
  1%|          | 7/1000 [00:00<01:25, 11.56it/s]
  1%|▏         | 13/1000 [00:00<00:47, 20.71it/s]
  2%|▏         | 19/1000 [00:00<00:33, 29.05it/s]
  2%|▎         | 25/1000 [00:01<00:27, 36.00it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.60it/s]
  4%|▎         | 37/1000 [00:01<00:20, 45.97it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.22it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.33it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.07it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.53it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.48it/s]
  7%|▋         | 73/1000 [00:01<00:16, 55.73it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.40it/s]
  8%|▊         | 85/1000 [00:02<00:16, 56.96it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.28it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.68it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.83it/s]
 11%|█         | 109/1000 [00:02<00:15, 58.07it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.26it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 58.16it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.16it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.19it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 58.17it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.21it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.26it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.23it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.05it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.72it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.37it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.42it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.49it/s]
 19%|█▉        | 193/1000 [00:03<00:14, 57.60it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.57it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.36it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.59it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.62it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.84it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.82it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.66it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.82it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.92it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 58.06it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 58.04it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.86it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 58.01it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 58.09it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.23it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 57.99it/s]
 30%|███       | 301/1000 [00:05<00:12, 58.00it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.04it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.27it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 57.52it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 57.59it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 57.64it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 57.78it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 57.79it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 57.89it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 57.91it/s]
 36%|███▌      | 361/1000 [00:06<00:11, 57.95it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.01it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.13it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 57.96it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.37it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.59it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 57.77it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.97it/s]
 41%|████      | 409/1000 [00:07<00:10, 57.80it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 57.98it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.09it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.16it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.10it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.23it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.26it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.31it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.26it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.42it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.17it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.31it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.27it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.20it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.17it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.09it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.24it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.21it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.27it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.33it/s]
 54%|█████▍    | 541/1000 [00:09<00:07, 58.40it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.35it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.26it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.33it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.16it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.24it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.33it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.36it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.27it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.36it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.21it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.25it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.39it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.36it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.33it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.24it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.20it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.27it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.26it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.31it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.35it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.33it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.35it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.32it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.25it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.32it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.37it/s]
 72%|███████▏  | 715/1000 [00:12<00:04, 58.34it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.36it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.30it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.30it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.33it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.23it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.33it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.23it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.36it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.34it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.37it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.34it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.21it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.23it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.19it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.25it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.24it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.37it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.43it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.48it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.36it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.40it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.37it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.39it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.36it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.30it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.24it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.35it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.31it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.26it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.37it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.26it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.36it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.42it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.33it/s]
 95%|█████████▍| 949/1000 [00:16<00:00, 58.33it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.46it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.34it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.44it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.57it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.59it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.55it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.44it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 58.00it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.96it/s]
+2022-12-21,11:21:29 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<12:28, 66.73it/s]
  1%|          | 512/50176 [00:06<10:01, 82.61it/s]
  2%|▏         | 768/50176 [00:08<08:53, 92.69it/s]
  2%|▏         | 1024/50176 [00:11<08:46, 93.38it/s]
  3%|▎         | 1280/50176 [00:14<08:47, 92.78it/s]
  3%|▎         | 1536/50176 [00:17<09:03, 89.45it/s]
  4%|▎         | 1792/50176 [00:20<08:53, 90.70it/s]
  4%|▍         | 2048/50176 [00:24<10:46, 74.39it/s]
  5%|▍         | 2304/50176 [00:27<10:01, 79.60it/s]
  5%|▌         | 2560/50176 [00:29<09:09, 86.67it/s]
  6%|▌         | 2816/50176 [00:32<09:00, 87.56it/s]
  6%|▌         | 3072/50176 [00:35<08:37, 91.03it/s]
  7%|▋         | 3328/50176 [00:37<08:17, 94.10it/s]
  7%|▋         | 3584/50176 [00:40<08:00, 96.94it/s]
  8%|▊         | 3840/50176 [00:43<08:07, 95.04it/s]
  8%|▊         | 4096/50176 [00:46<08:21, 91.80it/s]
  9%|▊         | 4352/50176 [00:48<08:02, 94.88it/s]
  9%|▉         | 4608/50176 [00:51<07:51, 96.64it/s]
 10%|▉         | 4864/50176 [00:53<07:45, 97.37it/s]
 10%|█         | 5120/50176 [00:56<07:35, 98.90it/s]
 11%|█         | 5376/50176 [00:58<07:34, 98.67it/s]
 11%|█         | 5632/50176 [01:01<07:41, 96.44it/s]
 12%|█▏        | 5888/50176 [01:04<07:34, 97.34it/s]
 12%|█▏        | 6144/50176 [01:07<07:53, 93.02it/s]
 13%|█▎        | 6400/50176 [01:09<07:47, 93.62it/s]
 13%|█▎        | 6656/50176 [01:12<07:33, 96.04it/s]
 14%|█▍        | 6912/50176 [01:15<07:35, 94.93it/s]
 14%|█▍        | 7168/50176 [01:17<07:27, 96.09it/s]
 15%|█▍        | 7424/50176 [01:20<07:12, 98.78it/s]
 15%|█▌        | 7680/50176 [01:22<07:02, 100.52it/s]
 16%|█▌        | 7936/50176 [01:24<06:48, 103.35it/s]
 16%|█▋        | 8192/50176 [01:27<06:48, 102.72it/s]
 17%|█▋        | 8448/50176 [01:29<06:45, 102.97it/s]
 17%|█▋        | 8704/50176 [01:32<06:45, 102.34it/s]
 18%|█▊        | 8960/50176 [01:35<06:48, 100.89it/s]
 18%|█▊        | 9216/50176 [01:37<06:36, 103.41it/s]
 19%|█▉        | 9472/50176 [01:39<06:30, 104.35it/s]
 19%|█▉        | 9728/50176 [01:42<06:40, 101.06it/s]
 20%|█▉        | 9984/50176 [01:45<06:40, 100.28it/s]
 20%|██        | 10240/50176 [01:47<06:23, 104.07it/s]
 21%|██        | 10496/50176 [01:49<06:20, 104.29it/s]
 21%|██▏       | 10752/50176 [01:52<06:27, 101.81it/s]
 22%|██▏       | 11008/50176 [01:55<06:23, 102.21it/s]
 22%|██▏       | 11264/50176 [01:57<06:21, 102.09it/s]
 23%|██▎       | 11520/50176 [01:59<06:11, 104.13it/s]
 23%|██▎       | 11776/50176 [02:02<06:09, 103.94it/s]
 24%|██▍       | 12032/50176 [02:04<06:13, 102.20it/s]
 24%|██▍       | 12288/50176 [02:07<06:08, 102.76it/s]
 25%|██▌       | 12544/50176 [02:09<06:02, 103.93it/s]
 26%|██▌       | 12800/50176 [02:12<06:02, 103.21it/s]
 26%|██▌       | 13056/50176 [02:14<05:51, 105.57it/s]
 27%|██▋       | 13312/50176 [02:17<05:50, 105.22it/s]
 27%|██▋       | 13568/50176 [02:19<06:07, 99.62it/s] 
 28%|██▊       | 13824/50176 [02:22<06:19, 95.81it/s]
 28%|██▊       | 14080/50176 [02:25<06:15, 96.25it/s]
 29%|██▊       | 14336/50176 [02:28<06:15, 95.41it/s]
 29%|██▉       | 14592/50176 [02:30<06:02, 98.10it/s]
 30%|██▉       | 14848/50176 [02:33<06:18, 93.37it/s]
 30%|███       | 15104/50176 [02:36<06:13, 93.86it/s]
 31%|███       | 15360/50176 [02:38<05:57, 97.38it/s]
 31%|███       | 15616/50176 [02:41<05:45, 100.06it/s]
 32%|███▏      | 15872/50176 [02:43<05:47, 98.76it/s] 
 32%|███▏      | 16128/50176 [02:46<05:43, 99.02it/s]
 33%|███▎      | 16384/50176 [02:48<05:32, 101.67it/s]
 33%|███▎      | 16640/50176 [02:51<05:38, 98.95it/s] 
 34%|███▎      | 16896/50176 [02:54<05:34, 99.51it/s]
 34%|███▍      | 17152/50176 [02:56<05:25, 101.51it/s]
 35%|███▍      | 17408/50176 [02:59<05:26, 100.37it/s]
 35%|███▌      | 17664/50176 [03:01<05:15, 103.04it/s]
 36%|███▌      | 17920/50176 [03:03<05:11, 103.63it/s]
 36%|███▌      | 18176/50176 [03:06<05:05, 104.58it/s]
 37%|███▋      | 18432/50176 [03:08<05:02, 104.81it/s]
 37%|███▋      | 18688/50176 [03:11<05:12, 100.69it/s]
 38%|███▊      | 18944/50176 [03:14<05:11, 100.22it/s]
 38%|███▊      | 19200/50176 [03:16<05:20, 96.71it/s] 
 39%|███▉      | 19456/50176 [03:19<05:22, 95.18it/s]
 39%|███▉      | 19712/50176 [03:22<05:20, 95.01it/s]
 40%|███▉      | 19968/50176 [03:25<05:15, 95.80it/s]
 40%|████      | 20224/50176 [03:27<04:56, 101.01it/s]
 41%|████      | 20480/50176 [03:29<04:54, 100.73it/s]
 41%|████▏     | 20736/50176 [03:32<04:46, 102.76it/s]
 42%|████▏     | 20992/50176 [03:34<04:34, 106.48it/s]
 42%|████▏     | 21248/50176 [03:36<04:33, 105.66it/s]
 43%|████▎     | 21504/50176 [03:39<04:29, 106.37it/s]
 43%|████▎     | 21760/50176 [03:41<04:29, 105.52it/s]
 44%|████▍     | 22016/50176 [03:44<04:22, 107.33it/s]
 44%|████▍     | 22272/50176 [03:46<04:17, 108.32it/s]
 45%|████▍     | 22528/50176 [03:48<04:11, 109.94it/s]
 45%|████▌     | 22784/50176 [03:51<04:20, 105.25it/s]
 46%|████▌     | 23040/50176 [03:53<04:22, 103.33it/s]
 46%|████▋     | 23296/50176 [03:56<04:28, 100.03it/s]
 47%|████▋     | 23552/50176 [03:59<04:32, 97.66it/s] 
 47%|████▋     | 23808/50176 [04:01<04:18, 102.03it/s]
 48%|████▊     | 24064/50176 [04:04<04:14, 102.46it/s]
 48%|████▊     | 24320/50176 [04:06<04:16, 100.76it/s]
 49%|████▉     | 24576/50176 [04:09<04:11, 101.68it/s]
 49%|████▉     | 24832/50176 [04:11<04:05, 103.12it/s]
 50%|█████     | 25088/50176 [04:13<03:59, 104.54it/s]
 51%|█████     | 25344/50176 [04:16<03:55, 105.63it/s]
 51%|█████     | 25600/50176 [04:18<03:53, 105.46it/s]
 52%|█████▏    | 25856/50176 [04:21<03:46, 107.25it/s]
 52%|█████▏    | 26112/50176 [04:23<03:47, 105.68it/s]
 53%|█████▎    | 26368/50176 [04:26<03:47, 104.55it/s]
 53%|█████▎    | 26624/50176 [04:28<03:42, 105.92it/s]
 54%|█████▎    | 26880/50176 [04:30<03:41, 105.18it/s]
 54%|█████▍    | 27136/50176 [04:33<03:38, 105.36it/s]
 55%|█████▍    | 27392/50176 [04:35<03:31, 107.89it/s]
 55%|█████▌    | 27648/50176 [04:37<03:24, 110.23it/s]
 56%|█████▌    | 27904/50176 [04:40<03:32, 104.76it/s]
 56%|█████▌    | 28160/50176 [04:43<03:35, 102.16it/s]
 57%|█████▋    | 28416/50176 [04:45<03:30, 103.21it/s]
 57%|█████▋    | 28672/50176 [04:48<03:32, 101.11it/s]
 58%|█████▊    | 28928/50176 [04:50<03:22, 105.09it/s]
 58%|█████▊    | 29184/50176 [04:52<03:20, 104.57it/s]
 59%|█████▊    | 29440/50176 [04:55<03:29, 98.88it/s] 
 59%|█████▉    | 29696/50176 [04:58<03:28, 98.37it/s]
 60%|█████▉    | 29952/50176 [05:00<03:23, 99.52it/s]
 60%|██████    | 30208/50176 [05:03<03:17, 101.32it/s]
 61%|██████    | 30464/50176 [05:05<03:06, 105.49it/s]
 61%|██████    | 30720/50176 [05:07<03:03, 105.96it/s]
 62%|██████▏   | 30976/50176 [05:10<02:57, 108.09it/s]
 62%|██████▏   | 31232/50176 [05:12<02:55, 108.25it/s]
 63%|██████▎   | 31488/50176 [05:14<02:48, 110.64it/s]
 63%|██████▎   | 31744/50176 [05:17<02:46, 110.37it/s]
 64%|██████▍   | 32000/50176 [05:19<02:47, 108.62it/s]
 64%|██████▍   | 32256/50176 [05:21<02:46, 107.59it/s]
 65%|██████▍   | 32512/50176 [05:24<02:55, 100.89it/s]
 65%|██████▌   | 32768/50176 [05:27<02:49, 102.42it/s]
 66%|██████▌   | 33024/50176 [05:29<02:44, 104.11it/s]
 66%|██████▋   | 33280/50176 [05:31<02:38, 106.67it/s]
 67%|██████▋   | 33536/50176 [05:34<02:38, 104.77it/s]
 67%|██████▋   | 33792/50176 [05:36<02:33, 106.94it/s]
 68%|██████▊   | 34048/50176 [05:38<02:29, 108.24it/s]
 68%|██████▊   | 34304/50176 [05:41<02:21, 111.78it/s]
 69%|██████▉   | 34560/50176 [05:43<02:26, 106.88it/s]
 69%|██████▉   | 34816/50176 [05:46<02:24, 105.99it/s]
 70%|██████▉   | 35072/50176 [05:48<02:22, 105.94it/s]
 70%|███████   | 35328/50176 [05:51<02:19, 106.40it/s]
 71%|███████   | 35584/50176 [05:53<02:14, 108.24it/s]
 71%|███████▏  | 35840/50176 [05:55<02:16, 104.86it/s]
 72%|███████▏  | 36096/50176 [05:58<02:12, 105.95it/s]
 72%|███████▏  | 36352/50176 [06:00<02:10, 105.85it/s]
 73%|███████▎  | 36608/50176 [06:03<02:07, 106.74it/s]
 73%|███████▎  | 36864/50176 [06:05<02:06, 105.57it/s]
 74%|███████▍  | 37120/50176 [06:07<02:02, 106.26it/s]
 74%|███████▍  | 37376/50176 [06:10<01:56, 109.95it/s]
 75%|███████▌  | 37632/50176 [06:12<01:53, 110.97it/s]
 76%|███████▌  | 37888/50176 [06:14<01:51, 110.28it/s]
 76%|███████▌  | 38144/50176 [06:16<01:47, 111.77it/s]
 77%|███████▋  | 38400/50176 [06:19<01:48, 108.81it/s]
 77%|███████▋  | 38656/50176 [06:21<01:48, 106.34it/s]
 78%|███████▊  | 38912/50176 [06:24<01:46, 105.78it/s]
 78%|███████▊  | 39168/50176 [06:26<01:44, 105.44it/s]
 79%|███████▊  | 39424/50176 [06:29<01:39, 108.03it/s]
 79%|███████▉  | 39680/50176 [06:31<01:38, 106.40it/s]
 80%|███████▉  | 39936/50176 [06:33<01:34, 108.49it/s]
 80%|████████  | 40192/50176 [06:36<01:32, 108.44it/s]
 81%|████████  | 40448/50176 [06:38<01:32, 104.61it/s]
 81%|████████  | 40704/50176 [06:41<01:33, 101.69it/s]
 82%|████████▏ | 40960/50176 [06:43<01:27, 105.00it/s]
 82%|████████▏ | 41216/50176 [06:46<01:24, 106.53it/s]
 83%|████████▎ | 41472/50176 [06:48<01:21, 106.92it/s]
 83%|████████▎ | 41728/50176 [06:50<01:17, 109.45it/s]
 84%|████████▎ | 41984/50176 [06:53<01:16, 107.37it/s]
 84%|████████▍ | 42240/50176 [06:55<01:14, 107.11it/s]
 85%|████████▍ | 42496/50176 [06:57<01:09, 110.94it/s]
 85%|████████▌ | 42752/50176 [07:00<01:08, 108.73it/s]
 86%|████████▌ | 43008/50176 [07:02<01:06, 107.43it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 107.01it/s]
 87%|████████▋ | 43520/50176 [07:07<01:03, 104.54it/s]
 87%|████████▋ | 43776/50176 [07:10<01:01, 104.26it/s]
 88%|████████▊ | 44032/50176 [07:12<00:58, 105.03it/s]
 88%|████████▊ | 44288/50176 [07:14<00:54, 107.35it/s]
 89%|████████▉ | 44544/50176 [07:17<00:54, 103.93it/s]
 89%|████████▉ | 44800/50176 [07:19<00:52, 102.06it/s]
 90%|████████▉ | 45056/50176 [07:22<00:53, 96.32it/s] 
 90%|█████████ | 45312/50176 [07:25<00:52, 93.43it/s]
 91%|█████████ | 45568/50176 [07:27<00:45, 100.52it/s]
 91%|█████████▏| 45824/50176 [07:30<00:43, 100.38it/s]
 92%|█████████▏| 46080/50176 [07:33<00:40, 100.95it/s]
 92%|█████████▏| 46336/50176 [07:35<00:37, 102.56it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 105.75it/s]
 93%|█████████▎| 46848/50176 [07:40<00:31, 104.92it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.47it/s]
 94%|█████████▍| 47360/50176 [07:45<00:27, 102.01it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 101.60it/s]
 95%|█████████▌| 47872/50176 [07:50<00:22, 103.56it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 104.56it/s]
 96%|█████████▋| 48384/50176 [07:55<00:17, 105.08it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 105.75it/s]
 97%|█████████▋| 48896/50176 [07:59<00:12, 105.89it/s]
 98%|█████████▊| 49152/50176 [08:02<00:09, 103.08it/s]
 98%|█████████▊| 49408/50176 [08:05<00:07, 102.36it/s]
 99%|█████████▉| 49664/50176 [08:07<00:05, 101.10it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 95.27it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 124.82it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.13it/s]
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6720	imagenet-zeroshot-val-top5: 0.9051
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_59.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_59.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4925242d55e685dec30e4b2b3113b6439ee3a69d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_59.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e1d5f9abca797edc511ede942d8fb141529ff9419a1cb5ac2584b3b3d22854de
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_59.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_59.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2fe3e3c4fdd5e37dff4cbd40042516ebb72bc677
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_59.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_59.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_59.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_59.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_59.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_59.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:11 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:11 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:37,  1.57it/s]
  1%|          | 6/1000 [00:00<01:36, 10.35it/s]
  1%|          | 12/1000 [00:00<00:49, 20.05it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.61it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.81it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.55it/s]
  4%|▎         | 36/1000 [00:01<00:20, 45.91it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.16it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.55it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.30it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.55it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.50it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.84it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.83it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.15it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.54it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.74it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.00it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.13it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.04it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.91it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.91it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.98it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.05it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.09it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.86it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.77it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.79it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.77it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.71it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.71it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.66it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.46it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.34it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.42it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.40it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.60it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.55it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.48it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.55it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.65it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.76it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.81it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.82it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.94it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.16it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.15it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.13it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.13it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.95it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.99it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.01it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.11it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.92it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.94it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.01it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.98it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.62it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.71it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.52it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.77it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.92it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.96it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.00it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.05it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.19it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.95it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.02it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.08it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.06it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.04it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.19it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.11it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.96it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.98it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.92it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.98it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.90it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.97it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.02it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.12it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.07it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.18it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.05it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.07it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.12it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.01it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.08it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.01it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.09it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.02it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.99it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.10it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.11it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.19it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.02it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.97it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.09it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.06it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.21it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.23it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.04it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.14it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.14it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.13it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.16it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.07it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.15it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.14it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.04it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.19it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.15it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.19it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.22it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.01it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.08it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.04it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.06it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.16it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.30it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.30it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.25it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.13it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.13it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.07it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.15it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.04it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.17it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.17it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.12it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.19it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.22it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.23it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.24it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.26it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.20it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.27it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.35it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.36it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.35it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.20it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.05it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.89it/s]
+2022-12-21,11:21:29 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<12:55, 64.41it/s]
  1%|          | 512/50176 [00:06<10:12, 81.13it/s]
  2%|▏         | 768/50176 [00:08<08:58, 91.69it/s]
  2%|▏         | 1024/50176 [00:11<08:49, 92.77it/s]
  3%|▎         | 1280/50176 [00:14<08:48, 92.48it/s]
  3%|▎         | 1536/50176 [00:17<09:01, 89.88it/s]
  4%|▎         | 1792/50176 [00:20<08:52, 90.92it/s]
  4%|▍         | 2048/50176 [00:24<10:45, 74.57it/s]
  5%|▍         | 2304/50176 [00:27<09:59, 79.82it/s]
  5%|▌         | 2560/50176 [00:30<09:08, 86.78it/s]
  6%|▌         | 2816/50176 [00:32<09:03, 87.08it/s]
  6%|▌         | 3072/50176 [00:35<08:39, 90.72it/s]
  7%|▋         | 3328/50176 [00:37<08:18, 93.99it/s]
  7%|▋         | 3584/50176 [00:40<08:01, 96.84it/s]
  8%|▊         | 3840/50176 [00:43<08:08, 94.84it/s]
  8%|▊         | 4096/50176 [00:46<08:22, 91.78it/s]
  9%|▊         | 4352/50176 [00:48<08:02, 94.95it/s]
  9%|▉         | 4608/50176 [00:51<07:52, 96.45it/s]
 10%|▉         | 4864/50176 [00:53<07:45, 97.33it/s]
 10%|█         | 5120/50176 [00:56<07:35, 98.91it/s]
 11%|█         | 5376/50176 [00:58<07:33, 98.68it/s]
 11%|█         | 5632/50176 [01:01<07:41, 96.44it/s]
 12%|█▏        | 5888/50176 [01:04<07:35, 97.28it/s]
 12%|█▏        | 6144/50176 [01:07<07:53, 93.06it/s]
 13%|█▎        | 6400/50176 [01:10<07:47, 93.64it/s]
 13%|█▎        | 6656/50176 [01:12<07:32, 96.10it/s]
 14%|█▍        | 6912/50176 [01:15<07:35, 94.89it/s]
 14%|█▍        | 7168/50176 [01:17<07:27, 96.06it/s]
 15%|█▍        | 7424/50176 [01:20<07:13, 98.72it/s]
 15%|█▌        | 7680/50176 [01:22<07:02, 100.59it/s]
 16%|█▌        | 7936/50176 [01:25<06:48, 103.37it/s]
 16%|█▋        | 8192/50176 [01:27<06:48, 102.73it/s]
 17%|█▋        | 8448/50176 [01:30<06:45, 102.94it/s]
 17%|█▋        | 8704/50176 [01:32<06:45, 102.37it/s]
 18%|█▊        | 8960/50176 [01:35<06:48, 100.92it/s]
 18%|█▊        | 9216/50176 [01:37<06:36, 103.41it/s]
 19%|█▉        | 9472/50176 [01:39<06:30, 104.32it/s]
 19%|█▉        | 9728/50176 [01:42<06:40, 101.09it/s]
 20%|█▉        | 9984/50176 [01:45<06:40, 100.27it/s]
 20%|██        | 10240/50176 [01:47<06:23, 104.03it/s]
 21%|██        | 10496/50176 [01:49<06:20, 104.26it/s]
 21%|██▏       | 10752/50176 [01:52<06:27, 101.86it/s]
 22%|██▏       | 11008/50176 [01:55<06:23, 102.20it/s]
 22%|██▏       | 11264/50176 [01:57<06:20, 102.20it/s]
 23%|██▎       | 11520/50176 [01:59<06:11, 104.10it/s]
 23%|██▎       | 11776/50176 [02:02<06:09, 103.92it/s]
 24%|██▍       | 12032/50176 [02:05<06:13, 102.18it/s]
 24%|██▍       | 12288/50176 [02:07<06:08, 102.76it/s]
 25%|██▌       | 12544/50176 [02:09<06:02, 103.91it/s]
 26%|██▌       | 12800/50176 [02:12<06:02, 103.13it/s]
 26%|██▌       | 13056/50176 [02:14<05:51, 105.56it/s]
 27%|██▋       | 13312/50176 [02:17<05:50, 105.26it/s]
 27%|██▋       | 13568/50176 [02:20<06:07, 99.70it/s] 
 28%|██▊       | 13824/50176 [02:22<06:19, 95.81it/s]
 28%|██▊       | 14080/50176 [02:25<06:15, 96.25it/s]
 29%|██▊       | 14336/50176 [02:28<06:15, 95.44it/s]
 29%|██▉       | 14592/50176 [02:30<06:02, 98.09it/s]
 30%|██▉       | 14848/50176 [02:33<06:18, 93.34it/s]
 30%|███       | 15104/50176 [02:36<06:13, 93.83it/s]
 31%|███       | 15360/50176 [02:38<05:57, 97.39it/s]
 31%|███       | 15616/50176 [02:41<05:45, 100.06it/s]
 32%|███▏      | 15872/50176 [02:44<05:47, 98.78it/s] 
 32%|███▏      | 16128/50176 [02:46<05:43, 99.01it/s]
 33%|███▎      | 16384/50176 [02:48<05:32, 101.69it/s]
 33%|███▎      | 16640/50176 [02:51<05:38, 98.95it/s] 
 34%|███▎      | 16896/50176 [02:54<05:34, 99.47it/s]
 34%|███▍      | 17152/50176 [02:56<05:25, 101.38it/s]
 35%|███▍      | 17408/50176 [02:59<05:26, 100.34it/s]
 35%|███▌      | 17664/50176 [03:01<05:15, 103.07it/s]
 36%|███▌      | 17920/50176 [03:04<05:10, 103.73it/s]
 36%|███▌      | 18176/50176 [03:06<05:05, 104.61it/s]
 37%|███▋      | 18432/50176 [03:08<05:02, 104.84it/s]
 37%|███▋      | 18688/50176 [03:11<05:12, 100.68it/s]
 38%|███▊      | 18944/50176 [03:14<05:11, 100.31it/s]
 38%|███▊      | 19200/50176 [03:17<05:20, 96.70it/s] 
 39%|███▉      | 19456/50176 [03:19<05:22, 95.14it/s]
 39%|███▉      | 19712/50176 [03:22<05:20, 95.00it/s]
 40%|███▉      | 19968/50176 [03:25<05:15, 95.83it/s]
 40%|████      | 20224/50176 [03:27<04:56, 100.98it/s]
 41%|████      | 20480/50176 [03:29<04:54, 100.72it/s]
 41%|████▏     | 20736/50176 [03:32<04:46, 102.79it/s]
 42%|████▏     | 20992/50176 [03:34<04:33, 106.55it/s]
 42%|████▏     | 21248/50176 [03:36<04:34, 105.51it/s]
 43%|████▎     | 21504/50176 [03:39<04:29, 106.44it/s]
 43%|████▎     | 21760/50176 [03:41<04:29, 105.40it/s]
 44%|████▍     | 22016/50176 [03:44<04:22, 107.39it/s]
 44%|████▍     | 22272/50176 [03:46<04:17, 108.38it/s]
 45%|████▍     | 22528/50176 [03:48<04:11, 109.83it/s]
 45%|████▌     | 22784/50176 [03:51<04:20, 105.28it/s]
 46%|████▌     | 23040/50176 [03:53<04:22, 103.26it/s]
 46%|████▋     | 23296/50176 [03:56<04:28, 100.07it/s]
 47%|████▋     | 23552/50176 [03:59<04:32, 97.71it/s] 
 47%|████▋     | 23808/50176 [04:01<04:18, 102.03it/s]
 48%|████▊     | 24064/50176 [04:04<04:14, 102.58it/s]
 48%|████▊     | 24320/50176 [04:06<04:16, 100.81it/s]
 49%|████▉     | 24576/50176 [04:09<04:12, 101.56it/s]
 49%|████▉     | 24832/50176 [04:11<04:06, 103.01it/s]
 50%|█████     | 25088/50176 [04:14<03:59, 104.65it/s]
 51%|█████     | 25344/50176 [04:16<03:55, 105.63it/s]
 51%|█████     | 25600/50176 [04:18<03:52, 105.51it/s]
 52%|█████▏    | 25856/50176 [04:21<03:46, 107.23it/s]
 52%|█████▏    | 26112/50176 [04:23<03:47, 105.74it/s]
 53%|█████▎    | 26368/50176 [04:26<03:47, 104.60it/s]
 53%|█████▎    | 26624/50176 [04:28<03:42, 105.90it/s]
 54%|█████▎    | 26880/50176 [04:30<03:41, 105.19it/s]
 54%|█████▍    | 27136/50176 [04:33<03:38, 105.53it/s]
 55%|█████▍    | 27392/50176 [04:35<03:31, 107.87it/s]
 55%|█████▌    | 27648/50176 [04:37<03:24, 110.21it/s]
 56%|█████▌    | 27904/50176 [04:40<03:32, 104.68it/s]
 56%|█████▌    | 28160/50176 [04:43<03:35, 102.15it/s]
 57%|█████▋    | 28416/50176 [04:45<03:30, 103.14it/s]
 57%|█████▋    | 28672/50176 [04:48<03:32, 101.13it/s]
 58%|█████▊    | 28928/50176 [04:50<03:22, 105.08it/s]
 58%|█████▊    | 29184/50176 [04:52<03:20, 104.62it/s]
 59%|█████▊    | 29440/50176 [04:55<03:29, 98.81it/s] 
 59%|█████▉    | 29696/50176 [04:58<03:28, 98.40it/s]
 60%|█████▉    | 29952/50176 [05:01<03:23, 99.53it/s]
 60%|██████    | 30208/50176 [05:03<03:17, 101.35it/s]
 61%|██████    | 30464/50176 [05:05<03:06, 105.46it/s]
 61%|██████    | 30720/50176 [05:08<03:03, 105.86it/s]
 62%|██████▏   | 30976/50176 [05:10<02:57, 108.10it/s]
 62%|██████▏   | 31232/50176 [05:12<02:54, 108.27it/s]
 63%|██████▎   | 31488/50176 [05:14<02:49, 110.37it/s]
 63%|██████▎   | 31744/50176 [05:17<02:46, 110.55it/s]
 64%|██████▍   | 32000/50176 [05:19<02:47, 108.75it/s]
 64%|██████▍   | 32256/50176 [05:22<02:46, 107.70it/s]
 65%|██████▍   | 32512/50176 [05:24<02:55, 100.85it/s]
 65%|██████▌   | 32768/50176 [05:27<02:50, 102.25it/s]
 66%|██████▌   | 33024/50176 [05:29<02:44, 104.08it/s]
 66%|██████▋   | 33280/50176 [05:31<02:38, 106.62it/s]
 67%|██████▋   | 33536/50176 [05:34<02:38, 104.92it/s]
 67%|██████▋   | 33792/50176 [05:36<02:33, 106.86it/s]
 68%|██████▊   | 34048/50176 [05:39<02:28, 108.25it/s]
 68%|██████▊   | 34304/50176 [05:41<02:21, 111.78it/s]
 69%|██████▉   | 34560/50176 [05:43<02:26, 106.66it/s]
 69%|██████▉   | 34816/50176 [05:46<02:24, 106.08it/s]
 70%|██████▉   | 35072/50176 [05:48<02:22, 105.96it/s]
 70%|███████   | 35328/50176 [05:51<02:19, 106.38it/s]
 71%|███████   | 35584/50176 [05:53<02:14, 108.31it/s]
 71%|███████▏  | 35840/50176 [05:56<02:16, 104.95it/s]
 72%|███████▏  | 36096/50176 [05:58<02:13, 105.86it/s]
 72%|███████▏  | 36352/50176 [06:00<02:10, 105.85it/s]
 73%|███████▎  | 36608/50176 [06:03<02:07, 106.81it/s]
 73%|███████▎  | 36864/50176 [06:05<02:06, 105.57it/s]
 74%|███████▍  | 37120/50176 [06:08<02:02, 106.33it/s]
 74%|███████▍  | 37376/50176 [06:10<01:56, 109.96it/s]
 75%|███████▌  | 37632/50176 [06:12<01:53, 110.93it/s]
 76%|███████▌  | 37888/50176 [06:14<01:51, 110.11it/s]
 76%|███████▌  | 38144/50176 [06:16<01:47, 111.72it/s]
 77%|███████▋  | 38400/50176 [06:19<01:48, 108.68it/s]
 77%|███████▋  | 38656/50176 [06:22<01:48, 106.32it/s]
 78%|███████▊  | 38912/50176 [06:24<01:46, 105.89it/s]
 78%|███████▊  | 39168/50176 [06:26<01:44, 105.48it/s]
 79%|███████▊  | 39424/50176 [06:29<01:39, 108.10it/s]
 79%|███████▉  | 39680/50176 [06:31<01:38, 106.55it/s]
 80%|███████▉  | 39936/50176 [06:33<01:34, 108.42it/s]
 80%|████████  | 40192/50176 [06:36<01:32, 108.52it/s]
 81%|████████  | 40448/50176 [06:38<01:33, 104.51it/s]
 81%|████████  | 40704/50176 [06:41<01:33, 101.60it/s]
 82%|████████▏ | 40960/50176 [06:43<01:27, 105.07it/s]
 82%|████████▏ | 41216/50176 [06:46<01:24, 106.65it/s]
 83%|████████▎ | 41472/50176 [06:48<01:21, 106.94it/s]
 83%|████████▎ | 41728/50176 [06:50<01:17, 109.43it/s]
 84%|████████▎ | 41984/50176 [06:53<01:16, 107.49it/s]
 84%|████████▍ | 42240/50176 [06:55<01:14, 107.06it/s]
 85%|████████▍ | 42496/50176 [06:57<01:09, 110.87it/s]
 85%|████████▌ | 42752/50176 [07:00<01:08, 108.76it/s]
 86%|████████▌ | 43008/50176 [07:02<01:06, 107.47it/s]
 86%|████████▌ | 43264/50176 [07:05<01:04, 106.99it/s]
 87%|████████▋ | 43520/50176 [07:07<01:03, 104.54it/s]
 87%|████████▋ | 43776/50176 [07:10<01:01, 104.26it/s]
 88%|████████▊ | 44032/50176 [07:12<00:58, 105.03it/s]
 88%|████████▊ | 44288/50176 [07:14<00:54, 107.38it/s]
 89%|████████▉ | 44544/50176 [07:17<00:54, 104.03it/s]
 89%|████████▉ | 44800/50176 [07:20<00:52, 102.10it/s]
 90%|████████▉ | 45056/50176 [07:23<00:53, 96.29it/s] 
 90%|█████████ | 45312/50176 [07:25<00:52, 93.30it/s]
 91%|█████████ | 45568/50176 [07:28<00:45, 100.56it/s]
 91%|█████████▏| 45824/50176 [07:30<00:43, 100.34it/s]
 92%|█████████▏| 46080/50176 [07:33<00:40, 101.02it/s]
 92%|█████████▏| 46336/50176 [07:35<00:37, 102.55it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 105.83it/s]
 93%|█████████▎| 46848/50176 [07:40<00:31, 104.80it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.50it/s]
 94%|█████████▍| 47360/50176 [07:45<00:27, 102.09it/s]
 95%|█████████▍| 47616/50176 [07:48<00:25, 101.59it/s]
 95%|█████████▌| 47872/50176 [07:50<00:22, 103.40it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 104.54it/s]
 96%|█████████▋| 48384/50176 [07:55<00:17, 105.05it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 105.87it/s]
 97%|█████████▋| 48896/50176 [07:59<00:12, 105.92it/s]
 98%|█████████▊| 49152/50176 [08:02<00:09, 103.18it/s]
 98%|█████████▊| 49408/50176 [08:05<00:07, 102.32it/s]
 99%|█████████▉| 49664/50176 [08:07<00:05, 101.00it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 95.27it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 124.85it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.11it/s]
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6718	imagenet-zeroshot-val-top5: 0.9057
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_6.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_6.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b696735bd6d5a556afe65057d59418ad34d0c4f8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_6.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3b5af0cd1af7a10e8c1169d00e4fff17597e43c98c854a35bd8a2a8917b0c85
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_6.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_6.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d0a155a504b865c98231c7905897d13f53a7633c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_6.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_6.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_6.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_6.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_6.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_6.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:11 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:11 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:34,  1.57it/s]
  1%|          | 6/1000 [00:00<01:35, 10.40it/s]
  1%|          | 12/1000 [00:00<00:48, 20.37it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.08it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.19it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.84it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.10it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.27it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.59it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.23it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.43it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.36it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.97it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.14it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.68it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.84it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.27it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.25it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.59it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.60it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.68it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.61it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.71it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.73it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.71it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.82it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.82it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.35it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.47it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.48it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.46it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.47it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.45it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.27it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.17it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.31it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.52it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.50it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.40it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.45it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.54it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.64it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.68it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.50it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.52it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.70it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.81it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.01it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.98it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.95it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.10it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.09it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.01it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.81it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.88it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.00it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.10it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.89it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.88it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.97it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.06it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.94it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.63it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.66it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.78it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.94it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.01it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.00it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.01it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.83it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.99it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.03it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.84it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.91it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.99it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.98it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.95it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.08it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.99it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.98it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.99it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.90it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.92it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.91it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.95it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.01it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.98it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.10it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.11it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.01it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.06it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.94it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.89it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.02it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.01it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.92it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.04it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.95it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.94it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.07it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.06it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.07it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.91it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.86it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.02it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 57.99it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.08it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.11it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.03it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.08it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.09it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.11it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.99it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.07it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.04it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.03it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.05it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.01it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.04it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.06it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.06it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.08it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.01it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.11it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.08it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.12it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.15it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.94it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.93it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.99it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.95it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.93it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.96it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.06it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.19it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.23it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.14it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.20it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.19it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.11it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.13it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 57.99it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.08it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.08it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.03it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.05it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.03it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.02it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.11it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.17it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.19it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.19it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.13it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.18it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.26it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.30it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.28it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.24it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.04it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.83it/s]
+2022-12-21,11:21:29 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<12:33, 66.29it/s]
  1%|          | 512/50176 [00:06<10:04, 82.10it/s]
  2%|▏         | 768/50176 [00:08<08:55, 92.28it/s]
  2%|▏         | 1024/50176 [00:11<08:46, 93.33it/s]
  3%|▎         | 1280/50176 [00:14<08:46, 92.79it/s]
  3%|▎         | 1536/50176 [00:17<09:03, 89.47it/s]
  4%|▎         | 1792/50176 [00:20<08:53, 90.64it/s]
  4%|▍         | 2048/50176 [00:24<10:46, 74.49it/s]
  5%|▍         | 2304/50176 [00:27<10:00, 79.70it/s]
  5%|▌         | 2560/50176 [00:29<09:08, 86.73it/s]
  6%|▌         | 2816/50176 [00:32<09:02, 87.22it/s]
  6%|▌         | 3072/50176 [00:35<08:38, 90.91it/s]
  7%|▋         | 3328/50176 [00:37<08:18, 93.96it/s]
  7%|▋         | 3584/50176 [00:40<08:01, 96.73it/s]
  8%|▊         | 3840/50176 [00:43<08:08, 94.84it/s]
  8%|▊         | 4096/50176 [00:46<08:22, 91.74it/s]
  9%|▊         | 4352/50176 [00:48<08:02, 94.97it/s]
  9%|▉         | 4608/50176 [00:51<07:51, 96.59it/s]
 10%|▉         | 4864/50176 [00:53<07:45, 97.34it/s]
 10%|█         | 5120/50176 [00:56<07:36, 98.80it/s]
 11%|█         | 5376/50176 [00:58<07:34, 98.65it/s]
 11%|█         | 5632/50176 [01:01<07:41, 96.46it/s]
 12%|█▏        | 5888/50176 [01:04<07:34, 97.34it/s]
 12%|█▏        | 6144/50176 [01:07<07:53, 93.00it/s]
 13%|█▎        | 6400/50176 [01:09<07:47, 93.61it/s]
 13%|█▎        | 6656/50176 [01:12<07:32, 96.22it/s]
 14%|█▍        | 6912/50176 [01:15<07:36, 94.82it/s]
 14%|█▍        | 7168/50176 [01:17<07:27, 96.08it/s]
 15%|█▍        | 7424/50176 [01:20<07:13, 98.71it/s]
 15%|█▌        | 7680/50176 [01:22<07:02, 100.50it/s]
 16%|█▌        | 7936/50176 [01:25<06:48, 103.48it/s]
 16%|█▋        | 8192/50176 [01:27<06:48, 102.66it/s]
 17%|█▋        | 8448/50176 [01:30<06:44, 103.04it/s]
 17%|█▋        | 8704/50176 [01:32<06:45, 102.37it/s]
 18%|█▊        | 8960/50176 [01:35<06:48, 100.87it/s]
 18%|█▊        | 9216/50176 [01:37<06:35, 103.46it/s]
 19%|█▉        | 9472/50176 [01:39<06:29, 104.38it/s]
 19%|█▉        | 9728/50176 [01:42<06:39, 101.15it/s]
 20%|█▉        | 9984/50176 [01:45<06:41, 100.23it/s]
 20%|██        | 10240/50176 [01:47<06:23, 104.04it/s]
 21%|██        | 10496/50176 [01:49<06:20, 104.20it/s]
 21%|██▏       | 10752/50176 [01:52<06:27, 101.77it/s]
 22%|██▏       | 11008/50176 [01:55<06:22, 102.28it/s]
 22%|██▏       | 11264/50176 [01:57<06:20, 102.20it/s]
 23%|██▎       | 11520/50176 [01:59<06:10, 104.21it/s]
 23%|██▎       | 11776/50176 [02:02<06:09, 103.89it/s]
 24%|██▍       | 12032/50176 [02:05<06:13, 102.13it/s]
 24%|██▍       | 12288/50176 [02:07<06:08, 102.75it/s]
 25%|██▌       | 12544/50176 [02:09<06:02, 103.88it/s]
 26%|██▌       | 12800/50176 [02:12<06:02, 103.10it/s]
 26%|██▌       | 13056/50176 [02:14<05:51, 105.56it/s]
 27%|██▋       | 13312/50176 [02:17<05:50, 105.27it/s]
 27%|██▋       | 13568/50176 [02:20<06:07, 99.59it/s] 
 28%|██▊       | 13824/50176 [02:22<06:18, 95.93it/s]
 28%|██▊       | 14080/50176 [02:25<06:14, 96.28it/s]
 29%|██▊       | 14336/50176 [02:28<06:15, 95.33it/s]
 29%|██▉       | 14592/50176 [02:30<06:03, 97.98it/s]
 30%|██▉       | 14848/50176 [02:33<06:18, 93.41it/s]
 30%|███       | 15104/50176 [02:36<06:13, 93.84it/s]
 31%|███       | 15360/50176 [02:38<05:57, 97.50it/s]
 31%|███       | 15616/50176 [02:41<05:45, 100.15it/s]
 32%|███▏      | 15872/50176 [02:43<05:47, 98.72it/s] 
 32%|███▏      | 16128/50176 [02:46<05:43, 99.05it/s]
 33%|███▎      | 16384/50176 [02:48<05:32, 101.55it/s]
 33%|███▎      | 16640/50176 [02:51<05:38, 98.94it/s] 
 34%|███▎      | 16896/50176 [02:54<05:34, 99.52it/s]
 34%|███▍      | 17152/50176 [02:56<05:25, 101.42it/s]
 35%|███▍      | 17408/50176 [02:59<05:26, 100.26it/s]
 35%|███▌      | 17664/50176 [03:01<05:15, 103.10it/s]
 36%|███▌      | 17920/50176 [03:03<05:11, 103.66it/s]
 36%|███▌      | 18176/50176 [03:06<05:05, 104.61it/s]
 37%|███▋      | 18432/50176 [03:08<05:02, 104.99it/s]
 37%|███▋      | 18688/50176 [03:11<05:12, 100.70it/s]
 38%|███▊      | 18944/50176 [03:14<05:11, 100.27it/s]
 38%|███▊      | 19200/50176 [03:16<05:20, 96.77it/s] 
 39%|███▉      | 19456/50176 [03:19<05:22, 95.13it/s]
 39%|███▉      | 19712/50176 [03:22<05:20, 94.97it/s]
 40%|███▉      | 19968/50176 [03:25<05:15, 95.72it/s]
 40%|████      | 20224/50176 [03:27<04:56, 100.99it/s]
 41%|████      | 20480/50176 [03:29<04:54, 100.74it/s]
 41%|████▏     | 20736/50176 [03:32<04:46, 102.67it/s]
 42%|████▏     | 20992/50176 [03:34<04:34, 106.49it/s]
 42%|████▏     | 21248/50176 [03:36<04:33, 105.60it/s]
 43%|████▎     | 21504/50176 [03:39<04:29, 106.40it/s]
 43%|████▎     | 21760/50176 [03:41<04:29, 105.46it/s]
 44%|████▍     | 22016/50176 [03:44<04:22, 107.47it/s]
 44%|████▍     | 22272/50176 [03:46<04:17, 108.27it/s]
 45%|████▍     | 22528/50176 [03:48<04:11, 109.87it/s]
 45%|████▌     | 22784/50176 [03:51<04:19, 105.40it/s]
 46%|████▌     | 23040/50176 [03:53<04:22, 103.25it/s]
 46%|████▋     | 23296/50176 [03:56<04:28, 100.02it/s]
 47%|████▋     | 23552/50176 [03:59<04:32, 97.76it/s] 
 47%|████▋     | 23808/50176 [04:01<04:18, 102.03it/s]
 48%|████▊     | 24064/50176 [04:04<04:14, 102.52it/s]
 48%|████▊     | 24320/50176 [04:06<04:16, 100.84it/s]
 49%|████▉     | 24576/50176 [04:09<04:11, 101.60it/s]
 49%|████▉     | 24832/50176 [04:11<04:06, 102.99it/s]
 50%|█████     | 25088/50176 [04:13<04:00, 104.52it/s]
 51%|█████     | 25344/50176 [04:16<03:54, 105.67it/s]
 51%|█████     | 25600/50176 [04:18<03:52, 105.50it/s]
 52%|█████▏    | 25856/50176 [04:21<03:46, 107.18it/s]
 52%|█████▏    | 26112/50176 [04:23<03:47, 105.79it/s]
 53%|█████▎    | 26368/50176 [04:26<03:47, 104.44it/s]
 53%|█████▎    | 26624/50176 [04:28<03:42, 106.09it/s]
 54%|█████▎    | 26880/50176 [04:30<03:41, 105.14it/s]
 54%|█████▍    | 27136/50176 [04:33<03:38, 105.48it/s]
 55%|█████▍    | 27392/50176 [04:35<03:31, 107.84it/s]
 55%|█████▌    | 27648/50176 [04:37<03:24, 110.21it/s]
 56%|█████▌    | 27904/50176 [04:40<03:32, 104.73it/s]
 56%|█████▌    | 28160/50176 [04:43<03:35, 102.25it/s]
 57%|█████▋    | 28416/50176 [04:45<03:30, 103.21it/s]
 57%|█████▋    | 28672/50176 [04:48<03:32, 101.11it/s]
 58%|█████▊    | 28928/50176 [04:50<03:22, 105.14it/s]
 58%|█████▊    | 29184/50176 [04:52<03:20, 104.52it/s]
 59%|█████▊    | 29440/50176 [04:55<03:29, 98.85it/s] 
 59%|█████▉    | 29696/50176 [04:58<03:28, 98.39it/s]
 60%|█████▉    | 29952/50176 [05:00<03:23, 99.50it/s]
 60%|██████    | 30208/50176 [05:03<03:17, 101.29it/s]
 61%|██████    | 30464/50176 [05:05<03:06, 105.48it/s]
 61%|██████    | 30720/50176 [05:07<03:03, 106.06it/s]
 62%|██████▏   | 30976/50176 [05:10<02:57, 108.04it/s]
 62%|██████▏   | 31232/50176 [05:12<02:54, 108.31it/s]
 63%|██████▎   | 31488/50176 [05:14<02:49, 110.48it/s]
 63%|██████▎   | 31744/50176 [05:17<02:46, 110.39it/s]
 64%|██████▍   | 32000/50176 [05:19<02:47, 108.61it/s]
 64%|██████▍   | 32256/50176 [05:21<02:46, 107.58it/s]
 65%|██████▍   | 32512/50176 [05:24<02:54, 100.95it/s]
 65%|██████▌   | 32768/50176 [05:27<02:49, 102.41it/s]
 66%|██████▌   | 33024/50176 [05:29<02:44, 104.00it/s]
 66%|██████▋   | 33280/50176 [05:31<02:38, 106.59it/s]
 67%|██████▋   | 33536/50176 [05:34<02:38, 104.81it/s]
 67%|██████▋   | 33792/50176 [05:36<02:33, 107.00it/s]
 68%|██████▊   | 34048/50176 [05:39<02:28, 108.36it/s]
 68%|██████▊   | 34304/50176 [05:41<02:22, 111.71it/s]
 69%|██████▉   | 34560/50176 [05:43<02:26, 106.73it/s]
 69%|██████▉   | 34816/50176 [05:46<02:24, 106.07it/s]
 70%|██████▉   | 35072/50176 [05:48<02:22, 105.98it/s]
 70%|███████   | 35328/50176 [05:51<02:19, 106.34it/s]
 71%|███████   | 35584/50176 [05:53<02:14, 108.39it/s]
 71%|███████▏  | 35840/50176 [05:55<02:16, 104.93it/s]
 72%|███████▏  | 36096/50176 [05:58<02:13, 105.82it/s]
 72%|███████▏  | 36352/50176 [06:00<02:10, 105.81it/s]
 73%|███████▎  | 36608/50176 [06:03<02:07, 106.74it/s]
 73%|███████▎  | 36864/50176 [06:05<02:06, 105.57it/s]
 74%|███████▍  | 37120/50176 [06:07<02:02, 106.26it/s]
 74%|███████▍  | 37376/50176 [06:10<01:56, 110.04it/s]
 75%|███████▌  | 37632/50176 [06:12<01:52, 111.05it/s]
 76%|███████▌  | 37888/50176 [06:14<01:51, 110.17it/s]
 76%|███████▌  | 38144/50176 [06:16<01:47, 111.68it/s]
 77%|███████▋  | 38400/50176 [06:19<01:48, 108.77it/s]
 77%|███████▋  | 38656/50176 [06:21<01:48, 106.26it/s]
 78%|███████▊  | 38912/50176 [06:24<01:46, 105.89it/s]
 78%|███████▊  | 39168/50176 [06:26<01:44, 105.52it/s]
 79%|███████▊  | 39424/50176 [06:29<01:39, 108.17it/s]
 79%|███████▉  | 39680/50176 [06:31<01:38, 106.34it/s]
 80%|███████▉  | 39936/50176 [06:33<01:34, 108.40it/s]
 80%|████████  | 40192/50176 [06:36<01:32, 108.45it/s]
 81%|████████  | 40448/50176 [06:38<01:33, 104.56it/s]
 81%|████████  | 40704/50176 [06:41<01:33, 101.58it/s]
 82%|████████▏ | 40960/50176 [06:43<01:27, 105.12it/s]
 82%|████████▏ | 41216/50176 [06:46<01:24, 106.55it/s]
 83%|████████▎ | 41472/50176 [06:48<01:21, 107.04it/s]
 83%|████████▎ | 41728/50176 [06:50<01:17, 109.45it/s]
 84%|████████▎ | 41984/50176 [06:53<01:16, 107.35it/s]
 84%|████████▍ | 42240/50176 [06:55<01:14, 107.06it/s]
 85%|████████▍ | 42496/50176 [06:57<01:09, 110.92it/s]
 85%|████████▌ | 42752/50176 [07:00<01:08, 108.99it/s]
 86%|████████▌ | 43008/50176 [07:02<01:06, 107.38it/s]
 86%|████████▌ | 43264/50176 [07:05<01:04, 107.04it/s]
 87%|████████▋ | 43520/50176 [07:07<01:03, 104.56it/s]
 87%|████████▋ | 43776/50176 [07:10<01:01, 104.19it/s]
 88%|████████▊ | 44032/50176 [07:12<00:58, 105.11it/s]
 88%|████████▊ | 44288/50176 [07:14<00:54, 107.26it/s]
 89%|████████▉ | 44544/50176 [07:17<00:54, 103.89it/s]
 89%|████████▉ | 44800/50176 [07:19<00:52, 102.01it/s]
 90%|████████▉ | 45056/50176 [07:22<00:53, 96.30it/s] 
 90%|█████████ | 45312/50176 [07:25<00:52, 93.37it/s]
 91%|█████████ | 45568/50176 [07:28<00:45, 100.60it/s]
 91%|█████████▏| 45824/50176 [07:30<00:43, 100.38it/s]
 92%|█████████▏| 46080/50176 [07:33<00:40, 100.94it/s]
 92%|█████████▏| 46336/50176 [07:35<00:37, 102.56it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 105.74it/s]
 93%|█████████▎| 46848/50176 [07:40<00:31, 104.86it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.60it/s]
 94%|█████████▍| 47360/50176 [07:45<00:27, 101.97it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 101.69it/s]
 95%|█████████▌| 47872/50176 [07:50<00:22, 103.42it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 104.52it/s]
 96%|█████████▋| 48384/50176 [07:55<00:17, 105.06it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 105.78it/s]
 97%|█████████▋| 48896/50176 [07:59<00:12, 105.89it/s]
 98%|█████████▊| 49152/50176 [08:02<00:09, 103.11it/s]
 98%|█████████▊| 49408/50176 [08:05<00:07, 102.42it/s]
 99%|█████████▉| 49664/50176 [08:07<00:05, 101.04it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 95.33it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 125.09it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.13it/s]
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4871	imagenet-zeroshot-val-top5: 0.7690
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_60.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_60.pt
new file mode 100644
index 0000000000000000000000000000000000000000..0cef4d2750936c1a85008287db724c8cb6a3022a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_60.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:96d7ccd1d84b99e06de21667916efa7ec0cb867601959a0e3144650c79c76cc8
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_60.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_60.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0369269d3b972753f5de30e6bea1daae01c784d3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_60.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_60.pt).
+2022-12-21,11:30:06 | INFO | Model:
+2022-12-21,11:30:06 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:06 | INFO | Params:
+2022-12-21,11:30:06 | INFO |   batch_size: 256
+2022-12-21,11:30:06 | INFO |   beta1: 0.9
+2022-12-21,11:30:06 | INFO |   beta2: 0.98
+2022-12-21,11:30:06 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_60.pt/checkpoints
+2022-12-21,11:30:06 | INFO |   copy_codebase: False
+2022-12-21,11:30:06 | INFO |   csv_caption_key: title
+2022-12-21,11:30:06 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:06 | INFO |   csv_separator: 	
+2022-12-21,11:30:06 | INFO |   dataset_resampled: False
+2022-12-21,11:30:06 | INFO |   dataset_type: auto
+2022-12-21,11:30:06 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:06 | INFO |   debug: False
+2022-12-21,11:30:06 | INFO |   device: cuda:0
+2022-12-21,11:30:06 | INFO |   dist_backend: nccl
+2022-12-21,11:30:06 | INFO |   dist_url: env://
+2022-12-21,11:30:06 | INFO |   distributed: False
+2022-12-21,11:30:06 | INFO |   epochs: 32
+2022-12-21,11:30:06 | INFO |   eps: 1e-06
+2022-12-21,11:30:06 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:06 | INFO |   gather_with_grad: False
+2022-12-21,11:30:06 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:06 | INFO |   horovod: False
+2022-12-21,11:30:06 | INFO |   image_mean: None
+2022-12-21,11:30:06 | INFO |   image_std: None
+2022-12-21,11:30:06 | INFO |   imagenet_v2: None
+2022-12-21,11:30:06 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:06 | INFO |   local_loss: False
+2022-12-21,11:30:06 | INFO |   local_rank: 0
+2022-12-21,11:30:06 | INFO |   lock_image: False
+2022-12-21,11:30:06 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:06 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:06 | INFO |   log_level: 20
+2022-12-21,11:30:06 | INFO |   log_local: False
+2022-12-21,11:30:06 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_60.pt/out.log
+2022-12-21,11:30:06 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:06 | INFO |   lr: 0.0005
+2022-12-21,11:30:06 | INFO |   model: ViT-B-16
+2022-12-21,11:30:06 | INFO |   name: epoch_60.pt
+2022-12-21,11:30:06 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:06 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:06 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:06 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_60.pt
+2022-12-21,11:30:06 | INFO |   pretrained_image: False
+2022-12-21,11:30:06 | INFO |   rank: 0
+2022-12-21,11:30:06 | INFO |   report_to: 
+2022-12-21,11:30:06 | INFO |   resume: None
+2022-12-21,11:30:06 | INFO |   save_frequency: 1
+2022-12-21,11:30:06 | INFO |   save_most_recent: False
+2022-12-21,11:30:06 | INFO |   seed: 0
+2022-12-21,11:30:06 | INFO |   skip_scheduler: False
+2022-12-21,11:30:06 | INFO |   tensorboard: False
+2022-12-21,11:30:06 | INFO |   tensorboard_path: 
+2022-12-21,11:30:06 | INFO |   torchscript: False
+2022-12-21,11:30:06 | INFO |   trace: False
+2022-12-21,11:30:06 | INFO |   train_data: None
+2022-12-21,11:30:06 | INFO |   train_num_samples: None
+2022-12-21,11:30:06 | INFO |   use_bn_sync: False
+2022-12-21,11:30:06 | INFO |   val_data: None
+2022-12-21,11:30:06 | INFO |   val_frequency: 1
+2022-12-21,11:30:06 | INFO |   val_num_samples: None
+2022-12-21,11:30:06 | INFO |   wandb: False
+2022-12-21,11:30:06 | INFO |   wandb_notes: 
+2022-12-21,11:30:06 | INFO |   warmup: 10000
+2022-12-21,11:30:06 | INFO |   wd: 0.2
+2022-12-21,11:30:06 | INFO |   workers: 1
+2022-12-21,11:30:06 | INFO |   world_size: 1
+2022-12-21,11:30:06 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:06 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:06 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:33,  1.44it/s]
  1%|          | 6/1000 [00:00<01:42,  9.71it/s]
  1%|          | 12/1000 [00:00<00:51, 19.13it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.34it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.56it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.43it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.97it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.41it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.98it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.83it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.19it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.00it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.72it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.23it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.78it/s]
  9%|▉         | 90/1000 [00:02<00:16, 55.58it/s]
 10%|▉         | 96/1000 [00:02<00:16, 56.18it/s]
 10%|█         | 102/1000 [00:02<00:15, 56.62it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.02it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.16it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.05it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 56.86it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 56.88it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.03it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.31it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.54it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.63it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.56it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.61it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.57it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.56it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.65it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.33it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.34it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.54it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.51it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.25it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.00it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.34it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.67it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.89it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.80it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.86it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.95it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.97it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.14it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.08it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.03it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.10it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.02it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.81it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.89it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.97it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.05it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.84it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.86it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.93it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.03it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.90it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.59it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.68it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.74it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.91it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.99it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.01it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.01it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.04it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.15it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.13it/s]
 44%|████▍     | 444/1000 [00:08<00:10, 52.25it/s]
 45%|████▌     | 450/1000 [00:08<00:10, 53.87it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 55.07it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 55.90it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 56.47it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 56.85it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.09it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.32it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.50it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.54it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.68it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.68it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.77it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.82it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.91it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.03it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.04it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.94it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.93it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.00it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.86it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.78it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.94it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.88it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.00it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.93it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.05it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.11it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.93it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.87it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.00it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.96it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.04it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.06it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.02it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.08it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.09it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.10it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.99it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.08it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.06it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.02it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.05it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.97it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.01it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.01it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.08it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.13it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.98it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.10it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.06it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.08it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.12it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.93it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.93it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.97it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.92it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.89it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.95it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.05it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.16it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.18it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.11it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.14it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.07it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.08it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.09it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.08it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.96it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.84it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.88it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.91it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.93it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.67it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.69it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.79it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.80it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.87it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.90it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.89it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.88it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.97it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.05it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.08it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.12it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.96it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.98it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.44it/s]
+2022-12-21,11:30:24 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:18, 73.52it/s]
  1%|          | 512/50176 [00:06<10:01, 82.59it/s]
  2%|▏         | 768/50176 [00:08<08:49, 93.27it/s]
  2%|▏         | 1024/50176 [00:11<08:36, 95.25it/s]
  3%|▎         | 1280/50176 [00:13<08:30, 95.78it/s]
  3%|▎         | 1536/50176 [00:17<09:02, 89.71it/s]
  4%|▎         | 1792/50176 [00:19<08:44, 92.26it/s]
  4%|▍         | 2048/50176 [00:22<08:23, 95.52it/s]
  5%|▍         | 2304/50176 [00:24<08:15, 96.59it/s]
  5%|▌         | 2560/50176 [00:27<07:53, 100.51it/s]
  6%|▌         | 2816/50176 [00:29<07:59, 98.81it/s] 
  6%|▌         | 3072/50176 [00:32<07:51, 99.99it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.34it/s]
  7%|▋         | 3584/50176 [00:37<07:45, 100.08it/s]
  8%|▊         | 3840/50176 [00:40<07:56, 97.27it/s] 
  8%|▊         | 4096/50176 [00:43<08:11, 93.79it/s]
  9%|▊         | 4352/50176 [00:45<07:58, 95.76it/s]
  9%|▉         | 4608/50176 [00:48<07:48, 97.26it/s]
 10%|▉         | 4864/50176 [00:50<07:41, 98.14it/s]
 10%|█         | 5120/50176 [00:53<07:33, 99.29it/s]
 11%|█         | 5376/50176 [00:56<07:37, 97.97it/s]
 11%|█         | 5632/50176 [00:58<07:36, 97.59it/s]
 12%|█▏        | 5888/50176 [01:01<07:34, 97.39it/s]
 12%|█▏        | 6144/50176 [01:04<07:59, 91.73it/s]
 13%|█▎        | 6400/50176 [01:07<07:46, 93.93it/s]
 13%|█▎        | 6656/50176 [01:09<07:32, 96.16it/s]
 14%|█▍        | 6912/50176 [01:12<07:36, 94.67it/s]
 14%|█▍        | 7168/50176 [01:14<07:25, 96.57it/s]
 15%|█▍        | 7424/50176 [01:17<07:12, 98.77it/s]
 15%|█▌        | 7680/50176 [01:19<07:06, 99.57it/s]
 16%|█▌        | 7936/50176 [01:22<06:44, 104.34it/s]
 16%|█▋        | 8192/50176 [01:24<06:46, 103.32it/s]
 17%|█▋        | 8448/50176 [01:27<06:41, 103.93it/s]
 17%|█▋        | 8704/50176 [01:29<06:38, 104.17it/s]
 18%|█▊        | 8960/50176 [01:32<06:45, 101.62it/s]
 18%|█▊        | 9216/50176 [01:34<06:38, 102.87it/s]
 19%|█▉        | 9472/50176 [01:36<06:31, 103.99it/s]
 19%|█▉        | 9728/50176 [01:39<06:41, 100.85it/s]
 20%|█▉        | 9984/50176 [01:42<06:37, 101.00it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.68it/s]
 21%|██        | 10496/50176 [01:46<06:24, 103.15it/s]
 21%|██▏       | 10752/50176 [01:49<06:31, 100.67it/s]
 22%|██▏       | 11008/50176 [01:52<06:28, 100.86it/s]
 22%|██▏       | 11264/50176 [01:54<06:23, 101.52it/s]
 23%|██▎       | 11520/50176 [01:57<06:20, 101.62it/s]
 23%|██▎       | 11776/50176 [01:59<06:18, 101.41it/s]
 24%|██▍       | 12032/50176 [02:02<06:17, 100.98it/s]
 24%|██▍       | 12288/50176 [02:04<06:13, 101.37it/s]
 25%|██▌       | 12544/50176 [02:07<06:12, 100.98it/s]
 26%|██▌       | 12800/50176 [02:09<06:10, 100.94it/s]
 26%|██▌       | 13056/50176 [02:12<06:05, 101.42it/s]
 27%|██▋       | 13312/50176 [02:14<06:03, 101.52it/s]
 27%|██▋       | 13568/50176 [02:17<06:13, 97.98it/s] 
 28%|██▊       | 13824/50176 [02:20<06:19, 95.88it/s]
 28%|██▊       | 14080/50176 [02:23<06:16, 95.84it/s]
 29%|██▊       | 14336/50176 [02:25<06:16, 95.08it/s]
 29%|██▉       | 14592/50176 [02:28<06:11, 95.79it/s]
 30%|██▉       | 14848/50176 [02:31<06:16, 93.82it/s]
 30%|███       | 15104/50176 [02:34<06:13, 93.82it/s]
 31%|███       | 15360/50176 [02:36<06:00, 96.64it/s]
 31%|███       | 15616/50176 [02:38<05:43, 100.57it/s]
 32%|███▏      | 15872/50176 [02:41<05:48, 98.56it/s] 
 32%|███▏      | 16128/50176 [02:44<05:47, 97.92it/s]
 33%|███▎      | 16384/50176 [02:46<05:36, 100.32it/s]
 33%|███▎      | 16640/50176 [02:49<05:46, 96.68it/s] 
 34%|███▎      | 16896/50176 [02:51<05:33, 99.87it/s]
 34%|███▍      | 17152/50176 [02:54<05:27, 100.87it/s]
 35%|███▍      | 17408/50176 [02:56<05:19, 102.60it/s]
 35%|███▌      | 17664/50176 [02:59<05:15, 103.04it/s]
 36%|███▌      | 17920/50176 [03:01<05:12, 103.12it/s]
 36%|███▌      | 18176/50176 [03:04<05:09, 103.26it/s]
 37%|███▋      | 18432/50176 [03:06<05:02, 104.83it/s]
 37%|███▋      | 18688/50176 [03:09<05:10, 101.34it/s]
 38%|███▊      | 18944/50176 [03:11<05:11, 100.33it/s]
 38%|███▊      | 19200/50176 [03:14<05:16, 97.94it/s] 
 39%|███▉      | 19456/50176 [03:17<05:20, 96.00it/s]
 39%|███▉      | 19712/50176 [03:20<05:18, 95.65it/s]
 40%|███▉      | 19968/50176 [03:22<05:13, 96.43it/s]
 40%|████      | 20224/50176 [03:25<05:02, 99.15it/s]
 41%|████      | 20480/50176 [03:27<04:55, 100.58it/s]
 41%|████▏     | 20736/50176 [03:29<04:45, 103.08it/s]
 42%|████▏     | 20992/50176 [03:32<04:35, 105.84it/s]
 42%|████▏     | 21248/50176 [03:34<04:42, 102.40it/s]
 43%|████▎     | 21504/50176 [03:37<04:37, 103.14it/s]
 43%|████▎     | 21760/50176 [03:39<04:35, 103.11it/s]
 44%|████▍     | 22016/50176 [03:42<04:29, 104.65it/s]
 44%|████▍     | 22272/50176 [03:44<04:20, 106.96it/s]
 45%|████▍     | 22528/50176 [03:46<04:16, 107.71it/s]
 45%|████▌     | 22784/50176 [03:49<04:24, 103.75it/s]
 46%|████▌     | 23040/50176 [03:52<04:28, 101.24it/s]
 46%|████▋     | 23296/50176 [03:54<04:30, 99.40it/s] 
 47%|████▋     | 23552/50176 [03:57<04:33, 97.50it/s]
 47%|████▋     | 23808/50176 [03:59<04:21, 100.95it/s]
 48%|████▊     | 24064/50176 [04:02<04:15, 102.06it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.35it/s]
 49%|████▉     | 24576/50176 [04:07<04:14, 100.71it/s]
 49%|████▉     | 24832/50176 [04:09<04:01, 104.88it/s]
 50%|█████     | 25088/50176 [04:12<03:55, 106.43it/s]
 51%|█████     | 25344/50176 [04:14<03:50, 107.66it/s]
 51%|█████     | 25600/50176 [04:16<03:54, 104.72it/s]
 52%|█████▏    | 25856/50176 [04:19<03:47, 106.82it/s]
 52%|█████▏    | 26112/50176 [04:21<03:41, 108.57it/s]
 53%|█████▎    | 26368/50176 [04:24<03:44, 106.05it/s]
 53%|█████▎    | 26624/50176 [04:26<03:43, 105.34it/s]
 54%|█████▎    | 26880/50176 [04:29<03:43, 104.04it/s]
 54%|█████▍    | 27136/50176 [04:31<03:42, 103.76it/s]
 55%|█████▍    | 27392/50176 [04:33<03:37, 104.85it/s]
 55%|█████▌    | 27648/50176 [04:36<03:27, 108.45it/s]
 56%|█████▌    | 27904/50176 [04:38<03:35, 103.13it/s]
 56%|█████▌    | 28160/50176 [04:41<03:39, 100.47it/s]
 57%|█████▋    | 28416/50176 [04:44<03:34, 101.29it/s]
 57%|█████▋    | 28672/50176 [04:46<03:32, 101.28it/s]
 58%|█████▊    | 28928/50176 [04:48<03:21, 105.37it/s]
 58%|█████▊    | 29184/50176 [04:51<03:22, 103.76it/s]
 59%|█████▊    | 29440/50176 [04:54<03:29, 99.10it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:25, 99.68it/s]
 60%|█████▉    | 29952/50176 [04:59<03:22, 99.68it/s]
 60%|██████    | 30208/50176 [05:01<03:17, 101.12it/s]
 61%|██████    | 30464/50176 [05:03<03:03, 107.22it/s]
 61%|██████    | 30720/50176 [05:06<03:04, 105.59it/s]
 62%|██████▏   | 30976/50176 [05:08<02:57, 108.14it/s]
 62%|██████▏   | 31232/50176 [05:10<02:53, 109.01it/s]
 63%|██████▎   | 31488/50176 [05:13<02:49, 110.12it/s]
 63%|██████▎   | 31744/50176 [05:15<02:49, 108.46it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.65it/s]
 64%|██████▍   | 32256/50176 [05:20<02:43, 109.33it/s]
 65%|██████▍   | 32512/50176 [05:22<02:49, 103.92it/s]
 65%|██████▌   | 32768/50176 [05:25<02:44, 106.02it/s]
 66%|██████▌   | 33024/50176 [05:27<02:41, 105.89it/s]
 66%|██████▋   | 33280/50176 [05:29<02:36, 108.20it/s]
 67%|██████▋   | 33536/50176 [05:32<02:39, 104.53it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.67it/s]
 68%|██████▊   | 34048/50176 [05:37<02:27, 109.19it/s]
 68%|██████▊   | 34304/50176 [05:39<02:22, 111.72it/s]
 69%|██████▉   | 34560/50176 [05:41<02:25, 107.45it/s]
 69%|██████▉   | 34816/50176 [05:44<02:25, 105.59it/s]
 70%|██████▉   | 35072/50176 [05:46<02:22, 105.85it/s]
 70%|███████   | 35328/50176 [05:49<02:18, 107.44it/s]
 71%|███████   | 35584/50176 [05:51<02:15, 108.08it/s]
 71%|███████▏  | 35840/50176 [05:54<02:17, 104.53it/s]
 72%|███████▏  | 36096/50176 [05:56<02:11, 107.07it/s]
 72%|███████▏  | 36352/50176 [05:58<02:11, 105.35it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 105.00it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.79it/s]
 74%|███████▍  | 37120/50176 [06:06<02:03, 105.66it/s]
 74%|███████▍  | 37376/50176 [06:08<01:56, 109.75it/s]
 75%|███████▌  | 37632/50176 [06:10<01:53, 110.15it/s]
 76%|███████▌  | 37888/50176 [06:13<01:54, 107.44it/s]
 76%|███████▌  | 38144/50176 [06:15<01:51, 108.32it/s]
 77%|███████▋  | 38400/50176 [06:17<01:50, 106.95it/s]
 77%|███████▋  | 38656/50176 [06:20<01:48, 105.91it/s]
 78%|███████▊  | 38912/50176 [06:22<01:48, 104.21it/s]
 78%|███████▊  | 39168/50176 [06:25<01:47, 102.11it/s]
 79%|███████▊  | 39424/50176 [06:27<01:42, 105.11it/s]
 79%|███████▉  | 39680/50176 [06:30<01:40, 104.94it/s]
 80%|███████▉  | 39936/50176 [06:32<01:36, 106.54it/s]
 80%|████████  | 40192/50176 [06:34<01:33, 106.45it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 103.65it/s]
 81%|████████  | 40704/50176 [06:40<01:34, 100.36it/s]
 82%|████████▏ | 40960/50176 [06:42<01:29, 103.10it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 105.10it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.32it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 105.72it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 107.50it/s]
 84%|████████▍ | 42240/50176 [06:54<01:14, 106.78it/s]
 85%|████████▍ | 42496/50176 [06:56<01:12, 106.18it/s]
 85%|████████▌ | 42752/50176 [06:59<01:09, 107.18it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.92it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 106.64it/s]
 87%|████████▋ | 43520/50176 [07:06<01:03, 104.34it/s]
 87%|████████▋ | 43776/50176 [07:09<01:00, 105.39it/s]
 88%|████████▊ | 44032/50176 [07:11<00:57, 106.22it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 108.41it/s]
 89%|████████▉ | 44544/50176 [07:16<00:53, 104.77it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.09it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 106.27it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.41it/s]
 91%|█████████ | 45568/50176 [07:25<00:43, 106.43it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 106.12it/s]
 92%|█████████▏| 46080/50176 [07:30<00:38, 105.18it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.29it/s]
 93%|█████████▎| 46592/50176 [07:35<00:33, 105.88it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.99it/s]
 94%|█████████▍| 47104/50176 [07:40<00:29, 103.08it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.52it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 99.53it/s] 
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.60it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 103.37it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 104.04it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.04it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 104.42it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.50it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 103.23it/s]
 99%|█████████▉| 49664/50176 [08:05<00:05, 101.16it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 95.18it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.06it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.50it/s]
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6758	imagenet-zeroshot-val-top5: 0.9062
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_61.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_61.pt
new file mode 100644
index 0000000000000000000000000000000000000000..c845599b3a90004c1963bd874f70d7aed27894b3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_61.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f5ceb87d9c287278f7148b870ba03822205e8574524fb66fc21815f18897e9d2
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_61.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_61.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d95ab541713c60fb9c34a0f5482c3ab3bcd3fd8e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_61.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_61.pt).
+2022-12-21,11:30:06 | INFO | Model:
+2022-12-21,11:30:06 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:06 | INFO | Params:
+2022-12-21,11:30:06 | INFO |   batch_size: 256
+2022-12-21,11:30:06 | INFO |   beta1: 0.9
+2022-12-21,11:30:06 | INFO |   beta2: 0.98
+2022-12-21,11:30:06 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_61.pt/checkpoints
+2022-12-21,11:30:06 | INFO |   copy_codebase: False
+2022-12-21,11:30:06 | INFO |   csv_caption_key: title
+2022-12-21,11:30:06 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:06 | INFO |   csv_separator: 	
+2022-12-21,11:30:06 | INFO |   dataset_resampled: False
+2022-12-21,11:30:06 | INFO |   dataset_type: auto
+2022-12-21,11:30:06 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:06 | INFO |   debug: False
+2022-12-21,11:30:06 | INFO |   device: cuda:0
+2022-12-21,11:30:06 | INFO |   dist_backend: nccl
+2022-12-21,11:30:06 | INFO |   dist_url: env://
+2022-12-21,11:30:06 | INFO |   distributed: False
+2022-12-21,11:30:06 | INFO |   epochs: 32
+2022-12-21,11:30:06 | INFO |   eps: 1e-06
+2022-12-21,11:30:06 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:06 | INFO |   gather_with_grad: False
+2022-12-21,11:30:06 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:06 | INFO |   horovod: False
+2022-12-21,11:30:06 | INFO |   image_mean: None
+2022-12-21,11:30:06 | INFO |   image_std: None
+2022-12-21,11:30:06 | INFO |   imagenet_v2: None
+2022-12-21,11:30:06 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:06 | INFO |   local_loss: False
+2022-12-21,11:30:06 | INFO |   local_rank: 0
+2022-12-21,11:30:06 | INFO |   lock_image: False
+2022-12-21,11:30:06 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:06 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:06 | INFO |   log_level: 20
+2022-12-21,11:30:06 | INFO |   log_local: False
+2022-12-21,11:30:06 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_61.pt/out.log
+2022-12-21,11:30:06 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:06 | INFO |   lr: 0.0005
+2022-12-21,11:30:06 | INFO |   model: ViT-B-16
+2022-12-21,11:30:06 | INFO |   name: epoch_61.pt
+2022-12-21,11:30:06 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:06 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:06 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:06 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_61.pt
+2022-12-21,11:30:06 | INFO |   pretrained_image: False
+2022-12-21,11:30:06 | INFO |   rank: 0
+2022-12-21,11:30:06 | INFO |   report_to: 
+2022-12-21,11:30:06 | INFO |   resume: None
+2022-12-21,11:30:06 | INFO |   save_frequency: 1
+2022-12-21,11:30:06 | INFO |   save_most_recent: False
+2022-12-21,11:30:06 | INFO |   seed: 0
+2022-12-21,11:30:06 | INFO |   skip_scheduler: False
+2022-12-21,11:30:06 | INFO |   tensorboard: False
+2022-12-21,11:30:06 | INFO |   tensorboard_path: 
+2022-12-21,11:30:06 | INFO |   torchscript: False
+2022-12-21,11:30:06 | INFO |   trace: False
+2022-12-21,11:30:06 | INFO |   train_data: None
+2022-12-21,11:30:06 | INFO |   train_num_samples: None
+2022-12-21,11:30:06 | INFO |   use_bn_sync: False
+2022-12-21,11:30:06 | INFO |   val_data: None
+2022-12-21,11:30:06 | INFO |   val_frequency: 1
+2022-12-21,11:30:06 | INFO |   val_num_samples: None
+2022-12-21,11:30:06 | INFO |   wandb: False
+2022-12-21,11:30:06 | INFO |   wandb_notes: 
+2022-12-21,11:30:06 | INFO |   warmup: 10000
+2022-12-21,11:30:06 | INFO |   wd: 0.2
+2022-12-21,11:30:06 | INFO |   workers: 1
+2022-12-21,11:30:06 | INFO |   world_size: 1
+2022-12-21,11:30:06 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:06 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:06 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:46,  1.55it/s]
  1%|          | 6/1000 [00:00<01:36, 10.30it/s]
  1%|          | 12/1000 [00:00<00:48, 20.19it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.76it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.81it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.46it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.76it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.99it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.36it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.10it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.33it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.27it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.89it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.33it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.78it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.99it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.31it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.50it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.80it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.83it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.79it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.84it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.91it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.94it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.76it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.53it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.56it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.55it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.47it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.45it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.40it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.21it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.13it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.24it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.21it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.43it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.43it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.38it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.45it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.56it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.51it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.34it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.34it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.68it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.90it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.89it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.86it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.01it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.99it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.90it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.73it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.82it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.93it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.01it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.81it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.81it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.87it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.95it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.81it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.50it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.56it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.59it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.73it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.82it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.80it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.82it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.86it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.98it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.98it/s]
 44%|████▍     | 444/1000 [00:08<00:10, 52.72it/s]
 45%|████▌     | 450/1000 [00:08<00:10, 54.19it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 55.26it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 55.99it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 56.50it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.02it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.20it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.34it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.48it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.49it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.61it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.60it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.71it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.78it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.91it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.86it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.97it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.97it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.88it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.89it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.73it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.64it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.64it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.81it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.79it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.73it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.84it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.76it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.75it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.89it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.89it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.98it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.79it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.75it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.92it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.89it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.95it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.01it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.94it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.98it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.98it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.01it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.90it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.98it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.95it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.93it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.97it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.95it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.98it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.96it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.04it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.12it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.94it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.03it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.99it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.03it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.83it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.66it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.69it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.78it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.74it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.75it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.80it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.92it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.07it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.10it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.02it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.08it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.04it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.05it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.95it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.01it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.00it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.87it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.99it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.99it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.94it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.97it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.59it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.64it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.80it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.89it/s]
 94%|█████████▍| 942/1000 [00:16<00:01, 57.90it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.93it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.95it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.92it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.00it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.04it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.09it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.10it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.06it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.58it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.59it/s]
+2022-12-21,11:30:24 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:31, 72.16it/s]
  1%|          | 512/50176 [00:06<10:03, 82.29it/s]
  2%|▏         | 768/50176 [00:08<08:50, 93.08it/s]
  2%|▏         | 1024/50176 [00:11<08:40, 94.47it/s]
  3%|▎         | 1280/50176 [00:13<08:32, 95.49it/s]
  3%|▎         | 1536/50176 [00:17<09:04, 89.34it/s]
  4%|▎         | 1792/50176 [00:19<08:44, 92.20it/s]
  4%|▍         | 2048/50176 [00:22<08:25, 95.17it/s]
  5%|▍         | 2304/50176 [00:24<08:16, 96.40it/s]
  5%|▌         | 2560/50176 [00:27<07:53, 100.53it/s]
  6%|▌         | 2816/50176 [00:29<07:58, 98.88it/s] 
  6%|▌         | 3072/50176 [00:32<07:51, 99.87it/s]
  7%|▋         | 3328/50176 [00:35<07:56, 98.31it/s]
  7%|▋         | 3584/50176 [00:37<07:45, 100.04it/s]
  8%|▊         | 3840/50176 [00:40<07:56, 97.21it/s] 
  8%|▊         | 4096/50176 [00:43<08:11, 93.76it/s]
  9%|▊         | 4352/50176 [00:45<07:57, 95.89it/s]
  9%|▉         | 4608/50176 [00:48<07:49, 97.11it/s]
 10%|▉         | 4864/50176 [00:50<07:42, 98.05it/s]
 10%|█         | 5120/50176 [00:53<07:33, 99.33it/s]
 11%|█         | 5376/50176 [00:56<07:36, 98.12it/s]
 11%|█         | 5632/50176 [00:58<07:37, 97.32it/s]
 12%|█▏        | 5888/50176 [01:01<07:34, 97.40it/s]
 12%|█▏        | 6144/50176 [01:04<07:58, 91.98it/s]
 13%|█▎        | 6400/50176 [01:07<07:45, 93.96it/s]
 13%|█▎        | 6656/50176 [01:09<07:32, 96.08it/s]
 14%|█▍        | 6912/50176 [01:12<07:36, 94.72it/s]
 14%|█▍        | 7168/50176 [01:14<07:25, 96.53it/s]
 15%|█▍        | 7424/50176 [01:17<07:12, 98.76it/s]
 15%|█▌        | 7680/50176 [01:19<07:07, 99.47it/s]
 16%|█▌        | 7936/50176 [01:22<06:44, 104.50it/s]
 16%|█▋        | 8192/50176 [01:24<06:46, 103.32it/s]
 17%|█▋        | 8448/50176 [01:27<06:42, 103.74it/s]
 17%|█▋        | 8704/50176 [01:29<06:38, 104.18it/s]
 18%|█▊        | 8960/50176 [01:32<06:45, 101.69it/s]
 18%|█▊        | 9216/50176 [01:34<06:38, 102.91it/s]
 19%|█▉        | 9472/50176 [01:37<06:31, 104.00it/s]
 19%|█▉        | 9728/50176 [01:39<06:41, 100.65it/s]
 20%|█▉        | 9984/50176 [01:42<06:37, 101.03it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.69it/s]
 21%|██        | 10496/50176 [01:47<06:24, 103.29it/s]
 21%|██▏       | 10752/50176 [01:49<06:31, 100.79it/s]
 22%|██▏       | 11008/50176 [01:52<06:28, 100.86it/s]
 22%|██▏       | 11264/50176 [01:54<06:23, 101.58it/s]
 23%|██▎       | 11520/50176 [01:57<06:20, 101.52it/s]
 23%|██▎       | 11776/50176 [01:59<06:18, 101.41it/s]
 24%|██▍       | 12032/50176 [02:02<06:17, 100.96it/s]
 24%|██▍       | 12288/50176 [02:04<06:13, 101.46it/s]
 25%|██▌       | 12544/50176 [02:07<06:12, 100.95it/s]
 26%|██▌       | 12800/50176 [02:09<06:10, 100.88it/s]
 26%|██▌       | 13056/50176 [02:12<06:05, 101.45it/s]
 27%|██▋       | 13312/50176 [02:14<06:02, 101.64it/s]
 27%|██▋       | 13568/50176 [02:17<06:13, 98.00it/s] 
 28%|██▊       | 13824/50176 [02:20<06:18, 95.93it/s]
 28%|██▊       | 14080/50176 [02:23<06:17, 95.72it/s]
 29%|██▊       | 14336/50176 [02:26<06:17, 95.03it/s]
 29%|██▉       | 14592/50176 [02:28<06:11, 95.75it/s]
 30%|██▉       | 14848/50176 [02:31<06:16, 93.86it/s]
 30%|███       | 15104/50176 [02:34<06:14, 93.71it/s]
 31%|███       | 15360/50176 [02:36<06:00, 96.65it/s]
 31%|███       | 15616/50176 [02:39<05:43, 100.58it/s]
 32%|███▏      | 15872/50176 [02:41<05:48, 98.56it/s] 
 32%|███▏      | 16128/50176 [02:44<05:47, 97.96it/s]
 33%|███▎      | 16384/50176 [02:46<05:36, 100.33it/s]
 33%|███▎      | 16640/50176 [02:49<05:46, 96.68it/s] 
 34%|███▎      | 16896/50176 [02:52<05:33, 99.81it/s]
 34%|███▍      | 17152/50176 [02:54<05:27, 100.96it/s]
 35%|███▍      | 17408/50176 [02:56<05:19, 102.63it/s]
 35%|███▌      | 17664/50176 [02:59<05:15, 103.08it/s]
 36%|███▌      | 17920/50176 [03:01<05:13, 102.97it/s]
 36%|███▌      | 18176/50176 [03:04<05:10, 103.18it/s]
 37%|███▋      | 18432/50176 [03:06<05:02, 104.79it/s]
 37%|███▋      | 18688/50176 [03:09<05:10, 101.33it/s]
 38%|███▊      | 18944/50176 [03:12<05:11, 100.33it/s]
 38%|███▊      | 19200/50176 [03:14<05:16, 97.85it/s] 
 39%|███▉      | 19456/50176 [03:17<05:19, 96.01it/s]
 39%|███▉      | 19712/50176 [03:20<05:18, 95.71it/s]
 40%|███▉      | 19968/50176 [03:22<05:13, 96.40it/s]
 40%|████      | 20224/50176 [03:25<05:02, 99.11it/s]
 41%|████      | 20480/50176 [03:27<04:55, 100.63it/s]
 41%|████▏     | 20736/50176 [03:30<04:45, 103.10it/s]
 42%|████▏     | 20992/50176 [03:32<04:35, 105.78it/s]
 42%|████▏     | 21248/50176 [03:35<04:42, 102.38it/s]
 43%|████▎     | 21504/50176 [03:37<04:37, 103.22it/s]
 43%|████▎     | 21760/50176 [03:39<04:35, 103.01it/s]
 44%|████▍     | 22016/50176 [03:42<04:28, 104.76it/s]
 44%|████▍     | 22272/50176 [03:44<04:20, 106.97it/s]
 45%|████▍     | 22528/50176 [03:46<04:16, 107.73it/s]
 45%|████▌     | 22784/50176 [03:49<04:23, 103.88it/s]
 46%|████▌     | 23040/50176 [03:52<04:28, 101.15it/s]
 46%|████▋     | 23296/50176 [03:54<04:30, 99.35it/s] 
 47%|████▋     | 23552/50176 [03:57<04:32, 97.53it/s]
 47%|████▋     | 23808/50176 [04:00<04:21, 100.97it/s]
 48%|████▊     | 24064/50176 [04:02<04:15, 102.12it/s]
 48%|████▊     | 24320/50176 [04:05<04:15, 101.27it/s]
 49%|████▉     | 24576/50176 [04:07<04:14, 100.65it/s]
 49%|████▉     | 24832/50176 [04:09<04:01, 104.86it/s]
 50%|█████     | 25088/50176 [04:12<03:55, 106.51it/s]
 51%|█████     | 25344/50176 [04:14<03:50, 107.60it/s]
 51%|█████     | 25600/50176 [04:17<03:54, 104.74it/s]
 52%|█████▏    | 25856/50176 [04:19<03:47, 106.74it/s]
 52%|█████▏    | 26112/50176 [04:21<03:41, 108.65it/s]
 53%|█████▎    | 26368/50176 [04:24<03:44, 105.92it/s]
 53%|█████▎    | 26624/50176 [04:26<03:43, 105.42it/s]
 54%|█████▎    | 26880/50176 [04:29<03:43, 104.14it/s]
 54%|█████▍    | 27136/50176 [04:31<03:42, 103.64it/s]
 55%|█████▍    | 27392/50176 [04:34<03:37, 104.85it/s]
 55%|█████▌    | 27648/50176 [04:36<03:27, 108.63it/s]
 56%|█████▌    | 27904/50176 [04:38<03:36, 103.08it/s]
 56%|█████▌    | 28160/50176 [04:41<03:38, 100.58it/s]
 57%|█████▋    | 28416/50176 [04:44<03:35, 101.13it/s]
 57%|█████▋    | 28672/50176 [04:46<03:32, 101.36it/s]
 58%|█████▊    | 28928/50176 [04:48<03:21, 105.40it/s]
 58%|█████▊    | 29184/50176 [04:51<03:22, 103.75it/s]
 59%|█████▊    | 29440/50176 [04:54<03:29, 99.08it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:25, 99.63it/s]
 60%|█████▉    | 29952/50176 [04:59<03:22, 99.77it/s]
 60%|██████    | 30208/50176 [05:01<03:17, 101.23it/s]
 61%|██████    | 30464/50176 [05:03<03:03, 107.14it/s]
 61%|██████    | 30720/50176 [05:06<03:04, 105.59it/s]
 62%|██████▏   | 30976/50176 [05:08<02:57, 108.08it/s]
 62%|██████▏   | 31232/50176 [05:10<02:53, 108.99it/s]
 63%|██████▎   | 31488/50176 [05:13<02:49, 110.00it/s]
 63%|██████▎   | 31744/50176 [05:15<02:49, 108.45it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.77it/s]
 64%|██████▍   | 32256/50176 [05:20<02:43, 109.36it/s]
 65%|██████▍   | 32512/50176 [05:23<02:50, 103.86it/s]
 65%|██████▌   | 32768/50176 [05:25<02:44, 106.05it/s]
 66%|██████▌   | 33024/50176 [05:27<02:41, 105.91it/s]
 66%|██████▋   | 33280/50176 [05:30<02:36, 108.23it/s]
 67%|██████▋   | 33536/50176 [05:32<02:39, 104.61it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.62it/s]
 68%|██████▊   | 34048/50176 [05:37<02:27, 109.12it/s]
 68%|██████▊   | 34304/50176 [05:39<02:22, 111.76it/s]
 69%|██████▉   | 34560/50176 [05:41<02:25, 107.57it/s]
 69%|██████▉   | 34816/50176 [05:44<02:25, 105.50it/s]
 70%|██████▉   | 35072/50176 [05:46<02:22, 105.80it/s]
 70%|███████   | 35328/50176 [05:49<02:18, 107.47it/s]
 71%|███████   | 35584/50176 [05:51<02:14, 108.12it/s]
 71%|███████▏  | 35840/50176 [05:54<02:17, 104.47it/s]
 72%|███████▏  | 36096/50176 [05:56<02:11, 106.94it/s]
 72%|███████▏  | 36352/50176 [05:58<02:11, 105.48it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 105.15it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.87it/s]
 74%|███████▍  | 37120/50176 [06:06<02:03, 105.48it/s]
 74%|███████▍  | 37376/50176 [06:08<01:56, 109.77it/s]
 75%|███████▌  | 37632/50176 [06:10<01:54, 110.02it/s]
 76%|███████▌  | 37888/50176 [06:13<01:54, 107.49it/s]
 76%|███████▌  | 38144/50176 [06:15<01:51, 108.25it/s]
 77%|███████▋  | 38400/50176 [06:17<01:50, 106.81it/s]
 77%|███████▋  | 38656/50176 [06:20<01:48, 105.85it/s]
 78%|███████▊  | 38912/50176 [06:22<01:48, 104.23it/s]
 78%|███████▊  | 39168/50176 [06:25<01:47, 102.00it/s]
 79%|███████▊  | 39424/50176 [06:27<01:42, 105.23it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.02it/s]
 80%|███████▉  | 39936/50176 [06:32<01:36, 106.52it/s]
 80%|████████  | 40192/50176 [06:35<01:33, 106.60it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 103.78it/s]
 81%|████████  | 40704/50176 [06:40<01:34, 100.34it/s]
 82%|████████▏ | 40960/50176 [06:42<01:29, 103.07it/s]
 82%|████████▏ | 41216/50176 [06:45<01:25, 105.11it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.33it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 105.73it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 107.61it/s]
 84%|████████▍ | 42240/50176 [06:54<01:14, 106.86it/s]
 85%|████████▍ | 42496/50176 [06:57<01:12, 106.27it/s]
 85%|████████▌ | 42752/50176 [06:59<01:09, 107.12it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.77it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 106.58it/s]
 87%|████████▋ | 43520/50176 [07:06<01:03, 104.29it/s]
 87%|████████▋ | 43776/50176 [07:09<01:00, 105.53it/s]
 88%|████████▊ | 44032/50176 [07:11<00:57, 106.22it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 108.46it/s]
 89%|████████▉ | 44544/50176 [07:16<00:53, 104.79it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.91it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 106.29it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.25it/s]
 91%|█████████ | 45568/50176 [07:26<00:43, 106.51it/s]
 91%|█████████▏| 45824/50176 [07:28<00:40, 106.17it/s]
 92%|█████████▏| 46080/50176 [07:30<00:38, 105.34it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.06it/s]
 93%|█████████▎| 46592/50176 [07:35<00:33, 106.03it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.84it/s]
 94%|█████████▍| 47104/50176 [07:40<00:29, 103.10it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.56it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 99.46it/s] 
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.58it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 103.38it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 104.03it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 103.99it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 104.40it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.57it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 103.19it/s]
 99%|█████████▉| 49664/50176 [08:05<00:05, 101.09it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 95.22it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.41it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.49it/s]
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6789	imagenet-zeroshot-val-top5: 0.9077
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_62.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_62.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b981efac301553208853a88df602ff22aeb3102e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_62.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82b8a8ca673428208d20cc93feea6fdc1a13ace42ca6b670ff3d5ef2b1dca07f
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_62.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_62.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7b9e25a4c41c1050193e23cc15a6f3ab61588f49
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_62.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_62.pt).
+2022-12-21,11:30:05 | INFO | Model:
+2022-12-21,11:30:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:05 | INFO | Params:
+2022-12-21,11:30:05 | INFO |   batch_size: 256
+2022-12-21,11:30:05 | INFO |   beta1: 0.9
+2022-12-21,11:30:05 | INFO |   beta2: 0.98
+2022-12-21,11:30:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_62.pt/checkpoints
+2022-12-21,11:30:05 | INFO |   copy_codebase: False
+2022-12-21,11:30:05 | INFO |   csv_caption_key: title
+2022-12-21,11:30:05 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:05 | INFO |   csv_separator: 	
+2022-12-21,11:30:05 | INFO |   dataset_resampled: False
+2022-12-21,11:30:05 | INFO |   dataset_type: auto
+2022-12-21,11:30:05 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:05 | INFO |   debug: False
+2022-12-21,11:30:05 | INFO |   device: cuda:0
+2022-12-21,11:30:05 | INFO |   dist_backend: nccl
+2022-12-21,11:30:05 | INFO |   dist_url: env://
+2022-12-21,11:30:05 | INFO |   distributed: False
+2022-12-21,11:30:05 | INFO |   epochs: 32
+2022-12-21,11:30:05 | INFO |   eps: 1e-06
+2022-12-21,11:30:05 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:05 | INFO |   gather_with_grad: False
+2022-12-21,11:30:05 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:05 | INFO |   horovod: False
+2022-12-21,11:30:05 | INFO |   image_mean: None
+2022-12-21,11:30:05 | INFO |   image_std: None
+2022-12-21,11:30:05 | INFO |   imagenet_v2: None
+2022-12-21,11:30:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:05 | INFO |   local_loss: False
+2022-12-21,11:30:05 | INFO |   local_rank: 0
+2022-12-21,11:30:05 | INFO |   lock_image: False
+2022-12-21,11:30:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:05 | INFO |   log_level: 20
+2022-12-21,11:30:05 | INFO |   log_local: False
+2022-12-21,11:30:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_62.pt/out.log
+2022-12-21,11:30:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:05 | INFO |   lr: 0.0005
+2022-12-21,11:30:05 | INFO |   model: ViT-B-16
+2022-12-21,11:30:05 | INFO |   name: epoch_62.pt
+2022-12-21,11:30:05 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:05 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:05 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_62.pt
+2022-12-21,11:30:05 | INFO |   pretrained_image: False
+2022-12-21,11:30:05 | INFO |   rank: 0
+2022-12-21,11:30:05 | INFO |   report_to: 
+2022-12-21,11:30:05 | INFO |   resume: None
+2022-12-21,11:30:05 | INFO |   save_frequency: 1
+2022-12-21,11:30:05 | INFO |   save_most_recent: False
+2022-12-21,11:30:05 | INFO |   seed: 0
+2022-12-21,11:30:05 | INFO |   skip_scheduler: False
+2022-12-21,11:30:05 | INFO |   tensorboard: False
+2022-12-21,11:30:05 | INFO |   tensorboard_path: 
+2022-12-21,11:30:05 | INFO |   torchscript: False
+2022-12-21,11:30:05 | INFO |   trace: False
+2022-12-21,11:30:05 | INFO |   train_data: None
+2022-12-21,11:30:05 | INFO |   train_num_samples: None
+2022-12-21,11:30:05 | INFO |   use_bn_sync: False
+2022-12-21,11:30:05 | INFO |   val_data: None
+2022-12-21,11:30:05 | INFO |   val_frequency: 1
+2022-12-21,11:30:05 | INFO |   val_num_samples: None
+2022-12-21,11:30:05 | INFO |   wandb: False
+2022-12-21,11:30:05 | INFO |   wandb_notes: 
+2022-12-21,11:30:05 | INFO |   warmup: 10000
+2022-12-21,11:30:05 | INFO |   wd: 0.2
+2022-12-21,11:30:05 | INFO |   workers: 1
+2022-12-21,11:30:05 | INFO |   world_size: 1
+2022-12-21,11:30:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:06 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:06 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:44,  1.55it/s]
  1%|          | 6/1000 [00:00<01:37, 10.24it/s]
  1%|          | 12/1000 [00:00<00:49, 20.04it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.72it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.91it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.60it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.80it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.07it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.47it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.24it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.47it/s]
  7%|▋         | 72/1000 [00:01<00:16, 56.11it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.54it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.94it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.18it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.52it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.70it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.99it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.15it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.13it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.09it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.13it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.11it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.11it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.17it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.21it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.05it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.90it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.86it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.82it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.76it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.74it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.69it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.50it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.51it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.48it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.69it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.68it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.58it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.62it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.70it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.82it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.86it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.66it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.84it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.94it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.16it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.15it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.15it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.28it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.26it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.18it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.00it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.08it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.16it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.27it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.99it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 58.02it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.11it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.22it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.80it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.88it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.98it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.12it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.20it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.19it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.22it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.25it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.35it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.35it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.13it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.20it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.24it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.21it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.16it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.20it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.14it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.15it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.08it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.14it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.05it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.13it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.16it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.25it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.20it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.30it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.30it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.21it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.21it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.15it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.08it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.21it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.13it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.23it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.14it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.12it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.27it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.22it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.32it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.17it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.09it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.24it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.15it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.25it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.28it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.13it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.23it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.23it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.15it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.25it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.26it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.25it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.27it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.19it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.22it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.24it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.24it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.32it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.39it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.23it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.34it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.27it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.31it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.35it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.15it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.12it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.18it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.15it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.07it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.32it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.25it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.32it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.30it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.34it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.26it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.32it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.30it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.18it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.28it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.26it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.23it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.25it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.23it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.19it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.23it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.14it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.22it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.15it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.25it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.25it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.21it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.22it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.04it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.96it/s]
+2022-12-21,11:30:24 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<12:06, 68.75it/s]
  1%|          | 512/50176 [00:06<10:17, 80.37it/s]
  2%|▏         | 768/50176 [00:08<09:00, 91.48it/s]
  2%|▏         | 1024/50176 [00:11<08:43, 93.82it/s]
  3%|▎         | 1280/50176 [00:14<08:34, 94.97it/s]
  3%|▎         | 1536/50176 [00:17<09:06, 88.93it/s]
  4%|▎         | 1792/50176 [00:19<08:46, 91.89it/s]
  4%|▍         | 2048/50176 [00:22<08:26, 94.96it/s]
  5%|▍         | 2304/50176 [00:25<08:17, 96.25it/s]
  5%|▌         | 2560/50176 [00:27<07:53, 100.50it/s]
  6%|▌         | 2816/50176 [00:30<07:59, 98.76it/s] 
  6%|▌         | 3072/50176 [00:32<07:51, 99.88it/s]
  7%|▋         | 3328/50176 [00:35<07:55, 98.52it/s]
  7%|▋         | 3584/50176 [00:37<07:46, 99.97it/s]
  8%|▊         | 3840/50176 [00:40<07:56, 97.28it/s]
  8%|▊         | 4096/50176 [00:43<08:11, 93.67it/s]
  9%|▊         | 4352/50176 [00:45<07:59, 95.66it/s]
  9%|▉         | 4608/50176 [00:48<07:48, 97.25it/s]
 10%|▉         | 4864/50176 [00:51<07:42, 98.03it/s]
 10%|█         | 5120/50176 [00:53<07:33, 99.40it/s]
 11%|█         | 5376/50176 [00:56<07:37, 97.95it/s]
 11%|█         | 5632/50176 [00:58<07:36, 97.53it/s]
 12%|█▏        | 5888/50176 [01:01<07:34, 97.36it/s]
 12%|█▏        | 6144/50176 [01:04<07:59, 91.92it/s]
 13%|█▎        | 6400/50176 [01:07<07:45, 93.99it/s]
 13%|█▎        | 6656/50176 [01:09<07:33, 96.06it/s]
 14%|█▍        | 6912/50176 [01:12<07:37, 94.63it/s]
 14%|█▍        | 7168/50176 [01:15<07:25, 96.61it/s]
 15%|█▍        | 7424/50176 [01:17<07:12, 98.78it/s]
 15%|█▌        | 7680/50176 [01:20<07:07, 99.45it/s]
 16%|█▌        | 7936/50176 [01:22<06:44, 104.32it/s]
 16%|█▋        | 8192/50176 [01:24<06:45, 103.42it/s]
 17%|█▋        | 8448/50176 [01:27<06:41, 104.01it/s]
 17%|█▋        | 8704/50176 [01:29<06:38, 104.15it/s]
 18%|█▊        | 8960/50176 [01:32<06:45, 101.57it/s]
 18%|█▊        | 9216/50176 [01:34<06:38, 102.77it/s]
 19%|█▉        | 9472/50176 [01:37<06:31, 103.99it/s]
 19%|█▉        | 9728/50176 [01:39<06:41, 100.64it/s]
 20%|█▉        | 9984/50176 [01:42<06:37, 101.07it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.63it/s]
 21%|██        | 10496/50176 [01:47<06:24, 103.23it/s]
 21%|██▏       | 10752/50176 [01:49<06:31, 100.70it/s]
 22%|██▏       | 11008/50176 [01:52<06:27, 100.98it/s]
 22%|██▏       | 11264/50176 [01:54<06:23, 101.51it/s]
 23%|██▎       | 11520/50176 [01:57<06:20, 101.59it/s]
 23%|██▎       | 11776/50176 [02:00<06:18, 101.41it/s]
 24%|██▍       | 12032/50176 [02:02<06:16, 101.18it/s]
 24%|██▍       | 12288/50176 [02:05<06:13, 101.31it/s]
 25%|██▌       | 12544/50176 [02:07<06:12, 100.96it/s]
 26%|██▌       | 12800/50176 [02:10<06:10, 100.86it/s]
 26%|██▌       | 13056/50176 [02:12<06:05, 101.47it/s]
 27%|██▋       | 13312/50176 [02:15<06:03, 101.47it/s]
 27%|██▋       | 13568/50176 [02:18<06:13, 97.99it/s] 
 28%|██▊       | 13824/50176 [02:20<06:19, 95.87it/s]
 28%|██▊       | 14080/50176 [02:23<06:16, 95.80it/s]
 29%|██▊       | 14336/50176 [02:26<06:16, 95.08it/s]
 29%|██▉       | 14592/50176 [02:28<06:11, 95.79it/s]
 30%|██▉       | 14848/50176 [02:31<06:16, 93.82it/s]
 30%|███       | 15104/50176 [02:34<06:13, 93.89it/s]
 31%|███       | 15360/50176 [02:36<06:00, 96.62it/s]
 31%|███       | 15616/50176 [02:39<05:43, 100.53it/s]
 32%|███▏      | 15872/50176 [02:41<05:48, 98.56it/s] 
 32%|███▏      | 16128/50176 [02:44<05:47, 97.93it/s]
 33%|███▎      | 16384/50176 [02:46<05:36, 100.31it/s]
 33%|███▎      | 16640/50176 [02:49<05:46, 96.68it/s] 
 34%|███▎      | 16896/50176 [02:52<05:33, 99.86it/s]
 34%|███▍      | 17152/50176 [02:54<05:27, 100.88it/s]
 35%|███▍      | 17408/50176 [02:57<05:19, 102.60it/s]
 35%|███▌      | 17664/50176 [02:59<05:15, 103.02it/s]
 36%|███▌      | 17920/50176 [03:02<05:12, 103.12it/s]
 36%|███▌      | 18176/50176 [03:04<05:09, 103.31it/s]
 37%|███▋      | 18432/50176 [03:06<05:02, 104.80it/s]
 37%|███▋      | 18688/50176 [03:09<05:10, 101.32it/s]
 38%|███▊      | 18944/50176 [03:12<05:11, 100.33it/s]
 38%|███▊      | 19200/50176 [03:14<05:16, 97.93it/s] 
 39%|███▉      | 19456/50176 [03:17<05:20, 95.99it/s]
 39%|███▉      | 19712/50176 [03:20<05:18, 95.68it/s]
 40%|███▉      | 19968/50176 [03:23<05:13, 96.44it/s]
 40%|████      | 20224/50176 [03:25<05:02, 99.16it/s]
 41%|████      | 20480/50176 [03:27<04:55, 100.57it/s]
 41%|████▏     | 20736/50176 [03:30<04:45, 103.09it/s]
 42%|████▏     | 20992/50176 [03:32<04:35, 105.81it/s]
 42%|████▏     | 21248/50176 [03:35<04:42, 102.40it/s]
 43%|████▎     | 21504/50176 [03:37<04:38, 103.13it/s]
 43%|████▎     | 21760/50176 [03:40<04:35, 103.14it/s]
 44%|████▍     | 22016/50176 [03:42<04:29, 104.64it/s]
 44%|████▍     | 22272/50176 [03:44<04:20, 106.93it/s]
 45%|████▍     | 22528/50176 [03:47<04:16, 107.71it/s]
 45%|████▌     | 22784/50176 [03:49<04:23, 103.76it/s]
 46%|████▌     | 23040/50176 [03:52<04:28, 101.22it/s]
 46%|████▋     | 23296/50176 [03:55<04:30, 99.42it/s] 
 47%|████▋     | 23552/50176 [03:57<04:33, 97.48it/s]
 47%|████▋     | 23808/50176 [04:00<04:21, 100.94it/s]
 48%|████▊     | 24064/50176 [04:02<04:15, 102.06it/s]
 48%|████▊     | 24320/50176 [04:05<04:14, 101.40it/s]
 49%|████▉     | 24576/50176 [04:07<04:14, 100.69it/s]
 49%|████▉     | 24832/50176 [04:10<04:01, 104.88it/s]
 50%|█████     | 25088/50176 [04:12<03:55, 106.42it/s]
 51%|█████     | 25344/50176 [04:14<03:50, 107.63it/s]
 51%|█████     | 25600/50176 [04:17<03:54, 104.71it/s]
 52%|█████▏    | 25856/50176 [04:19<03:47, 106.87it/s]
 52%|█████▏    | 26112/50176 [04:21<03:41, 108.58it/s]
 53%|█████▎    | 26368/50176 [04:24<03:44, 106.03it/s]
 53%|█████▎    | 26624/50176 [04:26<03:43, 105.35it/s]
 54%|█████▎    | 26880/50176 [04:29<03:44, 103.99it/s]
 54%|█████▍    | 27136/50176 [04:31<03:42, 103.76it/s]
 55%|█████▍    | 27392/50176 [04:34<03:37, 104.84it/s]
 55%|█████▌    | 27648/50176 [04:36<03:27, 108.47it/s]
 56%|█████▌    | 27904/50176 [04:39<03:36, 103.07it/s]
 56%|█████▌    | 28160/50176 [04:41<03:39, 100.49it/s]
 57%|█████▋    | 28416/50176 [04:44<03:34, 101.32it/s]
 57%|█████▋    | 28672/50176 [04:46<03:32, 101.28it/s]
 58%|█████▊    | 28928/50176 [04:49<03:21, 105.38it/s]
 58%|█████▊    | 29184/50176 [04:51<03:22, 103.81it/s]
 59%|█████▊    | 29440/50176 [04:54<03:29, 99.10it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:25, 99.67it/s]
 60%|█████▉    | 29952/50176 [04:59<03:22, 99.66it/s]
 60%|██████    | 30208/50176 [05:02<03:17, 101.13it/s]
 61%|██████    | 30464/50176 [05:04<03:03, 107.18it/s]
 61%|██████    | 30720/50176 [05:06<03:04, 105.60it/s]
 62%|██████▏   | 30976/50176 [05:08<02:57, 108.16it/s]
 62%|██████▏   | 31232/50176 [05:11<02:53, 108.99it/s]
 63%|██████▎   | 31488/50176 [05:13<02:49, 110.14it/s]
 63%|██████▎   | 31744/50176 [05:15<02:49, 108.57it/s]
 64%|██████▍   | 32000/50176 [05:18<02:47, 108.69it/s]
 64%|██████▍   | 32256/50176 [05:20<02:43, 109.29it/s]
 65%|██████▍   | 32512/50176 [05:23<02:50, 103.89it/s]
 65%|██████▌   | 32768/50176 [05:25<02:44, 106.01it/s]
 66%|██████▌   | 33024/50176 [05:27<02:41, 105.89it/s]
 66%|██████▋   | 33280/50176 [05:30<02:36, 108.18it/s]
 67%|██████▋   | 33536/50176 [05:32<02:39, 104.51it/s]
 67%|██████▋   | 33792/50176 [05:35<02:33, 106.68it/s]
 68%|██████▊   | 34048/50176 [05:37<02:27, 109.18it/s]
 68%|██████▊   | 34304/50176 [05:39<02:22, 111.73it/s]
 69%|██████▉   | 34560/50176 [05:42<02:25, 107.43it/s]
 69%|██████▉   | 34816/50176 [05:44<02:25, 105.59it/s]
 70%|██████▉   | 35072/50176 [05:47<02:22, 105.84it/s]
 70%|███████   | 35328/50176 [05:49<02:18, 107.45it/s]
 71%|███████   | 35584/50176 [05:51<02:15, 108.07it/s]
 71%|███████▏  | 35840/50176 [05:54<02:17, 104.55it/s]
 72%|███████▏  | 36096/50176 [05:56<02:11, 107.00it/s]
 72%|███████▏  | 36352/50176 [05:59<02:11, 105.37it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 105.02it/s]
 73%|███████▎  | 36864/50176 [06:04<02:09, 102.79it/s]
 74%|███████▍  | 37120/50176 [06:06<02:03, 105.56it/s]
 74%|███████▍  | 37376/50176 [06:08<01:56, 109.76it/s]
 75%|███████▌  | 37632/50176 [06:10<01:53, 110.18it/s]
 76%|███████▌  | 37888/50176 [06:13<01:54, 107.47it/s]
 76%|███████▌  | 38144/50176 [06:15<01:51, 108.25it/s]
 77%|███████▋  | 38400/50176 [06:18<01:50, 107.00it/s]
 77%|███████▋  | 38656/50176 [06:20<01:48, 105.70it/s]
 78%|███████▊  | 38912/50176 [06:23<01:47, 104.33it/s]
 78%|███████▊  | 39168/50176 [06:25<01:47, 102.18it/s]
 79%|███████▊  | 39424/50176 [06:28<01:42, 105.16it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 104.96it/s]
 80%|███████▉  | 39936/50176 [06:32<01:36, 106.43it/s]
 80%|████████  | 40192/50176 [06:35<01:33, 106.49it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 103.69it/s]
 81%|████████  | 40704/50176 [06:40<01:34, 100.38it/s]
 82%|████████▏ | 40960/50176 [06:42<01:29, 103.13it/s]
 82%|████████▏ | 41216/50176 [06:45<01:25, 105.11it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.34it/s]
 83%|████████▎ | 41728/50176 [06:50<01:19, 105.70it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 107.49it/s]
 84%|████████▍ | 42240/50176 [06:54<01:14, 106.77it/s]
 85%|████████▍ | 42496/50176 [06:57<01:12, 106.29it/s]
 85%|████████▌ | 42752/50176 [06:59<01:09, 107.10it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.94it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 106.57it/s]
 87%|████████▋ | 43520/50176 [07:06<01:03, 104.38it/s]
 87%|████████▋ | 43776/50176 [07:09<01:00, 105.38it/s]
 88%|████████▊ | 44032/50176 [07:11<00:57, 106.21it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 108.41it/s]
 89%|████████▉ | 44544/50176 [07:16<00:53, 104.76it/s]
 89%|████████▉ | 44800/50176 [07:19<00:51, 104.09it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 106.28it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.43it/s]
 91%|█████████ | 45568/50176 [07:26<00:43, 106.43it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 106.11it/s]
 92%|█████████▏| 46080/50176 [07:31<00:38, 105.20it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.23it/s]
 93%|█████████▎| 46592/50176 [07:35<00:33, 105.91it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.94it/s]
 94%|█████████▍| 47104/50176 [07:41<00:29, 103.15it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.50it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 99.45it/s] 
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.48it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 103.49it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 104.13it/s]
 97%|█████████▋| 48640/50176 [07:56<00:14, 103.93it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 104.38it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.53it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 103.31it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.06it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 95.20it/s] 
100%|██████████| 50176/50176 [08:09<00:00, 124.14it/s]
100%|██████████| 50176/50176 [08:09<00:00, 102.45it/s]
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6791	imagenet-zeroshot-val-top5: 0.9080
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_63.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_63.pt
new file mode 100644
index 0000000000000000000000000000000000000000..6fbb50bbde92aef5622d3df52081cf936b183cf7
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_63.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:983f29ea2f79f70bcd231f7f9b4b733ddfe0ae602835f5b2a092899aa0b54e3c
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_63.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_63.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d58429a32b11d9409d6f698506dbcddcbff62e29
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_63.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_63.pt).
+2022-12-21,11:30:04 | INFO | Model:
+2022-12-21,11:30:04 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:04 | INFO | Params:
+2022-12-21,11:30:04 | INFO |   batch_size: 256
+2022-12-21,11:30:04 | INFO |   beta1: 0.9
+2022-12-21,11:30:04 | INFO |   beta2: 0.98
+2022-12-21,11:30:04 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_63.pt/checkpoints
+2022-12-21,11:30:04 | INFO |   copy_codebase: False
+2022-12-21,11:30:04 | INFO |   csv_caption_key: title
+2022-12-21,11:30:04 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:04 | INFO |   csv_separator: 	
+2022-12-21,11:30:04 | INFO |   dataset_resampled: False
+2022-12-21,11:30:04 | INFO |   dataset_type: auto
+2022-12-21,11:30:04 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:04 | INFO |   debug: False
+2022-12-21,11:30:04 | INFO |   device: cuda:0
+2022-12-21,11:30:04 | INFO |   dist_backend: nccl
+2022-12-21,11:30:04 | INFO |   dist_url: env://
+2022-12-21,11:30:04 | INFO |   distributed: False
+2022-12-21,11:30:04 | INFO |   epochs: 32
+2022-12-21,11:30:04 | INFO |   eps: 1e-06
+2022-12-21,11:30:04 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:04 | INFO |   gather_with_grad: False
+2022-12-21,11:30:04 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:04 | INFO |   horovod: False
+2022-12-21,11:30:04 | INFO |   image_mean: None
+2022-12-21,11:30:04 | INFO |   image_std: None
+2022-12-21,11:30:04 | INFO |   imagenet_v2: None
+2022-12-21,11:30:04 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:04 | INFO |   local_loss: False
+2022-12-21,11:30:04 | INFO |   local_rank: 0
+2022-12-21,11:30:04 | INFO |   lock_image: False
+2022-12-21,11:30:04 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:04 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:04 | INFO |   log_level: 20
+2022-12-21,11:30:04 | INFO |   log_local: False
+2022-12-21,11:30:04 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_63.pt/out.log
+2022-12-21,11:30:04 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:04 | INFO |   lr: 0.0005
+2022-12-21,11:30:04 | INFO |   model: ViT-B-16
+2022-12-21,11:30:04 | INFO |   name: epoch_63.pt
+2022-12-21,11:30:04 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:04 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:04 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:04 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_63.pt
+2022-12-21,11:30:04 | INFO |   pretrained_image: False
+2022-12-21,11:30:04 | INFO |   rank: 0
+2022-12-21,11:30:04 | INFO |   report_to: 
+2022-12-21,11:30:04 | INFO |   resume: None
+2022-12-21,11:30:04 | INFO |   save_frequency: 1
+2022-12-21,11:30:04 | INFO |   save_most_recent: False
+2022-12-21,11:30:04 | INFO |   seed: 0
+2022-12-21,11:30:04 | INFO |   skip_scheduler: False
+2022-12-21,11:30:04 | INFO |   tensorboard: False
+2022-12-21,11:30:04 | INFO |   tensorboard_path: 
+2022-12-21,11:30:04 | INFO |   torchscript: False
+2022-12-21,11:30:04 | INFO |   trace: False
+2022-12-21,11:30:04 | INFO |   train_data: None
+2022-12-21,11:30:04 | INFO |   train_num_samples: None
+2022-12-21,11:30:04 | INFO |   use_bn_sync: False
+2022-12-21,11:30:04 | INFO |   val_data: None
+2022-12-21,11:30:04 | INFO |   val_frequency: 1
+2022-12-21,11:30:04 | INFO |   val_num_samples: None
+2022-12-21,11:30:04 | INFO |   wandb: False
+2022-12-21,11:30:04 | INFO |   wandb_notes: 
+2022-12-21,11:30:04 | INFO |   warmup: 10000
+2022-12-21,11:30:04 | INFO |   wd: 0.2
+2022-12-21,11:30:04 | INFO |   workers: 1
+2022-12-21,11:30:04 | INFO |   world_size: 1
+2022-12-21,11:30:04 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:04 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:04 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<16:21,  1.02it/s]
  1%|          | 6/1000 [00:01<02:18,  7.19it/s]
  1%|          | 11/1000 [00:01<01:12, 13.64it/s]
  2%|▏         | 17/1000 [00:01<00:45, 21.37it/s]
  2%|▏         | 23/1000 [00:01<00:34, 28.42it/s]
  3%|▎         | 29/1000 [00:01<00:28, 34.50it/s]
  4%|▎         | 35/1000 [00:01<00:24, 39.69it/s]
  4%|▍         | 41/1000 [00:01<00:21, 44.30it/s]
  5%|▍         | 47/1000 [00:01<00:20, 47.44it/s]
  5%|▌         | 53/1000 [00:01<00:19, 49.62it/s]
  6%|▌         | 59/1000 [00:02<00:18, 51.02it/s]
  6%|▋         | 65/1000 [00:02<00:17, 52.03it/s]
  7%|▋         | 71/1000 [00:02<00:17, 52.71it/s]
  8%|▊         | 77/1000 [00:02<00:17, 53.32it/s]
  8%|▊         | 83/1000 [00:02<00:16, 54.58it/s]
  9%|▉         | 89/1000 [00:02<00:16, 55.66it/s]
 10%|▉         | 95/1000 [00:02<00:16, 56.36it/s]
 10%|█         | 101/1000 [00:02<00:15, 56.91it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.44it/s]
 11%|█▏        | 113/1000 [00:03<00:15, 57.74it/s]
 12%|█▏        | 119/1000 [00:03<00:15, 57.88it/s]
 12%|█▎        | 125/1000 [00:03<00:15, 57.90it/s]
 13%|█▎        | 131/1000 [00:03<00:14, 58.04it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 58.01it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 58.05it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.15it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 58.04it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.91it/s]
 17%|█▋        | 173/1000 [00:04<00:14, 57.93it/s]
 18%|█▊        | 179/1000 [00:04<00:14, 57.94it/s]
 18%|█▊        | 185/1000 [00:04<00:14, 57.79it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.77it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.70it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.59it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.41it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.48it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.67it/s]
 23%|██▎       | 233/1000 [00:05<00:13, 57.74it/s]
 24%|██▍       | 239/1000 [00:05<00:13, 57.53it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.57it/s]
 25%|██▌       | 251/1000 [00:05<00:12, 57.65it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.88it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.91it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.67it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.75it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.92it/s]
 29%|██▊       | 287/1000 [00:06<00:12, 58.01it/s]
 29%|██▉       | 293/1000 [00:06<00:12, 58.24it/s]
 30%|██▉       | 299/1000 [00:06<00:12, 58.21it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.24it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.35it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.35it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.27it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 58.05it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.13it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.18it/s]
 35%|███▍      | 347/1000 [00:07<00:11, 58.21it/s]
 35%|███▌      | 353/1000 [00:07<00:11, 58.02it/s]
 36%|███▌      | 359/1000 [00:07<00:11, 58.15it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.11it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.26it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.89it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.88it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.97it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.11it/s]
 41%|████      | 407/1000 [00:08<00:10, 58.21it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 58.23it/s]
 42%|████▏     | 419/1000 [00:08<00:09, 58.26it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.21it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.34it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.34it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.12it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.22it/s]
 46%|████▌     | 461/1000 [00:09<00:09, 58.19it/s]
 47%|████▋     | 467/1000 [00:09<00:09, 58.18it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 58.32it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.22it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.21it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.16it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 58.01it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.16it/s]
 51%|█████     | 509/1000 [00:09<00:08, 58.05it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.08it/s]
 52%|█████▏    | 521/1000 [00:10<00:08, 58.13it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 58.24it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 58.10it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.21it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.25it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.17it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.13it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.22it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.11it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 58.03it/s]
 58%|█████▊    | 581/1000 [00:11<00:07, 58.20it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 58.12it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 58.08it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.17it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.07it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.14it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.22it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.24it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.30it/s]
 64%|██████▎   | 635/1000 [00:12<00:06, 58.15it/s]
 64%|██████▍   | 641/1000 [00:12<00:06, 58.01it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 58.23it/s]
 65%|██████▌   | 653/1000 [00:12<00:05, 58.17it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.26it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.27it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.27it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.31it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.08it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.17it/s]
 70%|██████▉   | 695/1000 [00:13<00:05, 58.09it/s]
 70%|███████   | 701/1000 [00:13<00:05, 58.22it/s]
 71%|███████   | 707/1000 [00:13<00:05, 58.24it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.25it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.29it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.20it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.24it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.19it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.28it/s]
 76%|███████▌  | 755/1000 [00:14<00:04, 58.41it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 58.23it/s]
 77%|███████▋  | 767/1000 [00:14<00:03, 58.27it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.25it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.29it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.32it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.12it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.13it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.14it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 815/1000 [00:15<00:03, 58.13it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 58.20it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 58.20it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.37it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.45it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.29it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.34it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.26it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.32it/s]
 87%|████████▋ | 869/1000 [00:16<00:02, 58.25it/s]
 88%|████████▊ | 875/1000 [00:16<00:02, 58.27it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 58.33it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 58.18it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.27it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.31it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.26it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.21it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.25it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.19it/s]
 93%|█████████▎| 929/1000 [00:17<00:01, 58.19it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 58.29it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 58.29it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.28it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.32it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.26it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.40it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.47it/s]
 99%|█████████▉| 989/1000 [00:18<00:00, 58.42it/s]
100%|█████████▉| 995/1000 [00:18<00:00, 58.27it/s]
100%|██████████| 1000/1000 [00:18<00:00, 54.73it/s]
+2022-12-21,11:30:22 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:05<16:27, 50.58it/s]
  1%|          | 512/50176 [00:07<12:02, 68.70it/s]
  2%|▏         | 768/50176 [00:10<09:57, 82.72it/s]
  2%|▏         | 1024/50176 [00:12<09:18, 88.05it/s]
  3%|▎         | 1280/50176 [00:15<08:56, 91.07it/s]
  3%|▎         | 1536/50176 [00:18<09:21, 86.63it/s]
  4%|▎         | 1792/50176 [00:21<08:55, 90.34it/s]
  4%|▍         | 2048/50176 [00:23<08:31, 94.08it/s]
  5%|▍         | 2304/50176 [00:26<08:21, 95.51it/s]
  5%|▌         | 2560/50176 [00:28<07:57, 99.77it/s]
  6%|▌         | 2816/50176 [00:31<08:01, 98.40it/s]
  6%|▌         | 3072/50176 [00:33<07:53, 99.54it/s]
  7%|▋         | 3328/50176 [00:36<07:57, 98.16it/s]
  7%|▋         | 3584/50176 [00:38<07:46, 99.83it/s]
  8%|▊         | 3840/50176 [00:41<07:56, 97.17it/s]
  8%|▊         | 4096/50176 [00:44<08:11, 93.73it/s]
  9%|▊         | 4352/50176 [00:47<07:58, 95.85it/s]
  9%|▉         | 4608/50176 [00:49<07:48, 97.21it/s]
 10%|▉         | 4864/50176 [00:52<07:42, 98.05it/s]
 10%|█         | 5120/50176 [00:54<07:34, 99.19it/s]
 11%|█         | 5376/50176 [00:57<07:37, 97.88it/s]
 11%|█         | 5632/50176 [01:00<07:36, 97.58it/s]
 12%|█▏        | 5888/50176 [01:02<07:35, 97.20it/s]
 12%|█▏        | 6144/50176 [01:06<08:00, 91.66it/s]
 13%|█▎        | 6400/50176 [01:08<07:45, 93.97it/s]
 13%|█▎        | 6656/50176 [01:11<07:32, 96.24it/s]
 14%|█▍        | 6912/50176 [01:13<07:37, 94.60it/s]
 14%|█▍        | 7168/50176 [01:16<07:24, 96.67it/s]
 15%|█▍        | 7424/50176 [01:18<07:12, 98.82it/s]
 15%|█▌        | 7680/50176 [01:21<07:06, 99.61it/s]
 16%|█▌        | 7936/50176 [01:23<06:44, 104.33it/s]
 16%|█▋        | 8192/50176 [01:26<06:46, 103.36it/s]
 17%|█▋        | 8448/50176 [01:28<06:41, 103.96it/s]
 17%|█▋        | 8704/50176 [01:31<06:37, 104.21it/s]
 18%|█▊        | 8960/50176 [01:33<06:45, 101.63it/s]
 18%|█▊        | 9216/50176 [01:36<06:38, 102.85it/s]
 19%|█▉        | 9472/50176 [01:38<06:31, 103.99it/s]
 19%|█▉        | 9728/50176 [01:41<06:41, 100.79it/s]
 20%|█▉        | 9984/50176 [01:43<06:37, 100.99it/s]
 20%|██        | 10240/50176 [01:46<06:25, 103.63it/s]
 21%|██        | 10496/50176 [01:48<06:24, 103.10it/s]
 21%|██▏       | 10752/50176 [01:51<06:31, 100.72it/s]
 22%|██▏       | 11008/50176 [01:53<06:28, 100.87it/s]
 22%|██▏       | 11264/50176 [01:56<06:23, 101.57it/s]
 23%|██▎       | 11520/50176 [01:58<06:20, 101.63it/s]
 23%|██▎       | 11776/50176 [02:01<06:18, 101.38it/s]
 24%|██▍       | 12032/50176 [02:03<06:17, 100.99it/s]
 24%|██▍       | 12288/50176 [02:06<06:13, 101.35it/s]
 25%|██▌       | 12544/50176 [02:08<06:12, 100.98it/s]
 26%|██▌       | 12800/50176 [02:11<06:10, 100.96it/s]
 26%|██▌       | 13056/50176 [02:13<06:05, 101.42it/s]
 27%|██▋       | 13312/50176 [02:16<06:03, 101.54it/s]
 27%|██▋       | 13568/50176 [02:19<06:13, 97.98it/s] 
 28%|██▊       | 13824/50176 [02:22<06:19, 95.88it/s]
 28%|██▊       | 14080/50176 [02:24<06:16, 95.84it/s]
 29%|██▊       | 14336/50176 [02:27<06:17, 95.06it/s]
 29%|██▉       | 14592/50176 [02:30<06:11, 95.73it/s]
 30%|██▉       | 14848/50176 [02:33<06:16, 93.87it/s]
 30%|███       | 15104/50176 [02:35<06:14, 93.77it/s]
 31%|███       | 15360/50176 [02:38<06:00, 96.67it/s]
 31%|███       | 15616/50176 [02:40<05:43, 100.62it/s]
 32%|███▏      | 15872/50176 [02:43<05:47, 98.59it/s] 
 32%|███▏      | 16128/50176 [02:45<05:47, 97.94it/s]
 33%|███▎      | 16384/50176 [02:48<05:36, 100.33it/s]
 33%|███▎      | 16640/50176 [02:51<05:47, 96.60it/s] 
 34%|███▎      | 16896/50176 [02:53<05:33, 99.83it/s]
 34%|███▍      | 17152/50176 [02:56<05:27, 100.90it/s]
 35%|███▍      | 17408/50176 [02:58<05:19, 102.63it/s]
 35%|███▌      | 17664/50176 [03:00<05:15, 103.04it/s]
 36%|███▌      | 17920/50176 [03:03<05:12, 103.12it/s]
 36%|███▌      | 18176/50176 [03:05<05:09, 103.25it/s]
 37%|███▋      | 18432/50176 [03:08<05:02, 104.83it/s]
 37%|███▋      | 18688/50176 [03:10<05:10, 101.33it/s]
 38%|███▊      | 18944/50176 [03:13<05:11, 100.34it/s]
 38%|███▊      | 19200/50176 [03:16<05:16, 97.95it/s] 
 39%|███▉      | 19456/50176 [03:19<05:19, 96.01it/s]
 39%|███▉      | 19712/50176 [03:21<05:18, 95.69it/s]
 40%|███▉      | 19968/50176 [03:24<05:13, 96.43it/s]
 40%|████      | 20224/50176 [03:26<05:02, 99.16it/s]
 41%|████      | 20480/50176 [03:29<04:55, 100.57it/s]
 41%|████▏     | 20736/50176 [03:31<04:45, 103.11it/s]
 42%|████▏     | 20992/50176 [03:33<04:35, 105.84it/s]
 42%|████▏     | 21248/50176 [03:36<04:42, 102.29it/s]
 43%|████▎     | 21504/50176 [03:38<04:38, 103.07it/s]
 43%|████▎     | 21760/50176 [03:41<04:35, 103.11it/s]
 44%|████▍     | 22016/50176 [03:43<04:29, 104.61it/s]
 44%|████▍     | 22272/50176 [03:46<04:20, 107.00it/s]
 45%|████▍     | 22528/50176 [03:48<04:16, 107.65it/s]
 45%|████▌     | 22784/50176 [03:51<04:23, 103.76it/s]
 46%|████▌     | 23040/50176 [03:53<04:27, 101.27it/s]
 46%|████▋     | 23296/50176 [03:56<04:30, 99.40it/s] 
 47%|████▋     | 23552/50176 [03:59<04:33, 97.49it/s]
 47%|████▋     | 23808/50176 [04:01<04:21, 101.00it/s]
 48%|████▊     | 24064/50176 [04:03<04:15, 102.07it/s]
 48%|████▊     | 24320/50176 [04:06<04:15, 101.30it/s]
 49%|████▉     | 24576/50176 [04:09<04:14, 100.72it/s]
 49%|████▉     | 24832/50176 [04:11<04:01, 104.89it/s]
 50%|█████     | 25088/50176 [04:13<03:55, 106.44it/s]
 51%|█████     | 25344/50176 [04:15<03:50, 107.67it/s]
 51%|█████     | 25600/50176 [04:18<03:54, 104.74it/s]
 52%|█████▏    | 25856/50176 [04:20<03:47, 106.87it/s]
 52%|█████▏    | 26112/50176 [04:23<03:41, 108.59it/s]
 53%|█████▎    | 26368/50176 [04:25<03:44, 106.03it/s]
 53%|█████▎    | 26624/50176 [04:28<03:43, 105.40it/s]
 54%|█████▎    | 26880/50176 [04:30<03:43, 104.03it/s]
 54%|█████▍    | 27136/50176 [04:33<03:42, 103.74it/s]
 55%|█████▍    | 27392/50176 [04:35<03:37, 104.83it/s]
 55%|█████▌    | 27648/50176 [04:37<03:27, 108.45it/s]
 56%|█████▌    | 27904/50176 [04:40<03:36, 103.10it/s]
 56%|█████▌    | 28160/50176 [04:43<03:39, 100.47it/s]
 57%|█████▋    | 28416/50176 [04:45<03:34, 101.27it/s]
 57%|█████▋    | 28672/50176 [04:48<03:32, 101.29it/s]
 58%|█████▊    | 28928/50176 [04:50<03:21, 105.40it/s]
 58%|█████▊    | 29184/50176 [04:52<03:22, 103.84it/s]
 59%|█████▊    | 29440/50176 [04:55<03:29, 99.09it/s] 
 59%|█████▉    | 29696/50176 [04:58<03:25, 99.65it/s]
 60%|█████▉    | 29952/50176 [05:00<03:22, 99.71it/s]
 60%|██████    | 30208/50176 [05:03<03:17, 101.12it/s]
 61%|██████    | 30464/50176 [05:05<03:03, 107.20it/s]
 61%|██████    | 30720/50176 [05:07<03:04, 105.61it/s]
 62%|██████▏   | 30976/50176 [05:10<02:57, 108.13it/s]
 62%|██████▏   | 31232/50176 [05:12<02:53, 108.95it/s]
 63%|██████▎   | 31488/50176 [05:14<02:49, 110.13it/s]
 63%|██████▎   | 31744/50176 [05:17<02:49, 108.58it/s]
 64%|██████▍   | 32000/50176 [05:19<02:47, 108.66it/s]
 64%|██████▍   | 32256/50176 [05:21<02:43, 109.35it/s]
 65%|██████▍   | 32512/50176 [05:24<02:50, 103.86it/s]
 65%|██████▌   | 32768/50176 [05:26<02:44, 105.99it/s]
 66%|██████▌   | 33024/50176 [05:29<02:41, 105.89it/s]
 66%|██████▋   | 33280/50176 [05:31<02:36, 108.19it/s]
 67%|██████▋   | 33536/50176 [05:34<02:39, 104.51it/s]
 67%|██████▋   | 33792/50176 [05:36<02:33, 106.56it/s]
 68%|██████▊   | 34048/50176 [05:38<02:27, 109.15it/s]
 68%|██████▊   | 34304/50176 [05:40<02:22, 111.68it/s]
 69%|██████▉   | 34560/50176 [05:43<02:25, 107.45it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.64it/s]
 70%|██████▉   | 35072/50176 [05:48<02:22, 105.88it/s]
 70%|███████   | 35328/50176 [05:50<02:18, 107.55it/s]
 71%|███████   | 35584/50176 [05:52<02:15, 108.08it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.51it/s]
 72%|███████▏  | 36096/50176 [05:57<02:11, 107.06it/s]
 72%|███████▏  | 36352/50176 [06:00<02:11, 105.39it/s]
 73%|███████▎  | 36608/50176 [06:02<02:09, 105.01it/s]
 73%|███████▎  | 36864/50176 [06:05<02:09, 102.80it/s]
 74%|███████▍  | 37120/50176 [06:07<02:03, 105.58it/s]
 74%|███████▍  | 37376/50176 [06:09<01:56, 109.77it/s]
 75%|███████▌  | 37632/50176 [06:12<01:53, 110.10it/s]
 76%|███████▌  | 37888/50176 [06:14<01:54, 107.47it/s]
 76%|███████▌  | 38144/50176 [06:16<01:51, 108.26it/s]
 77%|███████▋  | 38400/50176 [06:19<01:50, 106.96it/s]
 77%|███████▋  | 38656/50176 [06:21<01:48, 105.83it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 104.24it/s]
 78%|███████▊  | 39168/50176 [06:27<01:47, 102.09it/s]
 79%|███████▊  | 39424/50176 [06:29<01:42, 105.26it/s]
 79%|███████▉  | 39680/50176 [06:31<01:40, 104.95it/s]
 80%|███████▉  | 39936/50176 [06:34<01:36, 106.48it/s]
 80%|████████  | 40192/50176 [06:36<01:33, 106.40it/s]
 81%|████████  | 40448/50176 [06:39<01:33, 103.69it/s]
 81%|████████  | 40704/50176 [06:41<01:34, 100.40it/s]
 82%|████████▏ | 40960/50176 [06:44<01:29, 103.10it/s]
 82%|████████▏ | 41216/50176 [06:46<01:25, 105.11it/s]
 83%|████████▎ | 41472/50176 [06:48<01:22, 105.35it/s]
 83%|████████▎ | 41728/50176 [06:51<01:19, 105.76it/s]
 84%|████████▎ | 41984/50176 [06:53<01:16, 107.51it/s]
 84%|████████▍ | 42240/50176 [06:56<01:14, 106.79it/s]
 85%|████████▍ | 42496/50176 [06:58<01:12, 106.24it/s]
 85%|████████▌ | 42752/50176 [07:00<01:09, 107.14it/s]
 86%|████████▌ | 43008/50176 [07:03<01:06, 107.88it/s]
 86%|████████▌ | 43264/50176 [07:05<01:04, 106.73it/s]
 87%|████████▋ | 43520/50176 [07:08<01:03, 104.31it/s]
 87%|████████▋ | 43776/50176 [07:10<01:00, 105.37it/s]
 88%|████████▊ | 44032/50176 [07:12<00:57, 106.22it/s]
 88%|████████▊ | 44288/50176 [07:15<00:54, 108.43it/s]
 89%|████████▉ | 44544/50176 [07:17<00:53, 104.73it/s]
 89%|████████▉ | 44800/50176 [07:20<00:51, 104.04it/s]
 90%|████████▉ | 45056/50176 [07:22<00:48, 106.30it/s]
 90%|█████████ | 45312/50176 [07:25<00:46, 104.37it/s]
 91%|█████████ | 45568/50176 [07:27<00:43, 106.45it/s]
 91%|█████████▏| 45824/50176 [07:29<00:41, 106.15it/s]
 92%|█████████▏| 46080/50176 [07:32<00:38, 105.20it/s]
 92%|█████████▏| 46336/50176 [07:34<00:36, 104.31it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 105.81it/s]
 93%|█████████▎| 46848/50176 [07:39<00:31, 104.90it/s]
 94%|█████████▍| 47104/50176 [07:42<00:29, 103.08it/s]
 94%|█████████▍| 47360/50176 [07:44<00:27, 101.53it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 99.58it/s] 
 95%|█████████▌| 47872/50176 [07:50<00:22, 100.54it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 103.36it/s]
 96%|█████████▋| 48384/50176 [07:54<00:17, 104.04it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 104.05it/s]
 97%|█████████▋| 48896/50176 [07:59<00:12, 104.48it/s]
 98%|█████████▊| 49152/50176 [08:02<00:09, 104.55it/s]
 98%|█████████▊| 49408/50176 [08:04<00:07, 103.17it/s]
 99%|█████████▉| 49664/50176 [08:07<00:05, 101.09it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 95.29it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 124.34it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.18it/s]
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6807	imagenet-zeroshot-val-top5: 0.9083
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_64.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_64.pt
new file mode 100644
index 0000000000000000000000000000000000000000..d54f9af0dcae4f08cbf5cbfc85411bd8ab09096a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_64.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:332bc62e7f3086f7e4eba472794b57a386f4554e8c93ca1696713f1c9555cb0e
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_64.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_64.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..eb1ba1a43a52314b639875ad6b686a4ab829c601
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_64.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_64.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_64.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_64.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_64.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_64.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:26,  1.34it/s]
  1%|          | 6/1000 [00:00<01:49,  9.08it/s]
  1%|          | 11/1000 [00:00<00:59, 16.59it/s]
  2%|▏         | 17/1000 [00:01<00:39, 25.07it/s]
  2%|▏         | 23/1000 [00:01<00:30, 32.32it/s]
  3%|▎         | 29/1000 [00:01<00:25, 38.56it/s]
  4%|▎         | 35/1000 [00:01<00:22, 43.32it/s]
  4%|▍         | 41/1000 [00:01<00:20, 47.20it/s]
  5%|▍         | 47/1000 [00:01<00:19, 49.97it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.17it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.64it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.81it/s]
  7%|▋         | 71/1000 [00:02<00:16, 55.60it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.22it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.79it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.23it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.36it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.58it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.85it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.01it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.03it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.99it/s]
 13%|█▎        | 131/1000 [00:03<00:14, 58.07it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.97it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 58.00it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.08it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.14it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.99it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.84it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.86it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.83it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.68it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.67it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.63it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.52it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.33it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.39it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.33it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.53it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.63it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.50it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.55it/s]
 25%|██▌       | 251/1000 [00:05<00:12, 57.63it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.80it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.86it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.61it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.69it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.82it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.91it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 58.14it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 58.12it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.11it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.23it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.21it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.18it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.98it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.04it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.07it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.19it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 58.01it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.11it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.07it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.19it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.02it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.82it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.87it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.01it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.11it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.13it/s]
 42%|████▏     | 419/1000 [00:08<00:09, 58.15it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.09it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.22it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.24it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.04it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.17it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.13it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.10it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.20it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.12it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.09it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.08it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 58.01it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.08it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.94it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.01it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.07it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.07it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.14it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.19it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.08it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.04it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.97it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.13it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 58.06it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 58.00it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.11it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.01it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.18it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.24it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.09it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.99it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.21it/s]
 65%|██████▌   | 653/1000 [00:12<00:05, 58.14it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.25it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.24it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.19it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.24it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.22it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.29it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.16it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 707/1000 [00:12<00:05, 58.25it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.24it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.27it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.20it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.15it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.16it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.23it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.37it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.18it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 58.25it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.23it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.26it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.29it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.09it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.08it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.10it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.10it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.08it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.15it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 58.14it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.30it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.39it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.30it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.23it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.27it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.21it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.22it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.24it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 58.10it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.19it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.24it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.19it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.19it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.10it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.22it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.30it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.28it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.25it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.25it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.36it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.41it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.34it/s]
100%|█████████▉| 995/1000 [00:17<00:00, 58.21it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.53it/s]
+2022-12-21,11:39:16 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:04<13:25, 61.97it/s]
  1%|          | 512/50176 [00:06<10:52, 76.08it/s]
  2%|▏         | 768/50176 [00:09<09:24, 87.48it/s]
  2%|▏         | 1024/50176 [00:11<08:57, 91.51it/s]
  3%|▎         | 1280/50176 [00:14<08:47, 92.64it/s]
  3%|▎         | 1536/50176 [00:17<09:15, 87.57it/s]
  4%|▎         | 1792/50176 [00:20<09:00, 89.58it/s]
  4%|▍         | 2048/50176 [00:23<08:35, 93.43it/s]
  5%|▍         | 2304/50176 [00:25<08:32, 93.45it/s]
  5%|▌         | 2560/50176 [00:28<08:11, 96.98it/s]
  6%|▌         | 2816/50176 [00:31<08:16, 95.47it/s]
  6%|▌         | 3072/50176 [00:33<08:12, 95.58it/s]
  7%|▋         | 3328/50176 [00:36<08:02, 97.04it/s]
  7%|▋         | 3584/50176 [00:38<07:49, 99.14it/s]
  8%|▊         | 3840/50176 [00:41<08:09, 94.65it/s]
  8%|▊         | 4096/50176 [00:44<08:19, 92.33it/s]
  9%|▊         | 4352/50176 [00:47<08:08, 93.84it/s]
  9%|▉         | 4608/50176 [00:49<07:43, 98.35it/s]
 10%|▉         | 4864/50176 [00:52<07:36, 99.20it/s]
 10%|█         | 5120/50176 [00:54<07:34, 99.23it/s]
 11%|█         | 5376/50176 [00:57<07:30, 99.39it/s]
 11%|█         | 5632/50176 [01:00<07:46, 95.55it/s]
 12%|█▏        | 5888/50176 [01:02<07:35, 97.31it/s]
 12%|█▏        | 6144/50176 [01:05<08:05, 90.74it/s]
 13%|█▎        | 6400/50176 [01:08<07:50, 93.09it/s]
 13%|█▎        | 6656/50176 [01:11<07:37, 95.21it/s]
 14%|█▍        | 6912/50176 [01:13<07:33, 95.43it/s]
 14%|█▍        | 7168/50176 [01:16<07:22, 97.09it/s]
 15%|█▍        | 7424/50176 [01:18<07:11, 99.01it/s]
 15%|█▌        | 7680/50176 [01:21<07:01, 100.75it/s]
 16%|█▌        | 7936/50176 [01:23<06:45, 104.21it/s]
 16%|█▋        | 8192/50176 [01:25<06:46, 103.35it/s]
 17%|█▋        | 8448/50176 [01:28<06:48, 102.25it/s]
 17%|█▋        | 8704/50176 [01:31<06:51, 100.76it/s]
 18%|█▊        | 8960/50176 [01:33<06:58, 98.44it/s] 
 18%|█▊        | 9216/50176 [01:36<06:46, 100.67it/s]
 19%|█▉        | 9472/50176 [01:38<06:46, 100.25it/s]
 19%|█▉        | 9728/50176 [01:41<06:49, 98.75it/s] 
 20%|█▉        | 9984/50176 [01:44<06:50, 97.93it/s]
 20%|██        | 10240/50176 [01:46<06:30, 102.27it/s]
 21%|██        | 10496/50176 [01:48<06:19, 104.46it/s]
 21%|██▏       | 10752/50176 [01:51<06:26, 101.99it/s]
 22%|██▏       | 11008/50176 [01:53<06:22, 102.42it/s]
 22%|██▏       | 11264/50176 [01:56<06:24, 101.27it/s]
 23%|██▎       | 11520/50176 [01:58<06:13, 103.41it/s]
 23%|██▎       | 11776/50176 [02:01<06:09, 103.96it/s]
 24%|██▍       | 12032/50176 [02:03<06:04, 104.77it/s]
 24%|██▍       | 12288/50176 [02:06<06:01, 104.81it/s]
 25%|██▌       | 12544/50176 [02:08<05:57, 105.33it/s]
 26%|██▌       | 12800/50176 [02:11<05:56, 104.98it/s]
 26%|██▌       | 13056/50176 [02:13<05:49, 106.17it/s]
 27%|██▋       | 13312/50176 [02:15<05:51, 104.88it/s]
 27%|██▋       | 13568/50176 [02:18<06:08, 99.29it/s] 
 28%|██▊       | 13824/50176 [02:21<06:20, 95.43it/s]
 28%|██▊       | 14080/50176 [02:24<06:12, 96.96it/s]
 29%|██▊       | 14336/50176 [02:27<06:15, 95.35it/s]
 29%|██▉       | 14592/50176 [02:29<06:07, 96.83it/s]
 30%|██▉       | 14848/50176 [02:32<06:19, 93.13it/s]
 30%|███       | 15104/50176 [02:35<06:11, 94.33it/s]
 31%|███       | 15360/50176 [02:37<06:02, 95.96it/s]
 31%|███       | 15616/50176 [02:40<05:54, 97.42it/s]
 32%|███▏      | 15872/50176 [02:43<05:59, 95.39it/s]
 32%|███▏      | 16128/50176 [02:45<05:50, 97.10it/s]
 33%|███▎      | 16384/50176 [02:47<05:37, 100.16it/s]
 33%|███▎      | 16640/50176 [02:50<05:50, 95.57it/s] 
 34%|███▎      | 16896/50176 [02:53<05:45, 96.27it/s]
 34%|███▍      | 17152/50176 [02:56<05:36, 98.13it/s]
 35%|███▍      | 17408/50176 [02:58<05:29, 99.53it/s]
 35%|███▌      | 17664/50176 [03:00<05:18, 101.99it/s]
 36%|███▌      | 17920/50176 [03:03<05:19, 100.96it/s]
 36%|███▌      | 18176/50176 [03:06<05:17, 100.82it/s]
 37%|███▋      | 18432/50176 [03:08<05:06, 103.64it/s]
 37%|███▋      | 18688/50176 [03:11<05:20, 98.16it/s] 
 38%|███▊      | 18944/50176 [03:13<05:14, 99.20it/s]
 38%|███▊      | 19200/50176 [03:16<05:19, 97.02it/s]
 39%|███▉      | 19456/50176 [03:19<05:19, 96.13it/s]
 39%|███▉      | 19712/50176 [03:22<05:21, 94.70it/s]
 40%|███▉      | 19968/50176 [03:24<05:14, 96.10it/s]
 40%|████      | 20224/50176 [03:26<04:51, 102.72it/s]
 41%|████      | 20480/50176 [03:29<04:47, 103.43it/s]
 41%|████▏     | 20736/50176 [03:31<04:46, 102.73it/s]
 42%|████▏     | 20992/50176 [03:34<04:37, 105.22it/s]
 42%|████▏     | 21248/50176 [03:36<04:40, 103.28it/s]
 43%|████▎     | 21504/50176 [03:38<04:33, 105.03it/s]
 43%|████▎     | 21760/50176 [03:41<04:26, 106.47it/s]
 44%|████▍     | 22016/50176 [03:43<04:22, 107.30it/s]
 44%|████▍     | 22272/50176 [03:45<04:15, 109.28it/s]
 45%|████▍     | 22528/50176 [03:48<04:14, 108.76it/s]
 45%|████▌     | 22784/50176 [03:51<04:25, 103.18it/s]
 46%|████▌     | 23040/50176 [03:53<04:24, 102.67it/s]
 46%|████▋     | 23296/50176 [03:56<04:29, 99.88it/s] 
 47%|████▋     | 23552/50176 [03:58<04:27, 99.40it/s]
 47%|████▋     | 23808/50176 [04:01<04:17, 102.26it/s]
 48%|████▊     | 24064/50176 [04:03<04:14, 102.77it/s]
 48%|████▊     | 24320/50176 [04:06<04:18, 100.13it/s]
 49%|████▉     | 24576/50176 [04:08<04:12, 101.56it/s]
 49%|████▉     | 24832/50176 [04:11<04:05, 103.24it/s]
 50%|█████     | 25088/50176 [04:13<03:59, 104.92it/s]
 51%|█████     | 25344/50176 [04:15<03:55, 105.23it/s]
 51%|█████     | 25600/50176 [04:18<03:57, 103.60it/s]
 52%|█████▏    | 25856/50176 [04:21<03:55, 103.27it/s]
 52%|█████▏    | 26112/50176 [04:23<03:49, 104.93it/s]
 53%|█████▎    | 26368/50176 [04:25<03:49, 103.66it/s]
 53%|█████▎    | 26624/50176 [04:28<03:45, 104.47it/s]
 54%|█████▎    | 26880/50176 [04:30<03:44, 103.88it/s]
 54%|█████▍    | 27136/50176 [04:33<03:43, 102.95it/s]
 55%|█████▍    | 27392/50176 [04:35<03:34, 106.08it/s]
 55%|█████▌    | 27648/50176 [04:37<03:25, 109.85it/s]
 56%|█████▌    | 27904/50176 [04:40<03:32, 104.99it/s]
 56%|█████▌    | 28160/50176 [04:43<03:33, 102.90it/s]
 57%|█████▋    | 28416/50176 [04:45<03:30, 103.25it/s]
 57%|█████▋    | 28672/50176 [04:48<03:29, 102.76it/s]
 58%|█████▊    | 28928/50176 [04:50<03:21, 105.54it/s]
 58%|█████▊    | 29184/50176 [04:52<03:19, 105.47it/s]
 59%|█████▊    | 29440/50176 [04:55<03:28, 99.25it/s] 
 59%|█████▉    | 29696/50176 [04:58<03:27, 98.63it/s]
 60%|█████▉    | 29952/50176 [05:00<03:23, 99.18it/s]
 60%|██████    | 30208/50176 [05:03<03:19, 100.17it/s]
 61%|██████    | 30464/50176 [05:05<03:07, 105.27it/s]
 61%|██████    | 30720/50176 [05:08<03:07, 103.93it/s]
 62%|██████▏   | 30976/50176 [05:10<03:00, 106.26it/s]
 62%|██████▏   | 31232/50176 [05:12<02:56, 107.24it/s]
 63%|██████▎   | 31488/50176 [05:14<02:52, 108.65it/s]
 63%|██████▎   | 31744/50176 [05:17<02:46, 110.86it/s]
 64%|██████▍   | 32000/50176 [05:19<02:46, 109.15it/s]
 64%|██████▍   | 32256/50176 [05:21<02:43, 109.72it/s]
 65%|██████▍   | 32512/50176 [05:24<02:50, 103.38it/s]
 65%|██████▌   | 32768/50176 [05:26<02:45, 105.00it/s]
 66%|██████▌   | 33024/50176 [05:29<02:43, 105.07it/s]
 66%|██████▋   | 33280/50176 [05:31<02:37, 106.99it/s]
 67%|██████▋   | 33536/50176 [05:34<02:37, 105.40it/s]
 67%|██████▋   | 33792/50176 [05:36<02:32, 107.47it/s]
 68%|██████▊   | 34048/50176 [05:38<02:30, 107.12it/s]
 68%|██████▊   | 34304/50176 [05:41<02:24, 109.84it/s]
 69%|██████▉   | 34560/50176 [05:43<02:28, 105.48it/s]
 69%|██████▉   | 34816/50176 [05:46<02:26, 104.93it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.66it/s]
 70%|███████   | 35328/50176 [05:51<02:20, 105.99it/s]
 71%|███████   | 35584/50176 [05:53<02:17, 106.44it/s]
 71%|███████▏  | 35840/50176 [05:55<02:16, 105.37it/s]
 72%|███████▏  | 36096/50176 [05:58<02:12, 105.89it/s]
 72%|███████▏  | 36352/50176 [06:00<02:11, 104.92it/s]
 73%|███████▎  | 36608/50176 [06:03<02:10, 104.02it/s]
 73%|███████▎  | 36864/50176 [06:06<02:11, 101.17it/s]
 74%|███████▍  | 37120/50176 [06:08<02:06, 102.90it/s]
 74%|███████▍  | 37376/50176 [06:10<02:02, 104.35it/s]
 75%|███████▌  | 37632/50176 [06:13<01:58, 105.70it/s]
 76%|███████▌  | 37888/50176 [06:15<01:56, 105.56it/s]
 76%|███████▌  | 38144/50176 [06:17<01:51, 108.11it/s]
 77%|███████▋  | 38400/50176 [06:20<01:50, 106.94it/s]
 77%|███████▋  | 38656/50176 [06:22<01:48, 106.14it/s]
 78%|███████▊  | 38912/50176 [06:25<01:47, 104.39it/s]
 78%|███████▊  | 39168/50176 [06:27<01:45, 104.26it/s]
 79%|███████▊  | 39424/50176 [06:30<01:41, 105.99it/s]
 79%|███████▉  | 39680/50176 [06:32<01:39, 105.56it/s]
 80%|███████▉  | 39936/50176 [06:34<01:34, 107.82it/s]
 80%|████████  | 40192/50176 [06:37<01:33, 106.29it/s]
 81%|████████  | 40448/50176 [06:39<01:34, 102.43it/s]
 81%|████████  | 40704/50176 [06:42<01:36, 98.31it/s] 
 82%|████████▏ | 40960/50176 [06:44<01:28, 103.76it/s]
 82%|████████▏ | 41216/50176 [06:47<01:25, 104.49it/s]
 83%|████████▎ | 41472/50176 [06:49<01:24, 103.59it/s]
 83%|████████▎ | 41728/50176 [06:52<01:19, 105.84it/s]
 84%|████████▎ | 41984/50176 [06:54<01:16, 106.51it/s]
 84%|████████▍ | 42240/50176 [06:56<01:13, 108.19it/s]
 85%|████████▍ | 42496/50176 [06:59<01:09, 110.37it/s]
 85%|████████▌ | 42752/50176 [07:01<01:08, 108.10it/s]
 86%|████████▌ | 43008/50176 [07:03<01:05, 108.71it/s]
 86%|████████▌ | 43264/50176 [07:06<01:04, 106.90it/s]
 87%|████████▋ | 43520/50176 [07:08<01:03, 104.52it/s]
 87%|████████▋ | 43776/50176 [07:11<01:00, 105.36it/s]
 88%|████████▊ | 44032/50176 [07:13<00:58, 105.35it/s]
 88%|████████▊ | 44288/50176 [07:15<00:53, 109.23it/s]
 89%|████████▉ | 44544/50176 [07:18<00:54, 103.32it/s]
 89%|████████▉ | 44800/50176 [07:21<00:52, 103.23it/s]
 90%|████████▉ | 45056/50176 [07:23<00:48, 105.40it/s]
 90%|█████████ | 45312/50176 [07:25<00:46, 104.37it/s]
 91%|█████████ | 45568/50176 [07:28<00:42, 108.78it/s]
 91%|█████████▏| 45824/50176 [07:30<00:40, 107.70it/s]
 92%|█████████▏| 46080/50176 [07:33<00:38, 105.06it/s]
 92%|█████████▏| 46336/50176 [07:35<00:36, 103.88it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 107.94it/s]
 93%|█████████▎| 46848/50176 [07:40<00:31, 107.23it/s]
 94%|█████████▍| 47104/50176 [07:43<00:30, 101.13it/s]
 94%|█████████▍| 47360/50176 [07:45<00:28, 100.55it/s]
 95%|█████████▍| 47616/50176 [07:48<00:25, 99.77it/s] 
 95%|█████████▌| 47872/50176 [07:50<00:22, 101.88it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 103.60it/s]
 96%|█████████▋| 48384/50176 [07:55<00:17, 103.88it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 102.88it/s]
 97%|█████████▋| 48896/50176 [08:00<00:12, 101.91it/s]
 98%|█████████▊| 49152/50176 [08:03<00:10, 100.34it/s]
 98%|█████████▊| 49408/50176 [08:05<00:07, 102.86it/s]
 99%|█████████▉| 49664/50176 [08:08<00:04, 102.90it/s]
 99%|█████████▉| 49920/50176 [08:11<00:02, 96.18it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 125.50it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.04it/s]
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6809	imagenet-zeroshot-val-top5: 0.9087
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_65.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_65.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83a8fbf17635e496395de7b9ec59b6bbac8702bd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_65.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac8b48676828434d1e2ef73e9efb3604466a984266f496d3bf61152f6af77b69
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_65.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_65.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ec1682a0c2784e5c5037bb5ce56fc43be28e7389
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_65.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_65.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_65.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_65.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_65.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_65.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:09,  1.27it/s]
  1%|          | 7/1000 [00:00<01:39,  9.96it/s]
  1%|▏         | 13/1000 [00:01<00:53, 18.39it/s]
  2%|▏         | 19/1000 [00:01<00:37, 26.39it/s]
  2%|▎         | 25/1000 [00:01<00:29, 33.36it/s]
  3%|▎         | 31/1000 [00:01<00:24, 39.26it/s]
  4%|▎         | 37/1000 [00:01<00:21, 43.94it/s]
  4%|▍         | 43/1000 [00:01<00:20, 47.53it/s]
  5%|▍         | 49/1000 [00:01<00:18, 50.19it/s]
  6%|▌         | 55/1000 [00:01<00:18, 52.10it/s]
  6%|▌         | 61/1000 [00:01<00:17, 53.67it/s]
  7%|▋         | 67/1000 [00:01<00:17, 54.70it/s]
  7%|▋         | 73/1000 [00:02<00:16, 55.45it/s]
  8%|▊         | 79/1000 [00:02<00:16, 55.99it/s]
  8%|▊         | 85/1000 [00:02<00:16, 56.49it/s]
  9%|▉         | 91/1000 [00:02<00:16, 56.81it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.21it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.37it/s]
 11%|█         | 109/1000 [00:02<00:15, 57.55it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 57.71it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.62it/s]
 13%|█▎        | 127/1000 [00:03<00:15, 57.60it/s]
 13%|█▎        | 133/1000 [00:03<00:15, 57.62it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.59it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 57.66it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 57.70it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 57.46it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 57.50it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.32it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.35it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.33it/s]
 19%|█▊        | 187/1000 [00:04<00:14, 57.35it/s]
 19%|█▉        | 193/1000 [00:04<00:14, 57.33it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.25it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.07it/s]
 21%|██        | 211/1000 [00:04<00:13, 56.89it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.07it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.08it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.29it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.27it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.10it/s]
 25%|██▍       | 247/1000 [00:05<00:13, 57.25it/s]
 25%|██▌       | 253/1000 [00:05<00:13, 57.32it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.39it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.37it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.22it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.49it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 57.62it/s]
 30%|██▉       | 295/1000 [00:05<00:13, 53.53it/s]
 30%|███       | 301/1000 [00:06<00:12, 54.71it/s]
 31%|███       | 307/1000 [00:06<00:12, 55.55it/s]
 31%|███▏      | 313/1000 [00:06<00:12, 56.32it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 56.80it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 56.95it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 57.04it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 57.32it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 57.51it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 57.56it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 57.54it/s]
 36%|███▌      | 361/1000 [00:07<00:11, 57.62it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 57.71it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 57.81it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 57.59it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.35it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.44it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 57.52it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.63it/s]
 41%|████      | 409/1000 [00:07<00:10, 57.72it/s]
 42%|████▏     | 415/1000 [00:08<00:10, 57.78it/s]
 42%|████▏     | 421/1000 [00:08<00:10, 57.78it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 57.79it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 57.90it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 57.87it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 57.62it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 57.74it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 57.77it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 57.80it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 57.74it/s]
 48%|████▊     | 475/1000 [00:09<00:09, 57.89it/s]
 48%|████▊     | 481/1000 [00:09<00:08, 57.69it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 57.80it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 57.74it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 57.64it/s]
 50%|█████     | 505/1000 [00:09<00:08, 57.72it/s]
 51%|█████     | 511/1000 [00:09<00:08, 57.64it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 57.77it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 57.75it/s]
 53%|█████▎    | 529/1000 [00:10<00:08, 57.77it/s]
 54%|█████▎    | 535/1000 [00:10<00:08, 57.82it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 57.90it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 57.92it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 57.74it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 57.81it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 57.73it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 57.66it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 57.73it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 57.83it/s]
 59%|█████▉    | 589/1000 [00:11<00:07, 57.86it/s]
 60%|█████▉    | 595/1000 [00:11<00:07, 57.79it/s]
 60%|██████    | 601/1000 [00:11<00:06, 57.86it/s]
 61%|██████    | 607/1000 [00:11<00:06, 57.72it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 57.76it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 57.88it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 57.87it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 57.84it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 57.76it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 57.75it/s]
 65%|██████▍   | 649/1000 [00:12<00:06, 57.83it/s]
 66%|██████▌   | 655/1000 [00:12<00:05, 57.84it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 57.91it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 57.90it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 57.94it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 57.92it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 57.96it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 57.94it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 57.88it/s]
 70%|███████   | 703/1000 [00:13<00:05, 57.93it/s]
 71%|███████   | 709/1000 [00:13<00:05, 57.96it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 57.93it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 57.93it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 57.86it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 57.86it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 57.91it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 57.93it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 57.98it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.01it/s]
 76%|███████▋  | 763/1000 [00:14<00:04, 57.88it/s]
 77%|███████▋  | 769/1000 [00:14<00:03, 57.96it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 57.86it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 57.89it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 57.89it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 57.75it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 57.73it/s]
 80%|████████  | 805/1000 [00:14<00:03, 57.75it/s]
 81%|████████  | 811/1000 [00:14<00:03, 57.72it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 57.81it/s]
 82%|████████▏ | 823/1000 [00:15<00:03, 57.79it/s]
 83%|████████▎ | 829/1000 [00:15<00:02, 57.91it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.05it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.08it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 57.94it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 57.98it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 57.93it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 57.91it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 57.89it/s]
 88%|████████▊ | 877/1000 [00:16<00:02, 57.94it/s]
 88%|████████▊ | 883/1000 [00:16<00:02, 57.92it/s]
 89%|████████▉ | 889/1000 [00:16<00:01, 57.84it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 57.93it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 57.91it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 57.86it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 57.82it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 57.93it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 57.90it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 57.99it/s]
 94%|█████████▎| 937/1000 [00:17<00:01, 58.02it/s]
 94%|█████████▍| 943/1000 [00:17<00:00, 57.96it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 57.93it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.07it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 57.94it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.04it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 57.98it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 57.84it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 57.89it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 57.85it/s]
100%|█████████▉| 997/1000 [00:18<00:00, 57.69it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.09it/s]
+2022-12-21,11:39:16 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:46, 70.65it/s]
  1%|          | 512/50176 [00:06<10:14, 80.76it/s]
  2%|▏         | 768/50176 [00:08<09:02, 91.12it/s]
  2%|▏         | 1024/50176 [00:11<08:43, 93.89it/s]
  3%|▎         | 1280/50176 [00:14<08:39, 94.18it/s]
  3%|▎         | 1536/50176 [00:17<09:12, 88.05it/s]
  4%|▎         | 1792/50176 [00:20<08:55, 90.33it/s]
  4%|▍         | 2048/50176 [00:22<08:32, 93.98it/s]
  5%|▍         | 2304/50176 [00:25<08:30, 93.85it/s]
  5%|▌         | 2560/50176 [00:27<08:10, 97.09it/s]
  6%|▌         | 2816/50176 [00:30<08:15, 95.50it/s]
  6%|▌         | 3072/50176 [00:33<08:11, 95.79it/s]
  7%|▋         | 3328/50176 [00:35<08:02, 97.01it/s]
  7%|▋         | 3584/50176 [00:38<07:50, 99.07it/s]
  8%|▊         | 3840/50176 [00:41<08:08, 94.92it/s]
  8%|▊         | 4096/50176 [00:44<08:19, 92.34it/s]
  9%|▊         | 4352/50176 [00:46<08:08, 93.83it/s]
  9%|▉         | 4608/50176 [00:49<07:43, 98.28it/s]
 10%|▉         | 4864/50176 [00:51<07:37, 99.14it/s]
 10%|█         | 5120/50176 [00:54<07:34, 99.14it/s]
 11%|█         | 5376/50176 [00:56<07:31, 99.29it/s]
 11%|█         | 5632/50176 [00:59<07:46, 95.51it/s]
 12%|█▏        | 5888/50176 [01:02<07:33, 97.55it/s]
 12%|█▏        | 6144/50176 [01:05<08:05, 90.70it/s]
 13%|█▎        | 6400/50176 [01:08<07:50, 93.10it/s]
 13%|█▎        | 6656/50176 [01:10<07:37, 95.22it/s]
 14%|█▍        | 6912/50176 [01:13<07:33, 95.47it/s]
 14%|█▍        | 7168/50176 [01:15<07:22, 97.14it/s]
 15%|█▍        | 7424/50176 [01:18<07:11, 99.00it/s]
 15%|█▌        | 7680/50176 [01:20<07:01, 100.78it/s]
 16%|█▌        | 7936/50176 [01:22<06:44, 104.30it/s]
 16%|█▋        | 8192/50176 [01:25<06:46, 103.29it/s]
 17%|█▋        | 8448/50176 [01:28<06:49, 101.97it/s]
 17%|█▋        | 8704/50176 [01:30<06:51, 100.73it/s]
 18%|█▊        | 8960/50176 [01:33<06:57, 98.68it/s] 
 18%|█▊        | 9216/50176 [01:35<06:46, 100.70it/s]
 19%|█▉        | 9472/50176 [01:38<06:46, 100.22it/s]
 19%|█▉        | 9728/50176 [01:41<06:49, 98.81it/s] 
 20%|█▉        | 9984/50176 [01:43<06:49, 98.04it/s]
 20%|██        | 10240/50176 [01:45<06:30, 102.18it/s]
 21%|██        | 10496/50176 [01:48<06:19, 104.49it/s]
 21%|██▏       | 10752/50176 [01:50<06:26, 102.12it/s]
 22%|██▏       | 11008/50176 [01:53<06:21, 102.57it/s]
 22%|██▏       | 11264/50176 [01:56<06:24, 101.33it/s]
 23%|██▎       | 11520/50176 [01:58<06:13, 103.38it/s]
 23%|██▎       | 11776/50176 [02:00<06:09, 103.85it/s]
 24%|██▍       | 12032/50176 [02:03<06:04, 104.55it/s]
 24%|██▍       | 12288/50176 [02:05<06:01, 104.86it/s]
 25%|██▌       | 12544/50176 [02:08<05:57, 105.32it/s]
 26%|██▌       | 12800/50176 [02:10<05:55, 105.07it/s]
 26%|██▌       | 13056/50176 [02:12<05:50, 106.04it/s]
 27%|██▋       | 13312/50176 [02:15<05:51, 104.88it/s]
 27%|██▋       | 13568/50176 [02:18<06:08, 99.28it/s] 
 28%|██▊       | 13824/50176 [02:21<06:21, 95.41it/s]
 28%|██▊       | 14080/50176 [02:23<06:13, 96.65it/s]
 29%|██▊       | 14336/50176 [02:26<06:15, 95.55it/s]
 29%|██▉       | 14592/50176 [02:29<06:07, 96.88it/s]
 30%|██▉       | 14848/50176 [02:32<06:19, 93.14it/s]
 30%|███       | 15104/50176 [02:34<06:11, 94.33it/s]
 31%|███       | 15360/50176 [02:37<06:02, 95.95it/s]
 31%|███       | 15616/50176 [02:39<05:54, 97.59it/s]
 32%|███▏      | 15872/50176 [02:42<05:59, 95.37it/s]
 32%|███▏      | 16128/50176 [02:45<05:50, 97.04it/s]
 33%|███▎      | 16384/50176 [02:47<05:37, 100.26it/s]
 33%|███▎      | 16640/50176 [02:50<05:50, 95.57it/s] 
 34%|███▎      | 16896/50176 [02:53<05:45, 96.21it/s]
 34%|███▍      | 17152/50176 [02:55<05:36, 98.18it/s]
 35%|███▍      | 17408/50176 [02:58<05:29, 99.42it/s]
 35%|███▌      | 17664/50176 [03:00<05:18, 102.01it/s]
 36%|███▌      | 17920/50176 [03:03<05:19, 100.98it/s]
 36%|███▌      | 18176/50176 [03:05<05:17, 100.78it/s]
 37%|███▋      | 18432/50176 [03:07<05:06, 103.71it/s]
 37%|███▋      | 18688/50176 [03:10<05:20, 98.35it/s] 
 38%|███▊      | 18944/50176 [03:13<05:15, 99.14it/s]
 38%|███▊      | 19200/50176 [03:16<05:19, 96.94it/s]
 39%|███▉      | 19456/50176 [03:18<05:20, 95.95it/s]
 39%|███▉      | 19712/50176 [03:21<05:21, 94.64it/s]
 40%|███▉      | 19968/50176 [03:24<05:14, 96.08it/s]
 40%|████      | 20224/50176 [03:26<04:52, 102.56it/s]
 41%|████      | 20480/50176 [03:28<04:47, 103.35it/s]
 41%|████▏     | 20736/50176 [03:31<04:46, 102.84it/s]
 42%|████▏     | 20992/50176 [03:33<04:37, 105.27it/s]
 42%|████▏     | 21248/50176 [03:36<04:39, 103.47it/s]
 43%|████▎     | 21504/50176 [03:38<04:33, 105.02it/s]
 43%|████▎     | 21760/50176 [03:40<04:27, 106.39it/s]
 44%|████▍     | 22016/50176 [03:43<04:22, 107.24it/s]
 44%|████▍     | 22272/50176 [03:45<04:15, 109.26it/s]
 45%|████▍     | 22528/50176 [03:47<04:14, 108.74it/s]
 45%|████▌     | 22784/50176 [03:50<04:25, 103.14it/s]
 46%|████▌     | 23040/50176 [03:53<04:24, 102.73it/s]
 46%|████▋     | 23296/50176 [03:55<04:28, 99.94it/s] 
 47%|████▋     | 23552/50176 [03:58<04:27, 99.36it/s]
 47%|████▋     | 23808/50176 [04:00<04:17, 102.34it/s]
 48%|████▊     | 24064/50176 [04:03<04:14, 102.80it/s]
 48%|████▊     | 24320/50176 [04:05<04:18, 100.14it/s]
 49%|████▉     | 24576/50176 [04:08<04:12, 101.57it/s]
 49%|████▉     | 24832/50176 [04:10<04:05, 103.25it/s]
 50%|█████     | 25088/50176 [04:13<03:58, 105.00it/s]
 51%|█████     | 25344/50176 [04:15<03:56, 105.17it/s]
 51%|█████     | 25600/50176 [04:18<03:57, 103.62it/s]
 52%|█████▏    | 25856/50176 [04:20<03:55, 103.13it/s]
 52%|█████▏    | 26112/50176 [04:22<03:49, 105.01it/s]
 53%|█████▎    | 26368/50176 [04:25<03:49, 103.58it/s]
 53%|█████▎    | 26624/50176 [04:27<03:45, 104.41it/s]
 54%|█████▎    | 26880/50176 [04:30<03:44, 103.92it/s]
 54%|█████▍    | 27136/50176 [04:32<03:43, 102.95it/s]
 55%|█████▍    | 27392/50176 [04:35<03:34, 106.00it/s]
 55%|█████▌    | 27648/50176 [04:37<03:24, 109.98it/s]
 56%|█████▌    | 27904/50176 [04:39<03:32, 104.90it/s]
 56%|█████▌    | 28160/50176 [04:42<03:34, 102.86it/s]
 57%|█████▋    | 28416/50176 [04:44<03:30, 103.21it/s]
 57%|█████▋    | 28672/50176 [04:47<03:29, 102.88it/s]
 58%|█████▊    | 28928/50176 [04:49<03:21, 105.57it/s]
 58%|█████▊    | 29184/50176 [04:52<03:19, 105.40it/s]
 59%|█████▊    | 29440/50176 [04:55<03:28, 99.25it/s] 
 59%|█████▉    | 29696/50176 [04:57<03:27, 98.69it/s]
 60%|█████▉    | 29952/50176 [05:00<03:24, 99.12it/s]
 60%|██████    | 30208/50176 [05:02<03:19, 100.20it/s]
 61%|██████    | 30464/50176 [05:04<03:07, 105.21it/s]
 61%|██████    | 30720/50176 [05:07<03:07, 104.04it/s]
 62%|██████▏   | 30976/50176 [05:09<03:00, 106.12it/s]
 62%|██████▏   | 31232/50176 [05:12<02:56, 107.33it/s]
 63%|██████▎   | 31488/50176 [05:14<02:52, 108.59it/s]
 63%|██████▎   | 31744/50176 [05:16<02:46, 110.98it/s]
 64%|██████▍   | 32000/50176 [05:19<02:46, 109.13it/s]
 64%|██████▍   | 32256/50176 [05:21<02:43, 109.71it/s]
 65%|██████▍   | 32512/50176 [05:24<02:50, 103.40it/s]
 65%|██████▌   | 32768/50176 [05:26<02:45, 104.92it/s]
 66%|██████▌   | 33024/50176 [05:28<02:43, 105.03it/s]
 66%|██████▋   | 33280/50176 [05:31<02:37, 107.01it/s]
 67%|██████▋   | 33536/50176 [05:33<02:37, 105.55it/s]
 67%|██████▋   | 33792/50176 [05:36<02:32, 107.35it/s]
 68%|██████▊   | 34048/50176 [05:38<02:30, 107.04it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.98it/s]
 69%|██████▉   | 34560/50176 [05:43<02:28, 105.37it/s]
 69%|██████▉   | 34816/50176 [05:45<02:26, 104.96it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.82it/s]
 70%|███████   | 35328/50176 [05:50<02:20, 105.89it/s]
 71%|███████   | 35584/50176 [05:52<02:17, 106.39it/s]
 71%|███████▏  | 35840/50176 [05:55<02:15, 105.46it/s]
 72%|███████▏  | 36096/50176 [05:57<02:13, 105.78it/s]
 72%|███████▏  | 36352/50176 [06:00<02:11, 104.98it/s]
 73%|███████▎  | 36608/50176 [06:02<02:10, 103.98it/s]
 73%|███████▎  | 36864/50176 [06:05<02:11, 101.30it/s]
 74%|███████▍  | 37120/50176 [06:07<02:07, 102.79it/s]
 74%|███████▍  | 37376/50176 [06:10<02:02, 104.36it/s]
 75%|███████▌  | 37632/50176 [06:12<01:58, 105.72it/s]
 76%|███████▌  | 37888/50176 [06:15<01:56, 105.60it/s]
 76%|███████▌  | 38144/50176 [06:17<01:51, 108.17it/s]
 77%|███████▋  | 38400/50176 [06:19<01:50, 107.00it/s]
 77%|███████▋  | 38656/50176 [06:22<01:48, 106.09it/s]
 78%|███████▊  | 38912/50176 [06:24<01:47, 104.38it/s]
 78%|███████▊  | 39168/50176 [06:27<01:45, 104.32it/s]
 79%|███████▊  | 39424/50176 [06:29<01:41, 105.87it/s]
 79%|███████▉  | 39680/50176 [06:31<01:39, 105.50it/s]
 80%|███████▉  | 39936/50176 [06:34<01:34, 107.84it/s]
 80%|████████  | 40192/50176 [06:36<01:34, 106.14it/s]
 81%|████████  | 40448/50176 [06:39<01:35, 102.40it/s]
 81%|████████  | 40704/50176 [06:42<01:36, 98.34it/s] 
 82%|████████▏ | 40960/50176 [06:44<01:28, 103.56it/s]
 82%|████████▏ | 41216/50176 [06:46<01:25, 104.45it/s]
 83%|████████▎ | 41472/50176 [06:49<01:23, 103.71it/s]
 83%|████████▎ | 41728/50176 [06:51<01:19, 105.72it/s]
 84%|████████▎ | 41984/50176 [06:54<01:16, 106.72it/s]
 84%|████████▍ | 42240/50176 [06:56<01:13, 108.04it/s]
 85%|████████▍ | 42496/50176 [06:58<01:09, 110.33it/s]
 85%|████████▌ | 42752/50176 [07:00<01:08, 108.22it/s]
 86%|████████▌ | 43008/50176 [07:03<01:05, 108.81it/s]
 86%|████████▌ | 43264/50176 [07:05<01:04, 106.91it/s]
 87%|████████▋ | 43520/50176 [07:08<01:03, 104.43it/s]
 87%|████████▋ | 43776/50176 [07:10<01:00, 105.48it/s]
 88%|████████▊ | 44032/50176 [07:13<00:58, 105.45it/s]
 88%|████████▊ | 44288/50176 [07:15<00:53, 109.17it/s]
 89%|████████▉ | 44544/50176 [07:18<00:54, 103.26it/s]
 89%|████████▉ | 44800/50176 [07:20<00:52, 103.25it/s]
 90%|████████▉ | 45056/50176 [07:22<00:48, 105.26it/s]
 90%|█████████ | 45312/50176 [07:25<00:46, 104.48it/s]
 91%|█████████ | 45568/50176 [07:27<00:42, 108.67it/s]
 91%|█████████▏| 45824/50176 [07:30<00:40, 107.57it/s]
 92%|█████████▏| 46080/50176 [07:32<00:38, 105.34it/s]
 92%|█████████▏| 46336/50176 [07:35<00:36, 103.91it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 107.86it/s]
 93%|█████████▎| 46848/50176 [07:39<00:31, 107.15it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.29it/s]
 94%|█████████▍| 47360/50176 [07:45<00:28, 100.48it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 99.74it/s] 
 95%|█████████▌| 47872/50176 [07:50<00:22, 102.05it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 103.64it/s]
 96%|█████████▋| 48384/50176 [07:54<00:17, 103.95it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 102.85it/s]
 97%|█████████▋| 48896/50176 [08:00<00:12, 101.80it/s]
 98%|█████████▊| 49152/50176 [08:02<00:10, 100.28it/s]
 98%|█████████▊| 49408/50176 [08:05<00:07, 103.01it/s]
 99%|█████████▉| 49664/50176 [08:07<00:04, 102.80it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 96.22it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 125.64it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.15it/s]
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6812	imagenet-zeroshot-val-top5: 0.9086
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_7.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_7.pt
new file mode 100644
index 0000000000000000000000000000000000000000..b188ed41b96fba95d307e779e4704bc9864b394e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_7.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:04bdc25b68930198fb3dd1a25d4f65bbe4c49e6f27300f14b24f2bbcf0f67335
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_7.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_7.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6f4eb2b4ac035bc311784b894380fe83840b9acf
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_7.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_7.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_7.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_7.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_7.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_7.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<14:07,  1.18it/s]
  1%|          | 6/1000 [00:00<02:01,  8.21it/s]
  1%|          | 12/1000 [00:01<00:59, 16.55it/s]
  2%|▏         | 18/1000 [00:01<00:39, 24.72it/s]
  2%|▏         | 24/1000 [00:01<00:30, 32.00it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.20it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.15it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.02it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.90it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.09it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.69it/s]
  7%|▋         | 66/1000 [00:02<00:17, 54.89it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.72it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.29it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.85it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.18it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.55it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.74it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.99it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.12it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.04it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 58.00it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.06it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.03it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.05it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.15it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.01it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.88it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.90it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.86it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.78it/s]
 19%|█▉        | 192/1000 [00:04<00:13, 57.76it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.72it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.53it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.41it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.54it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.72it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.69it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.60it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.69it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.79it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.88it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.92it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.73it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.78it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.95it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.05it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.25it/s]
 30%|███       | 300/1000 [00:06<00:12, 58.24it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.20it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.34it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.32it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.20it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.00it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.08it/s]
 34%|███▍      | 342/1000 [00:06<00:12, 53.82it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 54.96it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 55.70it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 56.40it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 56.96it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.40it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.54it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.40it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.61it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.78it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.98it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.11it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 58.15it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.19it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.22it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.34it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.31it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.09it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.21it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.21it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.18it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.32it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.24it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.22it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.23it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.13it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.17it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.08it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.14it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.26it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.19it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.29it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.29it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.19it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.20it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.25it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.15it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.09it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.21it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 58.20it/s]
 59%|█████▉    | 594/1000 [00:11<00:06, 58.13it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.24it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.17it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.29it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.27it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.33it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.14it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.09it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.25it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.22it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.32it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.36it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.28it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.31it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.31it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.34it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.23it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.32it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.31it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.28it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.30it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.19it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.23it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.22it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.31it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.39it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 58.22it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.32it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.18it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.21it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.25it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.05it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.04it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.07it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 58.07it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.16it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.29it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.30it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.22it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.27it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.22it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.23it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.23it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.11it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.22it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.22it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.18it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.21it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.20it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.19it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.34it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.33it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.34it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.36it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.35it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.13it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.24it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.32it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.32it/s]
100%|█████████▉| 996/1000 [00:18<00:00, 58.19it/s]
100%|██████████| 1000/1000 [00:18<00:00, 55.26it/s]
+2022-12-21,11:39:17 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:34, 71.91it/s]
  1%|          | 512/50176 [00:06<10:09, 81.52it/s]
  2%|▏         | 768/50176 [00:08<09:00, 91.39it/s]
  2%|▏         | 1024/50176 [00:11<08:42, 94.11it/s]
  3%|▎         | 1280/50176 [00:14<08:37, 94.43it/s]
  3%|▎         | 1536/50176 [00:17<09:08, 88.71it/s]
  4%|▎         | 1792/50176 [00:20<08:54, 90.47it/s]
  4%|▍         | 2048/50176 [00:22<08:31, 94.05it/s]
  5%|▍         | 2304/50176 [00:25<08:28, 94.09it/s]
  5%|▌         | 2560/50176 [00:27<08:10, 97.09it/s]
  6%|▌         | 2816/50176 [00:30<08:15, 95.62it/s]
  6%|▌         | 3072/50176 [00:33<08:11, 95.89it/s]
  7%|▋         | 3328/50176 [00:35<08:02, 97.16it/s]
  7%|▋         | 3584/50176 [00:38<07:49, 99.29it/s]
  8%|▊         | 3840/50176 [00:41<08:08, 94.79it/s]
  8%|▊         | 4096/50176 [00:44<08:19, 92.28it/s]
  9%|▊         | 4352/50176 [00:46<08:08, 93.83it/s]
  9%|▉         | 4608/50176 [00:48<07:43, 98.39it/s]
 10%|▉         | 4864/50176 [00:51<07:37, 99.13it/s]
 10%|█         | 5120/50176 [00:54<07:34, 99.20it/s]
 11%|█         | 5376/50176 [00:56<07:30, 99.39it/s]
 11%|█         | 5632/50176 [00:59<07:46, 95.54it/s]
 12%|█▏        | 5888/50176 [01:02<07:34, 97.41it/s]
 12%|█▏        | 6144/50176 [01:05<08:04, 90.86it/s]
 13%|█▎        | 6400/50176 [01:07<07:50, 93.06it/s]
 13%|█▎        | 6656/50176 [01:10<07:37, 95.22it/s]
 14%|█▍        | 6912/50176 [01:13<07:33, 95.39it/s]
 14%|█▍        | 7168/50176 [01:15<07:22, 97.14it/s]
 15%|█▍        | 7424/50176 [01:18<07:11, 98.97it/s]
 15%|█▌        | 7680/50176 [01:20<07:01, 100.74it/s]
 16%|█▌        | 7936/50176 [01:22<06:45, 104.20it/s]
 16%|█▋        | 8192/50176 [01:25<06:46, 103.36it/s]
 17%|█▋        | 8448/50176 [01:27<06:48, 102.17it/s]
 17%|█▋        | 8704/50176 [01:30<06:51, 100.75it/s]
 18%|█▊        | 8960/50176 [01:33<06:58, 98.55it/s] 
 18%|█▊        | 9216/50176 [01:35<06:46, 100.72it/s]
 19%|█▉        | 9472/50176 [01:38<06:46, 100.17it/s]
 19%|█▉        | 9728/50176 [01:40<06:49, 98.71it/s] 
 20%|█▉        | 9984/50176 [01:43<06:50, 97.93it/s]
 20%|██        | 10240/50176 [01:45<06:30, 102.32it/s]
 21%|██        | 10496/50176 [01:48<06:20, 104.34it/s]
 21%|██▏       | 10752/50176 [01:50<06:26, 102.12it/s]
 22%|██▏       | 11008/50176 [01:53<06:21, 102.54it/s]
 22%|██▏       | 11264/50176 [01:55<06:23, 101.39it/s]
 23%|██▎       | 11520/50176 [01:58<06:13, 103.39it/s]
 23%|██▎       | 11776/50176 [02:00<06:09, 103.95it/s]
 24%|██▍       | 12032/50176 [02:03<06:04, 104.71it/s]
 24%|██▍       | 12288/50176 [02:05<06:01, 104.84it/s]
 25%|██▌       | 12544/50176 [02:07<05:57, 105.33it/s]
 26%|██▌       | 12800/50176 [02:10<05:55, 105.02it/s]
 26%|██▌       | 13056/50176 [02:12<05:50, 106.00it/s]
 27%|██▋       | 13312/50176 [02:15<05:51, 104.83it/s]
 27%|██▋       | 13568/50176 [02:18<06:08, 99.39it/s] 
 28%|██▊       | 13824/50176 [02:21<06:21, 95.40it/s]
 28%|██▊       | 14080/50176 [02:23<06:13, 96.71it/s]
 29%|██▊       | 14336/50176 [02:26<06:15, 95.47it/s]
 29%|██▉       | 14592/50176 [02:28<06:07, 96.79it/s]
 30%|██▉       | 14848/50176 [02:31<06:19, 93.16it/s]
 30%|███       | 15104/50176 [02:34<06:11, 94.34it/s]
 31%|███       | 15360/50176 [02:37<06:02, 96.06it/s]
 31%|███       | 15616/50176 [02:39<05:54, 97.52it/s]
 32%|███▏      | 15872/50176 [02:42<05:59, 95.34it/s]
 32%|███▏      | 16128/50176 [02:45<05:50, 97.11it/s]
 33%|███▎      | 16384/50176 [02:47<05:37, 100.18it/s]
 33%|███▎      | 16640/50176 [02:50<05:50, 95.61it/s] 
 34%|███▎      | 16896/50176 [02:52<05:45, 96.23it/s]
 34%|███▍      | 17152/50176 [02:55<05:36, 98.09it/s]
 35%|███▍      | 17408/50176 [02:57<05:29, 99.44it/s]
 35%|███▌      | 17664/50176 [03:00<05:18, 101.96it/s]
 36%|███▌      | 17920/50176 [03:02<05:19, 100.95it/s]
 36%|███▌      | 18176/50176 [03:05<05:17, 100.77it/s]
 37%|███▋      | 18432/50176 [03:07<05:06, 103.64it/s]
 37%|███▋      | 18688/50176 [03:10<05:20, 98.37it/s] 
 38%|███▊      | 18944/50176 [03:13<05:14, 99.15it/s]
 38%|███▊      | 19200/50176 [03:15<05:19, 96.97it/s]
 39%|███▉      | 19456/50176 [03:18<05:19, 96.00it/s]
 39%|███▉      | 19712/50176 [03:21<05:21, 94.69it/s]
 40%|███▉      | 19968/50176 [03:24<05:14, 96.07it/s]
 40%|████      | 20224/50176 [03:26<04:51, 102.58it/s]
 41%|████      | 20480/50176 [03:28<04:47, 103.37it/s]
 41%|████▏     | 20736/50176 [03:31<04:46, 102.77it/s]
 42%|████▏     | 20992/50176 [03:33<04:36, 105.40it/s]
 42%|████▏     | 21248/50176 [03:36<04:39, 103.35it/s]
 43%|████▎     | 21504/50176 [03:38<04:32, 105.03it/s]
 43%|████▎     | 21760/50176 [03:40<04:26, 106.56it/s]
 44%|████▍     | 22016/50176 [03:43<04:22, 107.24it/s]
 44%|████▍     | 22272/50176 [03:45<04:15, 109.17it/s]
 45%|████▍     | 22528/50176 [03:47<04:14, 108.76it/s]
 45%|████▌     | 22784/50176 [03:50<04:25, 103.12it/s]
 46%|████▌     | 23040/50176 [03:52<04:24, 102.71it/s]
 46%|████▋     | 23296/50176 [03:55<04:29, 99.89it/s] 
 47%|████▋     | 23552/50176 [03:58<04:27, 99.35it/s]
 47%|████▋     | 23808/50176 [04:00<04:17, 102.30it/s]
 48%|████▊     | 24064/50176 [04:03<04:13, 102.83it/s]
 48%|████▊     | 24320/50176 [04:05<04:18, 100.11it/s]
 49%|████▉     | 24576/50176 [04:08<04:12, 101.58it/s]
 49%|████▉     | 24832/50176 [04:10<04:05, 103.23it/s]
 50%|█████     | 25088/50176 [04:12<03:58, 105.00it/s]
 51%|█████     | 25344/50176 [04:15<03:55, 105.32it/s]
 51%|█████     | 25600/50176 [04:17<03:57, 103.63it/s]
 52%|█████▏    | 25856/50176 [04:20<03:55, 103.18it/s]
 52%|█████▏    | 26112/50176 [04:22<03:49, 104.89it/s]
 53%|█████▎    | 26368/50176 [04:25<03:49, 103.52it/s]
 53%|█████▎    | 26624/50176 [04:27<03:45, 104.44it/s]
 54%|█████▎    | 26880/50176 [04:30<03:44, 103.88it/s]
 54%|█████▍    | 27136/50176 [04:32<03:43, 103.01it/s]
 55%|█████▍    | 27392/50176 [04:35<03:35, 105.97it/s]
 55%|█████▌    | 27648/50176 [04:37<03:24, 109.99it/s]
 56%|█████▌    | 27904/50176 [04:39<03:32, 104.88it/s]
 56%|█████▌    | 28160/50176 [04:42<03:34, 102.87it/s]
 57%|█████▋    | 28416/50176 [04:44<03:30, 103.20it/s]
 57%|█████▋    | 28672/50176 [04:47<03:28, 102.91it/s]
 58%|█████▊    | 28928/50176 [04:49<03:21, 105.48it/s]
 58%|█████▊    | 29184/50176 [04:52<03:19, 105.43it/s]
 59%|█████▊    | 29440/50176 [04:55<03:28, 99.23it/s] 
 59%|█████▉    | 29696/50176 [04:57<03:27, 98.71it/s]
 60%|█████▉    | 29952/50176 [05:00<03:24, 99.12it/s]
 60%|██████    | 30208/50176 [05:02<03:19, 100.23it/s]
 61%|██████    | 30464/50176 [05:04<03:07, 105.16it/s]
 61%|██████    | 30720/50176 [05:07<03:06, 104.07it/s]
 62%|██████▏   | 30976/50176 [05:09<03:00, 106.10it/s]
 62%|██████▏   | 31232/50176 [05:12<02:56, 107.37it/s]
 63%|██████▎   | 31488/50176 [05:14<02:52, 108.55it/s]
 63%|██████▎   | 31744/50176 [05:16<02:46, 110.85it/s]
 64%|██████▍   | 32000/50176 [05:18<02:46, 109.11it/s]
 64%|██████▍   | 32256/50176 [05:21<02:43, 109.76it/s]
 65%|██████▍   | 32512/50176 [05:24<02:50, 103.39it/s]
 65%|██████▌   | 32768/50176 [05:26<02:45, 104.97it/s]
 66%|██████▌   | 33024/50176 [05:28<02:43, 104.99it/s]
 66%|██████▋   | 33280/50176 [05:31<02:37, 107.07it/s]
 67%|██████▋   | 33536/50176 [05:33<02:37, 105.51it/s]
 67%|██████▋   | 33792/50176 [05:35<02:32, 107.39it/s]
 68%|██████▊   | 34048/50176 [05:38<02:30, 107.13it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.86it/s]
 69%|██████▉   | 34560/50176 [05:43<02:28, 105.38it/s]
 69%|██████▉   | 34816/50176 [05:45<02:26, 104.97it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.71it/s]
 70%|███████   | 35328/50176 [05:50<02:20, 105.94it/s]
 71%|███████   | 35584/50176 [05:52<02:17, 106.40it/s]
 71%|███████▏  | 35840/50176 [05:55<02:15, 105.49it/s]
 72%|███████▏  | 36096/50176 [05:57<02:13, 105.80it/s]
 72%|███████▏  | 36352/50176 [06:00<02:11, 105.01it/s]
 73%|███████▎  | 36608/50176 [06:02<02:10, 103.97it/s]
 73%|███████▎  | 36864/50176 [06:05<02:11, 101.35it/s]
 74%|███████▍  | 37120/50176 [06:07<02:07, 102.77it/s]
 74%|███████▍  | 37376/50176 [06:10<02:02, 104.37it/s]
 75%|███████▌  | 37632/50176 [06:12<01:58, 105.73it/s]
 76%|███████▌  | 37888/50176 [06:14<01:56, 105.61it/s]
 76%|███████▌  | 38144/50176 [06:17<01:51, 108.19it/s]
 77%|███████▋  | 38400/50176 [06:19<01:50, 106.88it/s]
 77%|███████▋  | 38656/50176 [06:22<01:48, 106.12it/s]
 78%|███████▊  | 38912/50176 [06:24<01:47, 104.42it/s]
 78%|███████▊  | 39168/50176 [06:27<01:45, 104.22it/s]
 79%|███████▊  | 39424/50176 [06:29<01:41, 105.94it/s]
 79%|███████▉  | 39680/50176 [06:31<01:39, 105.56it/s]
 80%|███████▉  | 39936/50176 [06:34<01:34, 107.88it/s]
 80%|████████  | 40192/50176 [06:36<01:34, 106.15it/s]
 81%|████████  | 40448/50176 [06:39<01:34, 102.44it/s]
 81%|████████  | 40704/50176 [06:42<01:36, 98.35it/s] 
 82%|████████▏ | 40960/50176 [06:44<01:28, 103.69it/s]
 82%|████████▏ | 41216/50176 [06:46<01:25, 104.39it/s]
 83%|████████▎ | 41472/50176 [06:49<01:23, 103.72it/s]
 83%|████████▎ | 41728/50176 [06:51<01:19, 105.68it/s]
 84%|████████▎ | 41984/50176 [06:53<01:16, 106.69it/s]
 84%|████████▍ | 42240/50176 [06:56<01:13, 108.03it/s]
 85%|████████▍ | 42496/50176 [06:58<01:09, 110.33it/s]
 85%|████████▌ | 42752/50176 [07:00<01:08, 108.16it/s]
 86%|████████▌ | 43008/50176 [07:03<01:05, 108.74it/s]
 86%|████████▌ | 43264/50176 [07:05<01:04, 106.87it/s]
 87%|████████▋ | 43520/50176 [07:08<01:03, 104.49it/s]
 87%|████████▋ | 43776/50176 [07:10<01:00, 105.50it/s]
 88%|████████▊ | 44032/50176 [07:13<00:58, 105.39it/s]
 88%|████████▊ | 44288/50176 [07:15<00:53, 109.21it/s]
 89%|████████▉ | 44544/50176 [07:18<00:54, 103.30it/s]
 89%|████████▉ | 44800/50176 [07:20<00:52, 103.28it/s]
 90%|████████▉ | 45056/50176 [07:22<00:48, 105.31it/s]
 90%|█████████ | 45312/50176 [07:25<00:46, 104.43it/s]
 91%|█████████ | 45568/50176 [07:27<00:42, 108.69it/s]
 91%|█████████▏| 45824/50176 [07:29<00:40, 107.55it/s]
 92%|█████████▏| 46080/50176 [07:32<00:38, 105.22it/s]
 92%|█████████▏| 46336/50176 [07:34<00:36, 103.92it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 107.87it/s]
 93%|█████████▎| 46848/50176 [07:39<00:31, 107.12it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.19it/s]
 94%|█████████▍| 47360/50176 [07:45<00:28, 100.49it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 99.76it/s] 
 95%|█████████▌| 47872/50176 [07:50<00:22, 101.93it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 103.76it/s]
 96%|█████████▋| 48384/50176 [07:54<00:17, 103.99it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 102.86it/s]
 97%|█████████▋| 48896/50176 [07:59<00:12, 101.80it/s]
 98%|█████████▊| 49152/50176 [08:02<00:10, 100.30it/s]
 98%|█████████▊| 49408/50176 [08:04<00:07, 102.94it/s]
 99%|█████████▉| 49664/50176 [08:07<00:04, 102.83it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 96.20it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 125.34it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.16it/s]
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5052	imagenet-zeroshot-val-top5: 0.7821
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_8.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_8.pt
new file mode 100644
index 0000000000000000000000000000000000000000..4a0671b0afab25a4b5912bfcb240a3e8cc87eaf5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_8.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e138b420bcfe22328f01d8489d787ec0ad8e9c7bb47366ae22af240e4363a757
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_8.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_8.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9e89262f16f7e86528fdd91dc009edf73e7fc445
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_8.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_8.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_8.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_8.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_8.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_8.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:48,  1.30it/s]
  1%|          | 6/1000 [00:00<01:51,  8.88it/s]
  1%|          | 12/1000 [00:00<00:55, 17.78it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.18it/s]
  2%|▏         | 24/1000 [00:01<00:29, 33.41it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.46it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.20it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.85it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.60it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.58it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.05it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.15it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.88it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.36it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.92it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.04it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.44it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.65it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.96it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.09it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.07it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.05it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.10it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.07it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.10it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.17it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.20it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.99it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.84it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.87it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.84it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.77it/s]
 19%|█▉        | 192/1000 [00:04<00:13, 57.73it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.66it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.48it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.40it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.53it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.70it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.70it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.58it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.65it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.70it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.82it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.68it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.54it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.84it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.98it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.16it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.15it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.15it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.30it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.30it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.18it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.02it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.10it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.21it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.29it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.09it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 58.09it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.18it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.26it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.14it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.83it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.89it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.99it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.11it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.10it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.15it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.21it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.32it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.12it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.19it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.24it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.22it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.18it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.33it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.24it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.22it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.21it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.11it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.16it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.27it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.23it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.33it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.31it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.21it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.21it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.05it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.95it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.12it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 594/1000 [00:11<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.19it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.13it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.14it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.26it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.25it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.33it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.16it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.12it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.27it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.23it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.30it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.29it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.30it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.31it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.21it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.32it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.30it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.27it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.28it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.25it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.27it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.29it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.27it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.34it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.42it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.27it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.28it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.23it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.29it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.34it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.14it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.13it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.19it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.16it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.15it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.26it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.40it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.45it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.37it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.41it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.37it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.38it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.31it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.35it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.30it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.18it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.29it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.29it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.23it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.27it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.26it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.26it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.36it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.39it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.40it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.41it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.43it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.38it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.43it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.50it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.32it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.06it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.89it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 57.68it/s]
100%|██████████| 1000/1000 [00:17<00:00, 55.58it/s]
+2022-12-21,11:39:16 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:51, 70.12it/s]
  1%|          | 512/50176 [00:06<10:14, 80.82it/s]
  2%|▏         | 768/50176 [00:08<09:02, 91.00it/s]
  2%|▏         | 1024/50176 [00:11<08:42, 94.10it/s]
  3%|▎         | 1280/50176 [00:14<08:39, 94.06it/s]
  3%|▎         | 1536/50176 [00:17<09:12, 88.07it/s]
  4%|▎         | 1792/50176 [00:20<08:56, 90.14it/s]
  4%|▍         | 2048/50176 [00:22<08:33, 93.76it/s]
  5%|▍         | 2304/50176 [00:25<08:30, 93.83it/s]
  5%|▌         | 2560/50176 [00:27<08:10, 97.11it/s]
  6%|▌         | 2816/50176 [00:30<08:16, 95.35it/s]
  6%|▌         | 3072/50176 [00:33<08:11, 95.78it/s]
  7%|▋         | 3328/50176 [00:35<08:03, 96.99it/s]
  7%|▋         | 3584/50176 [00:38<07:50, 99.11it/s]
  8%|▊         | 3840/50176 [00:41<08:09, 94.71it/s]
  8%|▊         | 4096/50176 [00:44<08:18, 92.43it/s]
  9%|▊         | 4352/50176 [00:46<08:08, 93.89it/s]
  9%|▉         | 4608/50176 [00:49<07:43, 98.29it/s]
 10%|▉         | 4864/50176 [00:51<07:36, 99.23it/s]
 10%|█         | 5120/50176 [00:54<07:34, 99.16it/s]
 11%|█         | 5376/50176 [00:56<07:31, 99.23it/s]
 11%|█         | 5632/50176 [00:59<07:45, 95.63it/s]
 12%|█▏        | 5888/50176 [01:02<07:34, 97.36it/s]
 12%|█▏        | 6144/50176 [01:05<08:05, 90.78it/s]
 13%|█▎        | 6400/50176 [01:08<07:50, 93.01it/s]
 13%|█▎        | 6656/50176 [01:10<07:36, 95.33it/s]
 14%|█▍        | 6912/50176 [01:13<07:33, 95.40it/s]
 14%|█▍        | 7168/50176 [01:15<07:22, 97.20it/s]
 15%|█▍        | 7424/50176 [01:18<07:11, 99.03it/s]
 15%|█▌        | 7680/50176 [01:20<07:01, 100.71it/s]
 16%|█▌        | 7936/50176 [01:22<06:45, 104.15it/s]
 16%|█▋        | 8192/50176 [01:25<06:46, 103.38it/s]
 17%|█▋        | 8448/50176 [01:28<06:48, 102.03it/s]
 17%|█▋        | 8704/50176 [01:30<06:52, 100.65it/s]
 18%|█▊        | 8960/50176 [01:33<06:57, 98.67it/s] 
 18%|█▊        | 9216/50176 [01:35<06:46, 100.81it/s]
 19%|█▉        | 9472/50176 [01:38<06:46, 100.22it/s]
 19%|█▉        | 9728/50176 [01:41<06:49, 98.70it/s] 
 20%|█▉        | 9984/50176 [01:43<06:50, 97.95it/s]
 20%|██        | 10240/50176 [01:46<06:30, 102.20it/s]
 21%|██        | 10496/50176 [01:48<06:20, 104.35it/s]
 21%|██▏       | 10752/50176 [01:50<06:26, 102.12it/s]
 22%|██▏       | 11008/50176 [01:53<06:22, 102.50it/s]
 22%|██▏       | 11264/50176 [01:56<06:23, 101.38it/s]
 23%|██▎       | 11520/50176 [01:58<06:13, 103.53it/s]
 23%|██▎       | 11776/50176 [02:00<06:09, 104.03it/s]
 24%|██▍       | 12032/50176 [02:03<06:04, 104.78it/s]
 24%|██▍       | 12288/50176 [02:05<06:01, 104.88it/s]
 25%|██▌       | 12544/50176 [02:08<05:57, 105.36it/s]
 26%|██▌       | 12800/50176 [02:10<05:56, 104.93it/s]
 26%|██▌       | 13056/50176 [02:12<05:50, 105.99it/s]
 27%|██▋       | 13312/50176 [02:15<05:51, 104.74it/s]
 27%|██▋       | 13568/50176 [02:18<06:08, 99.22it/s] 
 28%|██▊       | 13824/50176 [02:21<06:21, 95.41it/s]
 28%|██▊       | 14080/50176 [02:23<06:13, 96.66it/s]
 29%|██▊       | 14336/50176 [02:26<06:15, 95.43it/s]
 29%|██▉       | 14592/50176 [02:29<06:07, 96.90it/s]
 30%|██▉       | 14848/50176 [02:32<06:18, 93.27it/s]
 30%|███       | 15104/50176 [02:34<06:11, 94.37it/s]
 31%|███       | 15360/50176 [02:37<06:02, 95.95it/s]
 31%|███       | 15616/50176 [02:39<05:54, 97.53it/s]
 32%|███▏      | 15872/50176 [02:42<05:59, 95.37it/s]
 32%|███▏      | 16128/50176 [02:45<05:50, 97.09it/s]
 33%|███▎      | 16384/50176 [02:47<05:37, 100.13it/s]
 33%|███▎      | 16640/50176 [02:50<05:50, 95.60it/s] 
 34%|███▎      | 16896/50176 [02:53<05:46, 96.12it/s]
 34%|███▍      | 17152/50176 [02:55<05:36, 98.10it/s]
 35%|███▍      | 17408/50176 [02:58<05:29, 99.54it/s]
 35%|███▌      | 17664/50176 [03:00<05:19, 101.89it/s]
 36%|███▌      | 17920/50176 [03:03<05:19, 100.97it/s]
 36%|███▌      | 18176/50176 [03:05<05:17, 100.91it/s]
 37%|███▋      | 18432/50176 [03:07<05:06, 103.59it/s]
 37%|███▋      | 18688/50176 [03:10<05:19, 98.48it/s] 
 38%|███▊      | 18944/50176 [03:13<05:15, 99.09it/s]
 38%|███▊      | 19200/50176 [03:16<05:19, 97.00it/s]
 39%|███▉      | 19456/50176 [03:18<05:19, 96.00it/s]
 39%|███▉      | 19712/50176 [03:21<05:22, 94.59it/s]
 40%|███▉      | 19968/50176 [03:24<05:14, 96.08it/s]
 40%|████      | 20224/50176 [03:26<04:51, 102.61it/s]
 41%|████      | 20480/50176 [03:28<04:47, 103.43it/s]
 41%|████▏     | 20736/50176 [03:31<04:46, 102.76it/s]
 42%|████▏     | 20992/50176 [03:33<04:37, 105.24it/s]
 42%|████▏     | 21248/50176 [03:36<04:39, 103.37it/s]
 43%|████▎     | 21504/50176 [03:38<04:32, 105.09it/s]
 43%|████▎     | 21760/50176 [03:40<04:26, 106.46it/s]
 44%|████▍     | 22016/50176 [03:43<04:22, 107.24it/s]
 44%|████▍     | 22272/50176 [03:45<04:15, 109.19it/s]
 45%|████▍     | 22528/50176 [03:47<04:14, 108.78it/s]
 45%|████▌     | 22784/50176 [03:50<04:25, 103.24it/s]
 46%|████▌     | 23040/50176 [03:53<04:24, 102.65it/s]
 46%|████▋     | 23296/50176 [03:55<04:28, 100.01it/s]
 47%|████▋     | 23552/50176 [03:58<04:27, 99.38it/s] 
 47%|████▋     | 23808/50176 [04:00<04:17, 102.20it/s]
 48%|████▊     | 24064/50176 [04:03<04:14, 102.70it/s]
 48%|████▊     | 24320/50176 [04:05<04:18, 100.22it/s]
 49%|████▉     | 24576/50176 [04:08<04:12, 101.55it/s]
 49%|████▉     | 24832/50176 [04:10<04:05, 103.33it/s]
 50%|█████     | 25088/50176 [04:13<03:59, 104.84it/s]
 51%|█████     | 25344/50176 [04:15<03:56, 105.22it/s]
 51%|█████     | 25600/50176 [04:18<03:57, 103.68it/s]
 52%|█████▏    | 25856/50176 [04:20<03:55, 103.19it/s]
 52%|█████▏    | 26112/50176 [04:22<03:49, 104.97it/s]
 53%|█████▎    | 26368/50176 [04:25<03:49, 103.62it/s]
 53%|█████▎    | 26624/50176 [04:27<03:45, 104.40it/s]
 54%|█████▎    | 26880/50176 [04:30<03:44, 103.88it/s]
 54%|█████▍    | 27136/50176 [04:32<03:43, 103.02it/s]
 55%|█████▍    | 27392/50176 [04:35<03:34, 106.03it/s]
 55%|█████▌    | 27648/50176 [04:37<03:25, 109.84it/s]
 56%|█████▌    | 27904/50176 [04:39<03:32, 104.96it/s]
 56%|█████▌    | 28160/50176 [04:42<03:34, 102.85it/s]
 57%|█████▋    | 28416/50176 [04:45<03:30, 103.27it/s]
 57%|█████▋    | 28672/50176 [04:47<03:29, 102.76it/s]
 58%|█████▊    | 28928/50176 [04:49<03:21, 105.50it/s]
 58%|█████▊    | 29184/50176 [04:52<03:19, 105.43it/s]
 59%|█████▊    | 29440/50176 [04:55<03:28, 99.29it/s] 
 59%|█████▉    | 29696/50176 [04:57<03:27, 98.67it/s]
 60%|█████▉    | 29952/50176 [05:00<03:23, 99.18it/s]
 60%|██████    | 30208/50176 [05:02<03:19, 100.21it/s]
 61%|██████    | 30464/50176 [05:04<03:07, 105.28it/s]
 61%|██████    | 30720/50176 [05:07<03:07, 103.93it/s]
 62%|██████▏   | 30976/50176 [05:09<03:00, 106.19it/s]
 62%|██████▏   | 31232/50176 [05:12<02:56, 107.28it/s]
 63%|██████▎   | 31488/50176 [05:14<02:52, 108.60it/s]
 63%|██████▎   | 31744/50176 [05:16<02:46, 110.84it/s]
 64%|██████▍   | 32000/50176 [05:19<02:46, 109.12it/s]
 64%|██████▍   | 32256/50176 [05:21<02:43, 109.89it/s]
 65%|██████▍   | 32512/50176 [05:24<02:50, 103.48it/s]
 65%|██████▌   | 32768/50176 [05:26<02:45, 104.90it/s]
 66%|██████▌   | 33024/50176 [05:28<02:43, 104.98it/s]
 66%|██████▋   | 33280/50176 [05:31<02:37, 107.15it/s]
 67%|██████▋   | 33536/50176 [05:33<02:37, 105.36it/s]
 67%|██████▋   | 33792/50176 [05:36<02:32, 107.39it/s]
 68%|██████▊   | 34048/50176 [05:38<02:30, 107.12it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.84it/s]
 69%|██████▉   | 34560/50176 [05:43<02:28, 105.41it/s]
 69%|██████▉   | 34816/50176 [05:45<02:26, 105.04it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.66it/s]
 70%|███████   | 35328/50176 [05:50<02:20, 105.98it/s]
 71%|███████   | 35584/50176 [05:52<02:17, 106.37it/s]
 71%|███████▏  | 35840/50176 [05:55<02:15, 105.56it/s]
 72%|███████▏  | 36096/50176 [05:57<02:13, 105.81it/s]
 72%|███████▏  | 36352/50176 [06:00<02:11, 104.93it/s]
 73%|███████▎  | 36608/50176 [06:02<02:10, 103.93it/s]
 73%|███████▎  | 36864/50176 [06:05<02:11, 101.41it/s]
 74%|███████▍  | 37120/50176 [06:07<02:07, 102.80it/s]
 74%|███████▍  | 37376/50176 [06:10<02:02, 104.26it/s]
 75%|███████▌  | 37632/50176 [06:12<01:58, 105.66it/s]
 76%|███████▌  | 37888/50176 [06:15<01:56, 105.55it/s]
 76%|███████▌  | 38144/50176 [06:17<01:51, 108.12it/s]
 77%|███████▋  | 38400/50176 [06:19<01:50, 106.92it/s]
 77%|███████▋  | 38656/50176 [06:22<01:48, 106.11it/s]
 78%|███████▊  | 38912/50176 [06:24<01:47, 104.39it/s]
 78%|███████▊  | 39168/50176 [06:27<01:45, 104.30it/s]
 79%|███████▊  | 39424/50176 [06:29<01:41, 106.07it/s]
 79%|███████▉  | 39680/50176 [06:32<01:39, 105.51it/s]
 80%|███████▉  | 39936/50176 [06:34<01:35, 107.76it/s]
 80%|████████  | 40192/50176 [06:36<01:34, 106.20it/s]
 81%|████████  | 40448/50176 [06:39<01:34, 102.47it/s]
 81%|████████  | 40704/50176 [06:42<01:36, 98.31it/s] 
 82%|████████▏ | 40960/50176 [06:44<01:28, 103.77it/s]
 82%|████████▏ | 41216/50176 [06:46<01:25, 104.46it/s]
 83%|████████▎ | 41472/50176 [06:49<01:24, 103.58it/s]
 83%|████████▎ | 41728/50176 [06:51<01:19, 105.75it/s]
 84%|████████▎ | 41984/50176 [06:54<01:16, 106.56it/s]
 84%|████████▍ | 42240/50176 [06:56<01:13, 108.11it/s]
 85%|████████▍ | 42496/50176 [06:58<01:09, 110.35it/s]
 85%|████████▌ | 42752/50176 [07:01<01:08, 108.27it/s]
 86%|████████▌ | 43008/50176 [07:03<01:05, 108.69it/s]
 86%|████████▌ | 43264/50176 [07:05<01:04, 106.90it/s]
 87%|████████▋ | 43520/50176 [07:08<01:03, 104.45it/s]
 87%|████████▋ | 43776/50176 [07:10<01:00, 105.38it/s]
 88%|████████▊ | 44032/50176 [07:13<00:58, 105.36it/s]
 88%|████████▊ | 44288/50176 [07:15<00:53, 109.35it/s]
 89%|████████▉ | 44544/50176 [07:18<00:54, 103.36it/s]
 89%|████████▉ | 44800/50176 [07:20<00:52, 103.21it/s]
 90%|████████▉ | 45056/50176 [07:22<00:48, 105.27it/s]
 90%|█████████ | 45312/50176 [07:25<00:46, 104.51it/s]
 91%|█████████ | 45568/50176 [07:27<00:42, 108.66it/s]
 91%|█████████▏| 45824/50176 [07:30<00:40, 107.59it/s]
 92%|█████████▏| 46080/50176 [07:32<00:38, 105.18it/s]
 92%|█████████▏| 46336/50176 [07:35<00:36, 103.98it/s]
 93%|█████████▎| 46592/50176 [07:37<00:33, 107.82it/s]
 93%|█████████▎| 46848/50176 [07:39<00:31, 107.15it/s]
 94%|█████████▍| 47104/50176 [07:42<00:30, 101.38it/s]
 94%|█████████▍| 47360/50176 [07:45<00:28, 100.44it/s]
 95%|█████████▍| 47616/50176 [07:47<00:25, 99.72it/s] 
 95%|█████████▌| 47872/50176 [07:50<00:22, 101.90it/s]
 96%|█████████▌| 48128/50176 [07:52<00:19, 103.64it/s]
 96%|█████████▋| 48384/50176 [07:54<00:17, 103.88it/s]
 97%|█████████▋| 48640/50176 [07:57<00:14, 102.98it/s]
 97%|█████████▋| 48896/50176 [08:00<00:12, 101.80it/s]
 98%|█████████▊| 49152/50176 [08:02<00:10, 100.33it/s]
 98%|█████████▊| 49408/50176 [08:05<00:07, 102.90it/s]
 99%|█████████▉| 49664/50176 [08:07<00:04, 102.84it/s]
 99%|█████████▉| 49920/50176 [08:10<00:02, 96.27it/s] 
100%|██████████| 50176/50176 [08:11<00:00, 125.41it/s]
100%|██████████| 50176/50176 [08:11<00:00, 102.14it/s]
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5092	imagenet-zeroshot-val-top5: 0.7909
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_9.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_9.pt
new file mode 100644
index 0000000000000000000000000000000000000000..f9a327a57408712a12455e4af360dd62dbdb3476
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_9.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99161e32075327280a3693a1610f78b26f20c725ab5f51b89fc07cf565eaf2c7
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_9.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_9.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6df1b6741ae2190d8d663574df55b935b346ecc6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_9.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:47:43 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:47:43 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:47:44 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_9.pt).
+2022-12-21,11:47:51 | INFO | Model:
+2022-12-21,11:47:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:47:51 | INFO | Params:
+2022-12-21,11:47:51 | INFO |   batch_size: 256
+2022-12-21,11:47:51 | INFO |   beta1: 0.9
+2022-12-21,11:47:51 | INFO |   beta2: 0.98
+2022-12-21,11:47:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_9.pt/checkpoints
+2022-12-21,11:47:51 | INFO |   copy_codebase: False
+2022-12-21,11:47:51 | INFO |   csv_caption_key: title
+2022-12-21,11:47:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:47:51 | INFO |   csv_separator: 	
+2022-12-21,11:47:51 | INFO |   dataset_resampled: False
+2022-12-21,11:47:51 | INFO |   dataset_type: auto
+2022-12-21,11:47:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:47:51 | INFO |   debug: False
+2022-12-21,11:47:51 | INFO |   device: cuda:0
+2022-12-21,11:47:51 | INFO |   dist_backend: nccl
+2022-12-21,11:47:51 | INFO |   dist_url: env://
+2022-12-21,11:47:51 | INFO |   distributed: False
+2022-12-21,11:47:51 | INFO |   epochs: 32
+2022-12-21,11:47:51 | INFO |   eps: 1e-06
+2022-12-21,11:47:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:47:51 | INFO |   gather_with_grad: False
+2022-12-21,11:47:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:47:51 | INFO |   horovod: False
+2022-12-21,11:47:51 | INFO |   image_mean: None
+2022-12-21,11:47:51 | INFO |   image_std: None
+2022-12-21,11:47:51 | INFO |   imagenet_v2: None
+2022-12-21,11:47:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:47:51 | INFO |   local_loss: False
+2022-12-21,11:47:51 | INFO |   local_rank: 0
+2022-12-21,11:47:51 | INFO |   lock_image: False
+2022-12-21,11:47:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:47:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:47:51 | INFO |   log_level: 20
+2022-12-21,11:47:51 | INFO |   log_local: False
+2022-12-21,11:47:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_9.pt/out.log
+2022-12-21,11:47:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:47:51 | INFO |   lr: 0.0005
+2022-12-21,11:47:51 | INFO |   model: ViT-B-16
+2022-12-21,11:47:51 | INFO |   name: epoch_9.pt
+2022-12-21,11:47:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:47:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:47:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:47:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_9.pt
+2022-12-21,11:47:51 | INFO |   pretrained_image: False
+2022-12-21,11:47:51 | INFO |   rank: 0
+2022-12-21,11:47:51 | INFO |   report_to: 
+2022-12-21,11:47:51 | INFO |   resume: None
+2022-12-21,11:47:51 | INFO |   save_frequency: 1
+2022-12-21,11:47:51 | INFO |   save_most_recent: False
+2022-12-21,11:47:51 | INFO |   seed: 0
+2022-12-21,11:47:51 | INFO |   skip_scheduler: False
+2022-12-21,11:47:51 | INFO |   tensorboard: False
+2022-12-21,11:47:51 | INFO |   tensorboard_path: 
+2022-12-21,11:47:51 | INFO |   torchscript: False
+2022-12-21,11:47:51 | INFO |   trace: False
+2022-12-21,11:47:51 | INFO |   train_data: None
+2022-12-21,11:47:51 | INFO |   train_num_samples: None
+2022-12-21,11:47:51 | INFO |   use_bn_sync: False
+2022-12-21,11:47:51 | INFO |   val_data: None
+2022-12-21,11:47:51 | INFO |   val_frequency: 1
+2022-12-21,11:47:51 | INFO |   val_num_samples: None
+2022-12-21,11:47:51 | INFO |   wandb: False
+2022-12-21,11:47:51 | INFO |   wandb_notes: 
+2022-12-21,11:47:51 | INFO |   warmup: 10000
+2022-12-21,11:47:51 | INFO |   wd: 0.2
+2022-12-21,11:47:51 | INFO |   workers: 1
+2022-12-21,11:47:51 | INFO |   world_size: 1
+2022-12-21,11:47:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:47:52 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:47:52 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<08:57,  1.86it/s]
  1%|          | 6/1000 [00:00<01:26, 11.51it/s]
  1%|          | 12/1000 [00:00<00:45, 21.66it/s]
  2%|▏         | 18/1000 [00:00<00:32, 29.92it/s]
  2%|▏         | 24/1000 [00:00<00:26, 36.99it/s]
  3%|▎         | 30/1000 [00:01<00:22, 42.53it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.59it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.56it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.83it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.46it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.58it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.46it/s]
  7%|▋         | 72/1000 [00:01<00:16, 56.06it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.49it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.94it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.19it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.52it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.66it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.90it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.98it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.95it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.87it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.91it/s]
 14%|█▍        | 138/1000 [00:02<00:14, 57.87it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.89it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.99it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.87it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.73it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.75it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.49it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.49it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.23it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.29it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.18it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.07it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.24it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.24it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.48it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.51it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.42it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.29it/s]
 25%|██▌       | 252/1000 [00:04<00:13, 57.43it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.45it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.55it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.43it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.48it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.69it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.79it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.02it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.98it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.96it/s]
 31%|███       | 312/1000 [00:05<00:11, 58.12it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.10it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.02it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.82it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.91it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.01it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.09it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.89it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.90it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.97it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.06it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.92it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.63it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.70it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.80it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.93it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.99it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.03it/s]
 43%|████▎     | 426/1000 [00:07<00:09, 58.07it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.20it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.19it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.98it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.02it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.06it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.00it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.14it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.05it/s]
 49%|████▊     | 486/1000 [00:08<00:08, 58.03it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.03it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.95it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.99it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.96it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.99it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.10it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.09it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.01it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.06it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.94it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.88it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.01it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.01it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.94it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.02it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.90it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.95it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.09it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.08it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.15it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.96it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.92it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.08it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.03it/s]
 66%|██████▌   | 660/1000 [00:11<00:05, 58.13it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.14it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.06it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.06it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.09it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.11it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.02it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.13it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.12it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.10it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.11it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.08it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.08it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.15it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.23it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.07it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.15it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.09it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.14it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.16it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.97it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.99it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.05it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.00it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.97it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.99it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.09it/s]
 83%|████████▎ | 834/1000 [00:14<00:02, 58.20it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.22it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.14it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.20it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.13it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.13it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.06it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.11it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.00it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.11it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.05it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.08it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.08it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.20it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.19it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.21it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.23it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.29it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.31it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.28it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.24it/s]
100%|█████████▉| 996/1000 [00:17<00:00, 58.06it/s]
100%|██████████| 1000/1000 [00:17<00:00, 56.07it/s]
+2022-12-21,11:48:09 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:05<16:21, 50.87it/s]
  1%|          | 512/50176 [00:07<11:51, 69.83it/s]
  2%|▏         | 768/50176 [00:10<09:53, 83.18it/s]
  2%|▏         | 1024/50176 [00:12<09:08, 89.59it/s]
  3%|▎         | 1280/50176 [00:15<08:47, 92.73it/s]
  3%|▎         | 1536/50176 [00:18<08:59, 90.19it/s]
  4%|▎         | 1792/50176 [00:20<08:41, 92.78it/s]
  4%|▍         | 2048/50176 [00:23<08:19, 96.44it/s]
  5%|▍         | 2304/50176 [00:25<08:05, 98.51it/s]
  5%|▌         | 2560/50176 [00:27<07:44, 102.41it/s]
  6%|▌         | 2816/50176 [00:30<08:04, 97.76it/s] 
  6%|▌         | 3072/50176 [00:33<07:59, 98.27it/s]
  7%|▋         | 3328/50176 [00:35<07:46, 100.47it/s]
  7%|▋         | 3584/50176 [00:38<07:33, 102.80it/s]
  8%|▊         | 3840/50176 [00:40<07:45, 99.62it/s] 
  8%|▊         | 4096/50176 [00:43<08:00, 95.94it/s]
  9%|▊         | 4352/50176 [00:46<07:56, 96.13it/s]
  9%|▉         | 4608/50176 [00:48<07:42, 98.53it/s]
 10%|▉         | 4864/50176 [00:51<07:28, 101.00it/s]
 10%|█         | 5120/50176 [00:53<07:23, 101.50it/s]
 11%|█         | 5376/50176 [00:56<07:38, 97.65it/s] 
 11%|█         | 5632/50176 [00:59<07:38, 97.05it/s]
 12%|█▏        | 5888/50176 [01:01<07:33, 97.70it/s]
 12%|█▏        | 6144/50176 [01:05<07:58, 92.11it/s]
 13%|█▎        | 6400/50176 [01:07<07:40, 95.16it/s]
 13%|█▎        | 6656/50176 [01:10<07:29, 96.76it/s]
 14%|█▍        | 6912/50176 [01:12<07:27, 96.65it/s]
 14%|█▍        | 7168/50176 [01:15<07:15, 98.77it/s]
 15%|█▍        | 7424/50176 [01:17<07:02, 101.16it/s]
 15%|█▌        | 7680/50176 [01:20<06:58, 101.60it/s]
 16%|█▌        | 7936/50176 [01:22<06:43, 104.68it/s]
 16%|█▋        | 8192/50176 [01:24<06:42, 104.23it/s]
 17%|█▋        | 8448/50176 [01:27<06:41, 103.93it/s]
 17%|█▋        | 8704/50176 [01:29<06:43, 102.72it/s]
 18%|█▊        | 8960/50176 [01:32<06:43, 102.03it/s]
 18%|█▊        | 9216/50176 [01:34<06:32, 104.24it/s]
 19%|█▉        | 9472/50176 [01:37<06:30, 104.17it/s]
 19%|█▉        | 9728/50176 [01:39<06:37, 101.80it/s]
 20%|█▉        | 9984/50176 [01:42<06:38, 100.92it/s]
 20%|██        | 10240/50176 [01:44<06:16, 106.12it/s]
 21%|██        | 10496/50176 [01:46<06:06, 108.29it/s]
 21%|██▏       | 10752/50176 [01:49<06:19, 103.85it/s]
 22%|██▏       | 11008/50176 [01:52<06:25, 101.54it/s]
 22%|██▏       | 11264/50176 [01:54<06:23, 101.41it/s]
 23%|██▎       | 11520/50176 [01:57<06:19, 101.75it/s]
 23%|██▎       | 11776/50176 [01:59<06:15, 102.28it/s]
 24%|██▍       | 12032/50176 [02:02<06:08, 103.51it/s]
 24%|██▍       | 12288/50176 [02:04<06:17, 100.41it/s]
 25%|██▌       | 12544/50176 [02:07<06:07, 102.46it/s]
 26%|██▌       | 12800/50176 [02:09<06:03, 102.71it/s]
 26%|██▌       | 13056/50176 [02:12<05:56, 104.03it/s]
 27%|██▋       | 13312/50176 [02:14<05:53, 104.39it/s]
 27%|██▋       | 13568/50176 [02:17<06:00, 101.61it/s]
 28%|██▊       | 13824/50176 [02:20<06:15, 96.72it/s] 
 28%|██▊       | 14080/50176 [02:22<06:10, 97.53it/s]
 29%|██▊       | 14336/50176 [02:25<06:07, 97.60it/s]
 29%|██▉       | 14592/50176 [02:27<05:52, 101.01it/s]
 30%|██▉       | 14848/50176 [02:30<06:02, 97.53it/s] 
 30%|███       | 15104/50176 [02:33<06:01, 96.98it/s]
 31%|███       | 15360/50176 [02:35<05:59, 96.79it/s]
 31%|███       | 15616/50176 [02:38<05:44, 100.42it/s]
 32%|███▏      | 15872/50176 [02:40<05:47, 98.73it/s] 
 32%|███▏      | 16128/50176 [02:43<05:45, 98.67it/s]
 33%|███▎      | 16384/50176 [02:45<05:34, 101.16it/s]
 33%|███▎      | 16640/50176 [02:48<05:38, 99.21it/s] 
 34%|███▎      | 16896/50176 [02:51<05:30, 100.56it/s]
 34%|███▍      | 17152/50176 [02:53<05:24, 101.71it/s]
 35%|███▍      | 17408/50176 [02:56<05:27, 100.06it/s]
 35%|███▌      | 17664/50176 [02:58<05:21, 101.18it/s]
 36%|███▌      | 17920/50176 [03:00<05:13, 102.96it/s]
 36%|███▌      | 18176/50176 [03:03<05:09, 103.36it/s]
 37%|███▋      | 18432/50176 [03:05<04:59, 106.08it/s]
 37%|███▋      | 18688/50176 [03:08<05:05, 103.15it/s]
 38%|███▊      | 18944/50176 [03:10<05:05, 102.13it/s]
 38%|███▊      | 19200/50176 [03:13<05:10, 99.86it/s] 
 39%|███▉      | 19456/50176 [03:16<05:15, 97.36it/s]
 39%|███▉      | 19712/50176 [03:19<05:14, 96.86it/s]
 40%|███▉      | 19968/50176 [03:21<05:10, 97.44it/s]
 40%|████      | 20224/50176 [03:23<04:50, 102.93it/s]
 41%|████      | 20480/50176 [03:26<04:48, 102.91it/s]
 41%|████▏     | 20736/50176 [03:28<04:44, 103.38it/s]
 42%|████▏     | 20992/50176 [03:31<04:37, 105.24it/s]
 42%|████▏     | 21248/50176 [03:33<04:44, 101.83it/s]
 43%|████▎     | 21504/50176 [03:36<04:37, 103.50it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.58it/s]
 44%|████▍     | 22016/50176 [03:40<04:26, 105.73it/s]
 44%|████▍     | 22272/50176 [03:43<04:17, 108.46it/s]
 45%|████▍     | 22528/50176 [03:45<04:13, 109.12it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.17it/s]
 46%|████▌     | 23040/50176 [03:50<04:26, 101.65it/s]
 46%|████▋     | 23296/50176 [03:53<04:27, 100.44it/s]
 47%|████▋     | 23552/50176 [03:56<04:29, 98.63it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.25it/s]
 48%|████▊     | 24064/50176 [04:00<04:15, 102.02it/s]
 48%|████▊     | 24320/50176 [04:03<04:12, 102.43it/s]
 49%|████▉     | 24576/50176 [04:06<04:11, 101.72it/s]
 49%|████▉     | 24832/50176 [04:08<04:06, 102.80it/s]
 50%|█████     | 25088/50176 [04:10<03:56, 106.12it/s]
 51%|█████     | 25344/50176 [04:12<03:48, 108.63it/s]
 51%|█████     | 25600/50176 [04:15<03:47, 108.05it/s]
 52%|█████▏    | 25856/50176 [04:17<03:48, 106.56it/s]
 52%|█████▏    | 26112/50176 [04:20<03:46, 106.40it/s]
 53%|█████▎    | 26368/50176 [04:22<03:49, 103.77it/s]
 53%|█████▎    | 26624/50176 [04:25<03:45, 104.59it/s]
 54%|█████▎    | 26880/50176 [04:27<03:44, 103.82it/s]
 54%|█████▍    | 27136/50176 [04:30<03:43, 103.00it/s]
 55%|█████▍    | 27392/50176 [04:32<03:37, 104.72it/s]
 55%|█████▌    | 27648/50176 [04:34<03:26, 109.04it/s]
 56%|█████▌    | 27904/50176 [04:37<03:36, 102.68it/s]
 56%|█████▌    | 28160/50176 [04:40<03:38, 100.62it/s]
 57%|█████▋    | 28416/50176 [04:42<03:32, 102.21it/s]
 57%|█████▋    | 28672/50176 [04:45<03:30, 102.32it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.42it/s]
 58%|█████▊    | 29184/50176 [04:50<03:25, 102.03it/s]
 59%|█████▊    | 29440/50176 [04:53<03:33, 97.32it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:29, 97.81it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.99it/s]
 60%|██████    | 30208/50176 [05:00<03:21, 99.28it/s]
 61%|██████    | 30464/50176 [05:02<03:08, 104.52it/s]
 61%|██████    | 30720/50176 [05:05<03:06, 104.06it/s]
 62%|██████▏   | 30976/50176 [05:07<02:59, 107.03it/s]
 62%|██████▏   | 31232/50176 [05:09<02:54, 108.40it/s]
 63%|██████▎   | 31488/50176 [05:12<02:50, 109.66it/s]
 63%|██████▎   | 31744/50176 [05:14<02:47, 109.77it/s]
 64%|██████▍   | 32000/50176 [05:16<02:43, 111.23it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.66it/s]
 65%|██████▍   | 32512/50176 [05:21<02:47, 105.37it/s]
 65%|██████▌   | 32768/50176 [05:24<02:44, 105.97it/s]
 66%|██████▌   | 33024/50176 [05:26<02:41, 106.00it/s]
 66%|██████▋   | 33280/50176 [05:28<02:38, 106.48it/s]
 67%|██████▋   | 33536/50176 [05:31<02:37, 105.33it/s]
 67%|██████▋   | 33792/50176 [05:33<02:32, 107.29it/s]
 68%|██████▊   | 34048/50176 [05:35<02:26, 109.74it/s]
 68%|██████▊   | 34304/50176 [05:38<02:23, 110.55it/s]
 69%|██████▉   | 34560/50176 [05:40<02:25, 107.16it/s]
 69%|██████▉   | 34816/50176 [05:43<02:24, 106.08it/s]
 70%|██████▉   | 35072/50176 [05:45<02:23, 105.55it/s]
 70%|███████   | 35328/50176 [05:48<02:19, 106.82it/s]
 71%|███████   | 35584/50176 [05:50<02:14, 108.66it/s]
 71%|███████▏  | 35840/50176 [05:52<02:15, 105.44it/s]
 72%|███████▏  | 36096/50176 [05:55<02:11, 107.01it/s]
 72%|███████▏  | 36352/50176 [05:57<02:11, 104.79it/s]
 73%|███████▎  | 36608/50176 [06:00<02:10, 103.90it/s]
 73%|███████▎  | 36864/50176 [06:02<02:11, 100.91it/s]
 74%|███████▍  | 37120/50176 [06:05<02:07, 102.71it/s]
 74%|███████▍  | 37376/50176 [06:07<01:59, 107.15it/s]
 75%|███████▌  | 37632/50176 [06:09<01:56, 107.61it/s]
 76%|███████▌  | 37888/50176 [06:12<01:52, 108.79it/s]
 76%|███████▌  | 38144/50176 [06:14<01:50, 109.16it/s]
 77%|███████▋  | 38400/50176 [06:16<01:50, 107.03it/s]
 77%|███████▋  | 38656/50176 [06:19<01:48, 106.48it/s]
 78%|███████▊  | 38912/50176 [06:21<01:44, 107.42it/s]
 78%|███████▊  | 39168/50176 [06:24<01:44, 104.86it/s]
 79%|███████▊  | 39424/50176 [06:26<01:40, 106.49it/s]
 79%|███████▉  | 39680/50176 [06:29<01:38, 106.61it/s]
 80%|███████▉  | 39936/50176 [06:31<01:34, 108.04it/s]
 80%|████████  | 40192/50176 [06:33<01:32, 108.27it/s]
 81%|████████  | 40448/50176 [06:36<01:34, 102.95it/s]
 81%|████████  | 40704/50176 [06:39<01:35, 99.02it/s] 
 82%|████████▏ | 40960/50176 [06:41<01:28, 104.21it/s]
 82%|████████▏ | 41216/50176 [06:43<01:25, 105.14it/s]
 83%|████████▎ | 41472/50176 [06:46<01:22, 105.80it/s]
 83%|████████▎ | 41728/50176 [06:48<01:18, 107.57it/s]
 84%|████████▎ | 41984/50176 [06:50<01:16, 107.47it/s]
 84%|████████▍ | 42240/50176 [06:53<01:15, 105.81it/s]
 85%|████████▍ | 42496/50176 [06:55<01:10, 109.16it/s]
 85%|████████▌ | 42752/50176 [06:57<01:08, 108.27it/s]
 86%|████████▌ | 43008/50176 [07:00<01:07, 106.65it/s]
 86%|████████▌ | 43264/50176 [07:02<01:06, 104.71it/s]
 87%|████████▋ | 43520/50176 [07:05<01:03, 104.19it/s]
 87%|████████▋ | 43776/50176 [07:07<01:01, 104.22it/s]
 88%|████████▊ | 44032/50176 [07:10<00:58, 105.21it/s]
 88%|████████▊ | 44288/50176 [07:12<00:54, 108.09it/s]
 89%|████████▉ | 44544/50176 [07:15<00:54, 103.53it/s]
 89%|████████▉ | 44800/50176 [07:17<00:52, 102.40it/s]
 90%|████████▉ | 45056/50176 [07:20<00:48, 105.25it/s]
 90%|█████████ | 45312/50176 [07:22<00:47, 103.29it/s]
 91%|█████████ | 45568/50176 [07:24<00:42, 107.32it/s]
 91%|█████████▏| 45824/50176 [07:27<00:41, 103.91it/s]
 92%|█████████▏| 46080/50176 [07:29<00:39, 103.79it/s]
 92%|█████████▏| 46336/50176 [07:32<00:37, 103.51it/s]
 93%|█████████▎| 46592/50176 [07:34<00:33, 106.09it/s]
 93%|█████████▎| 46848/50176 [07:37<00:31, 105.54it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 100.17it/s]
 94%|█████████▍| 47360/50176 [07:42<00:28, 99.54it/s] 
 95%|█████████▍| 47616/50176 [07:45<00:25, 100.60it/s]
 95%|█████████▌| 47872/50176 [07:47<00:22, 101.54it/s]
 96%|█████████▌| 48128/50176 [07:49<00:19, 103.52it/s]
 96%|█████████▋| 48384/50176 [07:52<00:17, 103.38it/s]
 97%|█████████▋| 48640/50176 [07:54<00:14, 105.04it/s]
 97%|█████████▋| 48896/50176 [07:57<00:12, 105.45it/s]
 98%|█████████▊| 49152/50176 [07:59<00:10, 102.01it/s]
 98%|█████████▊| 49408/50176 [08:02<00:07, 100.80it/s]
 99%|█████████▉| 49664/50176 [08:05<00:05, 100.32it/s]
 99%|█████████▉| 49920/50176 [08:07<00:02, 96.33it/s] 
100%|██████████| 50176/50176 [08:08<00:00, 126.31it/s]
100%|██████████| 50176/50176 [08:08<00:00, 102.70it/s]
+2022-12-21,11:56:18 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:56:18 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5230	imagenet-zeroshot-val-top5: 0.8007
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_latest.pt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_latest.pt
new file mode 100644
index 0000000000000000000000000000000000000000..83a8fbf17635e496395de7b9ec59b6bbac8702bd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_latest.pt
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac8b48676828434d1e2ef73e9efb3604466a984266f496d3bf61152f6af77b69
+size 1795822561
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_latest.pt.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_latest.pt.txt
new file mode 100644
index 0000000000000000000000000000000000000000..adc5728a8321e341fae9f8268059b6980f07ad07
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_latest.pt.txt
@@ -0,0 +1,432 @@
+2022-12-21,11:47:43 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:47:43 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:47:44 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_latest.pt).
+2022-12-21,11:47:52 | INFO | Model:
+2022-12-21,11:47:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:47:52 | INFO | Params:
+2022-12-21,11:47:52 | INFO |   batch_size: 256
+2022-12-21,11:47:52 | INFO |   beta1: 0.9
+2022-12-21,11:47:52 | INFO |   beta2: 0.98
+2022-12-21,11:47:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_latest.pt/checkpoints
+2022-12-21,11:47:52 | INFO |   copy_codebase: False
+2022-12-21,11:47:52 | INFO |   csv_caption_key: title
+2022-12-21,11:47:52 | INFO |   csv_img_key: filepath
+2022-12-21,11:47:52 | INFO |   csv_separator: 	
+2022-12-21,11:47:52 | INFO |   dataset_resampled: False
+2022-12-21,11:47:52 | INFO |   dataset_type: auto
+2022-12-21,11:47:52 | INFO |   ddp_static_graph: False
+2022-12-21,11:47:52 | INFO |   debug: False
+2022-12-21,11:47:52 | INFO |   device: cuda:0
+2022-12-21,11:47:52 | INFO |   dist_backend: nccl
+2022-12-21,11:47:52 | INFO |   dist_url: env://
+2022-12-21,11:47:52 | INFO |   distributed: False
+2022-12-21,11:47:52 | INFO |   epochs: 32
+2022-12-21,11:47:52 | INFO |   eps: 1e-06
+2022-12-21,11:47:52 | INFO |   force_quick_gelu: False
+2022-12-21,11:47:52 | INFO |   gather_with_grad: False
+2022-12-21,11:47:52 | INFO |   grad_checkpointing: False
+2022-12-21,11:47:52 | INFO |   horovod: False
+2022-12-21,11:47:52 | INFO |   image_mean: None
+2022-12-21,11:47:52 | INFO |   image_std: None
+2022-12-21,11:47:52 | INFO |   imagenet_v2: None
+2022-12-21,11:47:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:47:52 | INFO |   local_loss: False
+2022-12-21,11:47:52 | INFO |   local_rank: 0
+2022-12-21,11:47:52 | INFO |   lock_image: False
+2022-12-21,11:47:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:47:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:47:52 | INFO |   log_level: 20
+2022-12-21,11:47:52 | INFO |   log_local: False
+2022-12-21,11:47:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_latest.pt/out.log
+2022-12-21,11:47:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:47:52 | INFO |   lr: 0.0005
+2022-12-21,11:47:52 | INFO |   model: ViT-B-16
+2022-12-21,11:47:52 | INFO |   name: epoch_latest.pt
+2022-12-21,11:47:52 | INFO |   no_set_device_rank: False
+2022-12-21,11:47:52 | INFO |   norm_gradient_clip: None
+2022-12-21,11:47:52 | INFO |   precision: amp_bfloat16
+2022-12-21,11:47:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_latest.pt
+2022-12-21,11:47:52 | INFO |   pretrained_image: False
+2022-12-21,11:47:52 | INFO |   rank: 0
+2022-12-21,11:47:52 | INFO |   report_to: 
+2022-12-21,11:47:52 | INFO |   resume: None
+2022-12-21,11:47:52 | INFO |   save_frequency: 1
+2022-12-21,11:47:52 | INFO |   save_most_recent: False
+2022-12-21,11:47:52 | INFO |   seed: 0
+2022-12-21,11:47:52 | INFO |   skip_scheduler: False
+2022-12-21,11:47:52 | INFO |   tensorboard: False
+2022-12-21,11:47:52 | INFO |   tensorboard_path: 
+2022-12-21,11:47:52 | INFO |   torchscript: False
+2022-12-21,11:47:52 | INFO |   trace: False
+2022-12-21,11:47:52 | INFO |   train_data: None
+2022-12-21,11:47:52 | INFO |   train_num_samples: None
+2022-12-21,11:47:52 | INFO |   use_bn_sync: False
+2022-12-21,11:47:52 | INFO |   val_data: None
+2022-12-21,11:47:52 | INFO |   val_frequency: 1
+2022-12-21,11:47:52 | INFO |   val_num_samples: None
+2022-12-21,11:47:52 | INFO |   wandb: False
+2022-12-21,11:47:52 | INFO |   wandb_notes: 
+2022-12-21,11:47:52 | INFO |   warmup: 10000
+2022-12-21,11:47:52 | INFO |   wd: 0.2
+2022-12-21,11:47:52 | INFO |   workers: 1
+2022-12-21,11:47:52 | INFO |   world_size: 1
+2022-12-21,11:47:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:47:52 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:47:52 | INFO | Building zero-shot classifier
+
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<09:34,  1.74it/s]
  1%|          | 7/1000 [00:00<01:16, 12.94it/s]
  1%|▏         | 13/1000 [00:00<00:44, 22.43it/s]
  2%|▏         | 19/1000 [00:00<00:31, 30.68it/s]
  2%|▎         | 25/1000 [00:01<00:26, 37.41it/s]
  3%|▎         | 31/1000 [00:01<00:22, 42.90it/s]
  4%|▎         | 37/1000 [00:01<00:20, 47.07it/s]
  4%|▍         | 43/1000 [00:01<00:19, 50.18it/s]
  5%|▍         | 49/1000 [00:01<00:18, 52.44it/s]
  6%|▌         | 55/1000 [00:01<00:17, 54.03it/s]
  6%|▌         | 61/1000 [00:01<00:16, 55.32it/s]
  7%|▋         | 67/1000 [00:01<00:16, 56.06it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.71it/s]
  8%|▊         | 79/1000 [00:01<00:16, 57.16it/s]
  8%|▊         | 85/1000 [00:02<00:15, 57.62it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.91it/s]
 10%|▉         | 97/1000 [00:02<00:15, 58.29it/s]
 10%|█         | 103/1000 [00:02<00:15, 58.44it/s]
 11%|█         | 109/1000 [00:02<00:15, 58.65it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.82it/s]
 12%|█▏        | 121/1000 [00:02<00:14, 58.73it/s]
 13%|█▎        | 127/1000 [00:02<00:14, 58.71it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.70it/s]
 14%|█▍        | 139/1000 [00:02<00:14, 58.67it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.73it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.78it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.73it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.72it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 58.51it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 58.51it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 58.47it/s]
 19%|█▊        | 187/1000 [00:03<00:13, 58.46it/s]
 19%|█▉        | 193/1000 [00:03<00:13, 58.41it/s]
 20%|█▉        | 199/1000 [00:03<00:13, 58.33it/s]
 20%|██        | 205/1000 [00:04<00:13, 58.13it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.90it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 58.10it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 58.12it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 58.24it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 58.24it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 58.09it/s]
 25%|██▍       | 247/1000 [00:04<00:12, 58.25it/s]
 25%|██▌       | 253/1000 [00:04<00:12, 58.32it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 58.46it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 58.43it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 58.28it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 58.41it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 58.53it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.66it/s]
 30%|██▉       | 295/1000 [00:05<00:11, 58.79it/s]
 30%|███       | 301/1000 [00:05<00:11, 58.78it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.76it/s]
 31%|███▏      | 313/1000 [00:05<00:11, 58.93it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.95it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.79it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.59it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.72it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.78it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.77it/s]
 36%|███▌      | 355/1000 [00:06<00:10, 58.69it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.74it/s]
 37%|███▋      | 367/1000 [00:06<00:10, 58.78it/s]
 37%|███▋      | 373/1000 [00:06<00:10, 58.86it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.60it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 58.43it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 58.52it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.58it/s]
 40%|████      | 403/1000 [00:07<00:10, 58.70it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.77it/s]
 42%|████▏     | 415/1000 [00:07<00:09, 58.84it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.86it/s]
 43%|████▎     | 427/1000 [00:07<00:09, 58.88it/s]
 43%|████▎     | 433/1000 [00:07<00:09, 58.99it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.98it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.71it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.81it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.83it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.86it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.80it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.96it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.77it/s]
 49%|████▊     | 487/1000 [00:08<00:08, 58.87it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.81it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.72it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.76it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.67it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.78it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.76it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.81it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.66it/s]
 54%|█████▍    | 541/1000 [00:09<00:07, 58.77it/s]
 55%|█████▍    | 547/1000 [00:09<00:07, 58.81it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.73it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.80it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.73it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.64it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.72it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.81it/s]
 59%|█████▉    | 589/1000 [00:10<00:06, 58.84it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.75it/s]
 60%|██████    | 601/1000 [00:10<00:06, 58.83it/s]
 61%|██████    | 607/1000 [00:10<00:06, 58.69it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.72it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.87it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.86it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.85it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.75it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.72it/s]
 65%|██████▍   | 649/1000 [00:11<00:05, 58.82it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.84it/s]
 66%|██████▌   | 661/1000 [00:11<00:05, 58.90it/s]
 67%|██████▋   | 667/1000 [00:11<00:05, 58.87it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.92it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.91it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.96it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.94it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.84it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.89it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.91it/s]
 72%|███████▏  | 715/1000 [00:12<00:04, 58.88it/s]
 72%|███████▏  | 721/1000 [00:12<00:04, 58.87it/s]
 73%|███████▎  | 727/1000 [00:12<00:04, 58.81it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.81it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.81it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.83it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.92it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.94it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.83it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.91it/s]
 78%|███████▊  | 775/1000 [00:13<00:03, 58.89it/s]
 78%|███████▊  | 781/1000 [00:13<00:03, 58.91it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.91it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.75it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.71it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.75it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.70it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.78it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.75it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.88it/s]
 84%|████████▎ | 835/1000 [00:14<00:02, 59.02it/s]
 84%|████████▍ | 841/1000 [00:14<00:02, 59.04it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.89it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.93it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.89it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.89it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.86it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.92it/s]
 88%|████████▊ | 883/1000 [00:15<00:01, 58.90it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.84it/s]
 90%|████████▉ | 895/1000 [00:15<00:01, 58.93it/s]
 90%|█████████ | 901/1000 [00:15<00:01, 58.91it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.87it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.82it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.94it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.92it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 59.00it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.98it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.93it/s]
 95%|█████████▍| 949/1000 [00:16<00:00, 58.94it/s]
 96%|█████████▌| 955/1000 [00:16<00:00, 59.08it/s]
 96%|█████████▌| 961/1000 [00:16<00:00, 58.96it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 59.06it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 59.16it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 59.16it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 59.12it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.99it/s]
100%|█████████▉| 997/1000 [00:17<00:00, 58.77it/s]
100%|██████████| 1000/1000 [00:17<00:00, 56.76it/s]
+2022-12-21,11:48:09 | INFO | Using classifier
+
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:05<16:34, 50.20it/s]
  1%|          | 512/50176 [00:07<11:58, 69.16it/s]
  2%|▏         | 768/50176 [00:10<09:55, 82.93it/s]
  2%|▏         | 1024/50176 [00:12<09:10, 89.30it/s]
  3%|▎         | 1280/50176 [00:15<08:48, 92.47it/s]
  3%|▎         | 1536/50176 [00:18<09:00, 89.96it/s]
  4%|▎         | 1792/50176 [00:20<08:42, 92.58it/s]
  4%|▍         | 2048/50176 [00:23<08:17, 96.65it/s]
  5%|▍         | 2304/50176 [00:25<08:06, 98.44it/s]
  5%|▌         | 2560/50176 [00:28<07:44, 102.40it/s]
  6%|▌         | 2816/50176 [00:30<08:05, 97.61it/s] 
  6%|▌         | 3072/50176 [00:33<07:59, 98.33it/s]
  7%|▋         | 3328/50176 [00:35<07:46, 100.53it/s]
  7%|▋         | 3584/50176 [00:38<07:33, 102.74it/s]
  8%|▊         | 3840/50176 [00:41<07:46, 99.43it/s] 
  8%|▊         | 4096/50176 [00:43<08:00, 95.96it/s]
  9%|▊         | 4352/50176 [00:46<07:56, 96.15it/s]
  9%|▉         | 4608/50176 [00:49<07:41, 98.64it/s]
 10%|▉         | 4864/50176 [00:51<07:28, 100.98it/s]
 10%|█         | 5120/50176 [00:53<07:24, 101.44it/s]
 11%|█         | 5376/50176 [00:56<07:39, 97.54it/s] 
 11%|█         | 5632/50176 [00:59<07:38, 97.06it/s]
 12%|█▏        | 5888/50176 [01:02<07:32, 97.92it/s]
 12%|█▏        | 6144/50176 [01:05<07:57, 92.16it/s]
 13%|█▎        | 6400/50176 [01:07<07:40, 95.03it/s]
 13%|█▎        | 6656/50176 [01:10<07:29, 96.77it/s]
 14%|█▍        | 6912/50176 [01:12<07:27, 96.76it/s]
 14%|█▍        | 7168/50176 [01:15<07:15, 98.67it/s]
 15%|█▍        | 7424/50176 [01:17<07:02, 101.24it/s]
 15%|█▌        | 7680/50176 [01:20<06:58, 101.46it/s]
 16%|█▌        | 7936/50176 [01:22<06:43, 104.64it/s]
 16%|█▋        | 8192/50176 [01:24<06:42, 104.31it/s]
 17%|█▋        | 8448/50176 [01:27<06:41, 103.83it/s]
 17%|█▋        | 8704/50176 [01:30<06:43, 102.75it/s]
 18%|█▊        | 8960/50176 [01:32<06:42, 102.33it/s]
 18%|█▊        | 9216/50176 [01:34<06:33, 104.17it/s]
 19%|█▉        | 9472/50176 [01:37<06:31, 103.97it/s]
 19%|█▉        | 9728/50176 [01:40<06:37, 101.72it/s]
 20%|█▉        | 9984/50176 [01:42<06:38, 100.85it/s]
 20%|██        | 10240/50176 [01:44<06:16, 106.17it/s]
 21%|██        | 10496/50176 [01:46<06:06, 108.39it/s]
 21%|██▏       | 10752/50176 [01:49<06:20, 103.69it/s]
 22%|██▏       | 11008/50176 [01:52<06:25, 101.55it/s]
 22%|██▏       | 11264/50176 [01:54<06:23, 101.56it/s]
 23%|██▎       | 11520/50176 [01:57<06:20, 101.65it/s]
 23%|██▎       | 11776/50176 [01:59<06:15, 102.28it/s]
 24%|██▍       | 12032/50176 [02:02<06:08, 103.53it/s]
 24%|██▍       | 12288/50176 [02:04<06:16, 100.65it/s]
 25%|██▌       | 12544/50176 [02:07<06:07, 102.45it/s]
 26%|██▌       | 12800/50176 [02:09<06:04, 102.58it/s]
 26%|██▌       | 13056/50176 [02:12<05:56, 104.11it/s]
 27%|██▋       | 13312/50176 [02:14<05:53, 104.42it/s]
 27%|██▋       | 13568/50176 [02:17<06:00, 101.47it/s]
 28%|██▊       | 13824/50176 [02:20<06:15, 96.80it/s] 
 28%|██▊       | 14080/50176 [02:22<06:10, 97.51it/s]
 29%|██▊       | 14336/50176 [02:25<06:07, 97.60it/s]
 29%|██▉       | 14592/50176 [02:27<05:52, 100.96it/s]
 30%|██▉       | 14848/50176 [02:30<06:02, 97.47it/s] 
 30%|███       | 15104/50176 [02:33<06:01, 96.98it/s]
 31%|███       | 15360/50176 [02:35<05:59, 96.76it/s]
 31%|███       | 15616/50176 [02:38<05:44, 100.36it/s]
 32%|███▏      | 15872/50176 [02:40<05:47, 98.78it/s] 
 32%|███▏      | 16128/50176 [02:43<05:45, 98.62it/s]
 33%|███▎      | 16384/50176 [02:45<05:34, 101.15it/s]
 33%|███▎      | 16640/50176 [02:48<05:37, 99.24it/s] 
 34%|███▎      | 16896/50176 [02:51<05:30, 100.62it/s]
 34%|███▍      | 17152/50176 [02:53<05:24, 101.85it/s]
 35%|███▍      | 17408/50176 [02:56<05:27, 100.03it/s]
 35%|███▌      | 17664/50176 [02:58<05:20, 101.33it/s]
 36%|███▌      | 17920/50176 [03:01<05:13, 102.88it/s]
 36%|███▌      | 18176/50176 [03:03<05:10, 103.22it/s]
 37%|███▋      | 18432/50176 [03:05<04:59, 106.11it/s]
 37%|███▋      | 18688/50176 [03:08<05:05, 103.19it/s]
 38%|███▊      | 18944/50176 [03:10<05:06, 102.02it/s]
 38%|███▊      | 19200/50176 [03:13<05:10, 99.91it/s] 
 39%|███▉      | 19456/50176 [03:16<05:15, 97.37it/s]
 39%|███▉      | 19712/50176 [03:19<05:14, 96.86it/s]
 40%|███▉      | 19968/50176 [03:21<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:23<04:51, 102.80it/s]
 41%|████      | 20480/50176 [03:26<04:48, 102.95it/s]
 41%|████▏     | 20736/50176 [03:28<04:44, 103.42it/s]
 42%|████▏     | 20992/50176 [03:31<04:37, 105.14it/s]
 42%|████▏     | 21248/50176 [03:33<04:43, 101.88it/s]
 43%|████▎     | 21504/50176 [03:36<04:36, 103.51it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.62it/s]
 44%|████▍     | 22016/50176 [03:41<04:26, 105.67it/s]
 44%|████▍     | 22272/50176 [03:43<04:17, 108.43it/s]
 45%|████▍     | 22528/50176 [03:45<04:13, 109.17it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.24it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.56it/s]
 46%|████▋     | 23296/50176 [03:53<04:27, 100.40it/s]
 47%|████▋     | 23552/50176 [03:56<04:29, 98.67it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.37it/s]
 48%|████▊     | 24064/50176 [04:01<04:16, 102.00it/s]
 48%|████▊     | 24320/50176 [04:03<04:12, 102.45it/s]
 49%|████▉     | 24576/50176 [04:06<04:11, 101.71it/s]
 49%|████▉     | 24832/50176 [04:08<04:06, 102.78it/s]
 50%|█████     | 25088/50176 [04:10<03:56, 106.10it/s]
 51%|█████     | 25344/50176 [04:12<03:48, 108.59it/s]
 51%|█████     | 25600/50176 [04:15<03:47, 108.08it/s]
 52%|█████▏    | 25856/50176 [04:17<03:48, 106.57it/s]
 52%|█████▏    | 26112/50176 [04:20<03:46, 106.42it/s]
 53%|█████▎    | 26368/50176 [04:22<03:49, 103.66it/s]
 53%|█████▎    | 26624/50176 [04:25<03:45, 104.62it/s]
 54%|█████▎    | 26880/50176 [04:27<03:44, 103.96it/s]
 54%|█████▍    | 27136/50176 [04:30<03:43, 102.95it/s]
 55%|█████▍    | 27392/50176 [04:32<03:37, 104.63it/s]
 55%|█████▌    | 27648/50176 [04:34<03:26, 109.07it/s]
 56%|█████▌    | 27904/50176 [04:37<03:36, 102.72it/s]
 56%|█████▌    | 28160/50176 [04:40<03:38, 100.71it/s]
 57%|█████▋    | 28416/50176 [04:42<03:32, 102.27it/s]
 57%|█████▋    | 28672/50176 [04:45<03:30, 102.18it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.34it/s]
 58%|█████▊    | 29184/50176 [04:50<03:25, 102.13it/s]
 59%|█████▊    | 29440/50176 [04:53<03:33, 97.32it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:29, 97.71it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.95it/s]
 60%|██████    | 30208/50176 [05:00<03:21, 99.33it/s]
 61%|██████    | 30464/50176 [05:02<03:08, 104.46it/s]
 61%|██████    | 30720/50176 [05:05<03:06, 104.11it/s]
 62%|██████▏   | 30976/50176 [05:07<02:59, 107.04it/s]
 62%|██████▏   | 31232/50176 [05:09<02:54, 108.51it/s]
 63%|██████▎   | 31488/50176 [05:12<02:50, 109.54it/s]
 63%|██████▎   | 31744/50176 [05:14<02:47, 109.79it/s]
 64%|██████▍   | 32000/50176 [05:16<02:43, 111.28it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.67it/s]
 65%|██████▍   | 32512/50176 [05:21<02:47, 105.42it/s]
 65%|██████▌   | 32768/50176 [05:24<02:44, 105.93it/s]
 66%|██████▌   | 33024/50176 [05:26<02:41, 106.02it/s]
 66%|██████▋   | 33280/50176 [05:29<02:38, 106.45it/s]
 67%|██████▋   | 33536/50176 [05:31<02:37, 105.37it/s]
 67%|██████▋   | 33792/50176 [05:33<02:32, 107.23it/s]
 68%|██████▊   | 34048/50176 [05:35<02:26, 109.80it/s]
 68%|██████▊   | 34304/50176 [05:38<02:23, 110.42it/s]
 69%|██████▉   | 34560/50176 [05:40<02:25, 107.27it/s]
 69%|██████▉   | 34816/50176 [05:43<02:24, 105.97it/s]
 70%|██████▉   | 35072/50176 [05:45<02:23, 105.57it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 106.91it/s]
 71%|███████   | 35584/50176 [05:50<02:14, 108.55it/s]
 71%|███████▏  | 35840/50176 [05:52<02:15, 105.61it/s]
 72%|███████▏  | 36096/50176 [05:55<02:11, 106.96it/s]
 72%|███████▏  | 36352/50176 [05:57<02:12, 104.62it/s]
 73%|███████▎  | 36608/50176 [06:00<02:10, 103.93it/s]
 73%|███████▎  | 36864/50176 [06:03<02:11, 101.04it/s]
 74%|███████▍  | 37120/50176 [06:05<02:07, 102.74it/s]
 74%|███████▍  | 37376/50176 [06:07<01:59, 107.10it/s]
 75%|███████▌  | 37632/50176 [06:09<01:56, 107.48it/s]
 76%|███████▌  | 37888/50176 [06:12<01:52, 108.80it/s]
 76%|███████▌  | 38144/50176 [06:14<01:50, 109.22it/s]
 77%|███████▋  | 38400/50176 [06:17<01:50, 107.04it/s]
 77%|███████▋  | 38656/50176 [06:19<01:48, 106.47it/s]
 78%|███████▊  | 38912/50176 [06:21<01:44, 107.40it/s]
 78%|███████▊  | 39168/50176 [06:24<01:44, 104.94it/s]
 79%|███████▊  | 39424/50176 [06:26<01:41, 106.38it/s]
 79%|███████▉  | 39680/50176 [06:29<01:38, 106.54it/s]
 80%|███████▉  | 39936/50176 [06:31<01:34, 108.08it/s]
 80%|████████  | 40192/50176 [06:33<01:32, 108.16it/s]
 81%|████████  | 40448/50176 [06:36<01:34, 103.14it/s]
 81%|████████  | 40704/50176 [06:39<01:35, 99.13it/s] 
 82%|████████▏ | 40960/50176 [06:41<01:28, 104.18it/s]
 82%|████████▏ | 41216/50176 [06:43<01:25, 105.08it/s]
 83%|████████▎ | 41472/50176 [06:46<01:22, 105.64it/s]
 83%|████████▎ | 41728/50176 [06:48<01:18, 107.59it/s]
 84%|████████▎ | 41984/50176 [06:50<01:16, 107.45it/s]
 84%|████████▍ | 42240/50176 [06:53<01:14, 105.90it/s]
 85%|████████▍ | 42496/50176 [06:55<01:10, 109.18it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 108.28it/s]
 86%|████████▌ | 43008/50176 [07:00<01:07, 106.74it/s]
 86%|████████▌ | 43264/50176 [07:03<01:06, 104.66it/s]
 87%|████████▋ | 43520/50176 [07:05<01:03, 104.22it/s]
 87%|████████▋ | 43776/50176 [07:07<01:01, 104.33it/s]
 88%|████████▊ | 44032/50176 [07:10<00:58, 105.25it/s]
 88%|████████▊ | 44288/50176 [07:12<00:54, 108.05it/s]
 89%|████████▉ | 44544/50176 [07:15<00:54, 103.35it/s]
 89%|████████▉ | 44800/50176 [07:17<00:52, 102.52it/s]
 90%|████████▉ | 45056/50176 [07:20<00:48, 105.47it/s]
 90%|█████████ | 45312/50176 [07:22<00:47, 103.31it/s]
 91%|█████████ | 45568/50176 [07:24<00:42, 107.22it/s]
 91%|█████████▏| 45824/50176 [07:27<00:41, 103.93it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.62it/s]
 92%|█████████▏| 46336/50176 [07:32<00:37, 103.47it/s]
 93%|█████████▎| 46592/50176 [07:34<00:33, 106.10it/s]
 93%|█████████▎| 46848/50176 [07:37<00:31, 105.63it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 100.09it/s]
 94%|█████████▍| 47360/50176 [07:42<00:28, 99.57it/s] 
 95%|█████████▍| 47616/50176 [07:45<00:25, 100.43it/s]
 95%|█████████▌| 47872/50176 [07:47<00:22, 101.58it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 103.42it/s]
 96%|█████████▋| 48384/50176 [07:52<00:17, 103.41it/s]
 97%|█████████▋| 48640/50176 [07:54<00:14, 104.98it/s]
 97%|█████████▋| 48896/50176 [07:57<00:12, 105.44it/s]
 98%|█████████▊| 49152/50176 [07:59<00:10, 102.05it/s]
 98%|█████████▊| 49408/50176 [08:02<00:07, 100.83it/s]
 99%|█████████▉| 49664/50176 [08:05<00:05, 100.23it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 96.38it/s] 
100%|██████████| 50176/50176 [08:08<00:00, 126.51it/s]
100%|██████████| 50176/50176 [08:08<00:00, 102.69it/s]
+2022-12-21,11:56:18 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:56:18 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6812	imagenet-zeroshot-val-top5: 0.9086
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c503438a96a8417450a698627705a5c4381c8339
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.08242, "imagenet-zeroshot-val-top5": 0.21466}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..3c05f61524c9e8ef707f3c6f3448b6bee81bd6da
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_1.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+2022-12-21,09:37:11 | INFO | Using classifier
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.0824	imagenet-zeroshot-val-top5: 0.2147
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5845e591cbfc4fb7c3732a3aef57dbb7befdb240
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_1.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_1.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f45f794abbb4cda35ba63be257078188e9c9de5e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5273, "imagenet-zeroshot-val-top5": 0.8075}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..3adc7dbb0f9af970c0e75361ee7935c46d334b24
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_10.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+2022-12-21,09:37:11 | INFO | Using classifier
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5273	imagenet-zeroshot-val-top5: 0.8075
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7305ff002667e4640175ca03037943b852bdd3f5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_10.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_10.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c2d65b1463f55952a207262aa8ac569da10b7ae3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5318, "imagenet-zeroshot-val-top5": 0.81266}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..0943a42f8edf35c0def39a2691c83cbe3ac44e6f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_11.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+2022-12-21,09:37:11 | INFO | Using classifier
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5318	imagenet-zeroshot-val-top5: 0.8127
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..67cac96e998f74a2dbb144090a7e104cc734ddb1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_11.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_11.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..90782f17a8bcd088f87fce8fd7d8a24d608803f4
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.53686, "imagenet-zeroshot-val-top5": 0.8153}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..43d2eadb3caa093e0a13f83b2095dfed0fe1dde8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt).
+2022-12-21,09:36:52 | INFO | Model:
+2022-12-21,09:36:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:36:52 | INFO | Params:
+2022-12-21,09:36:52 | INFO |   batch_size: 256
+2022-12-21,09:36:52 | INFO |   beta1: 0.9
+2022-12-21,09:36:52 | INFO |   beta2: 0.98
+2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/checkpoints
+2022-12-21,09:36:52 | INFO |   copy_codebase: False
+2022-12-21,09:36:52 | INFO |   csv_caption_key: title
+2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
+2022-12-21,09:36:52 | INFO |   csv_separator: 	
+2022-12-21,09:36:52 | INFO |   dataset_resampled: False
+2022-12-21,09:36:52 | INFO |   dataset_type: auto
+2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
+2022-12-21,09:36:52 | INFO |   debug: False
+2022-12-21,09:36:52 | INFO |   device: cuda:0
+2022-12-21,09:36:52 | INFO |   dist_backend: nccl
+2022-12-21,09:36:52 | INFO |   dist_url: env://
+2022-12-21,09:36:52 | INFO |   distributed: False
+2022-12-21,09:36:52 | INFO |   epochs: 32
+2022-12-21,09:36:52 | INFO |   eps: 1e-06
+2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
+2022-12-21,09:36:52 | INFO |   gather_with_grad: False
+2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
+2022-12-21,09:36:52 | INFO |   horovod: False
+2022-12-21,09:36:52 | INFO |   image_mean: None
+2022-12-21,09:36:52 | INFO |   image_std: None
+2022-12-21,09:36:52 | INFO |   imagenet_v2: None
+2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:36:52 | INFO |   local_loss: False
+2022-12-21,09:36:52 | INFO |   local_rank: 0
+2022-12-21,09:36:52 | INFO |   lock_image: False
+2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:36:52 | INFO |   log_level: 20
+2022-12-21,09:36:52 | INFO |   log_local: False
+2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/out.log
+2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:36:52 | INFO |   lr: 0.0005
+2022-12-21,09:36:52 | INFO |   model: ViT-B-16
+2022-12-21,09:36:52 | INFO |   name: epoch_12.pt
+2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
+2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
+2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
+2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt
+2022-12-21,09:36:52 | INFO |   pretrained_image: False
+2022-12-21,09:36:52 | INFO |   rank: 0
+2022-12-21,09:36:52 | INFO |   report_to: 
+2022-12-21,09:36:52 | INFO |   resume: None
+2022-12-21,09:36:52 | INFO |   save_frequency: 1
+2022-12-21,09:36:52 | INFO |   save_most_recent: False
+2022-12-21,09:36:52 | INFO |   seed: 0
+2022-12-21,09:36:52 | INFO |   skip_scheduler: False
+2022-12-21,09:36:52 | INFO |   tensorboard: False
+2022-12-21,09:36:52 | INFO |   tensorboard_path: 
+2022-12-21,09:36:52 | INFO |   torchscript: False
+2022-12-21,09:36:52 | INFO |   trace: False
+2022-12-21,09:36:52 | INFO |   train_data: None
+2022-12-21,09:36:52 | INFO |   train_num_samples: None
+2022-12-21,09:36:52 | INFO |   use_bn_sync: False
+2022-12-21,09:36:52 | INFO |   val_data: None
+2022-12-21,09:36:52 | INFO |   val_frequency: 1
+2022-12-21,09:36:52 | INFO |   val_num_samples: None
+2022-12-21,09:36:52 | INFO |   wandb: False
+2022-12-21,09:36:52 | INFO |   wandb_notes: 
+2022-12-21,09:36:52 | INFO |   warmup: 10000
+2022-12-21,09:36:52 | INFO |   wd: 0.2
+2022-12-21,09:36:52 | INFO |   workers: 1
+2022-12-21,09:36:52 | INFO |   world_size: 1
+2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:36:53 | INFO | Building zero-shot classifier
+2022-12-21,09:37:11 | INFO | Using classifier
+2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5369	imagenet-zeroshot-val-top5: 0.8153
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3c763280631973444556bbb1a96d21985960c0f7
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_12.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_12.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cbd2c4a967a561b167e246d40b59d3718153918e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5455, "imagenet-zeroshot-val-top5": 0.8192}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..7d4ad836bc62b4a2555cb6f62c8afc5250d6ba48
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_13.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+2022-12-21,09:46:23 | INFO | Using classifier
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5455	imagenet-zeroshot-val-top5: 0.8192
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4d6cca06a3c3ba50e25b3eb8177f8315e3bdaec5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_13.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_13.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..597095a81dd4e8b1093095468764d150fa342e6b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.54244, "imagenet-zeroshot-val-top5": 0.81838}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..9de5de4d097a7fd38dd1e4fc89a1241201b27567
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_14.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+2022-12-21,09:46:23 | INFO | Using classifier
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5424	imagenet-zeroshot-val-top5: 0.8184
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..dad8baea8b79e6432e0af1757a1cf2b9c1c555b4
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_14.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_14.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1072b5a0c02bce54e02d9b3646d0b10686eaede0
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.55144, "imagenet-zeroshot-val-top5": 0.82464}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..c04ef6414892a7c07ce21e779a64cf13a56267bb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_15.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+2022-12-21,09:46:23 | INFO | Using classifier
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5514	imagenet-zeroshot-val-top5: 0.8246
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6ce2c77775be63d938f4049b1be879351ff688ec
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_15.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_15.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..12c346abae9b02f592f28963728171544d109f67
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.55328, "imagenet-zeroshot-val-top5": 0.82584}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..c39be41c6a02e022baf55d27cd48e93c8003692c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt).
+2022-12-21,09:46:05 | INFO | Model:
+2022-12-21,09:46:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:46:05 | INFO | Params:
+2022-12-21,09:46:05 | INFO |   batch_size: 256
+2022-12-21,09:46:05 | INFO |   beta1: 0.9
+2022-12-21,09:46:05 | INFO |   beta2: 0.98
+2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/checkpoints
+2022-12-21,09:46:05 | INFO |   copy_codebase: False
+2022-12-21,09:46:05 | INFO |   csv_caption_key: title
+2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
+2022-12-21,09:46:05 | INFO |   csv_separator: 	
+2022-12-21,09:46:05 | INFO |   dataset_resampled: False
+2022-12-21,09:46:05 | INFO |   dataset_type: auto
+2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
+2022-12-21,09:46:05 | INFO |   debug: False
+2022-12-21,09:46:05 | INFO |   device: cuda:0
+2022-12-21,09:46:05 | INFO |   dist_backend: nccl
+2022-12-21,09:46:05 | INFO |   dist_url: env://
+2022-12-21,09:46:05 | INFO |   distributed: False
+2022-12-21,09:46:05 | INFO |   epochs: 32
+2022-12-21,09:46:05 | INFO |   eps: 1e-06
+2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
+2022-12-21,09:46:05 | INFO |   gather_with_grad: False
+2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
+2022-12-21,09:46:05 | INFO |   horovod: False
+2022-12-21,09:46:05 | INFO |   image_mean: None
+2022-12-21,09:46:05 | INFO |   image_std: None
+2022-12-21,09:46:05 | INFO |   imagenet_v2: None
+2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:46:05 | INFO |   local_loss: False
+2022-12-21,09:46:05 | INFO |   local_rank: 0
+2022-12-21,09:46:05 | INFO |   lock_image: False
+2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:46:05 | INFO |   log_level: 20
+2022-12-21,09:46:05 | INFO |   log_local: False
+2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/out.log
+2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:46:05 | INFO |   lr: 0.0005
+2022-12-21,09:46:05 | INFO |   model: ViT-B-16
+2022-12-21,09:46:05 | INFO |   name: epoch_16.pt
+2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
+2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
+2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
+2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt
+2022-12-21,09:46:05 | INFO |   pretrained_image: False
+2022-12-21,09:46:05 | INFO |   rank: 0
+2022-12-21,09:46:05 | INFO |   report_to: 
+2022-12-21,09:46:05 | INFO |   resume: None
+2022-12-21,09:46:05 | INFO |   save_frequency: 1
+2022-12-21,09:46:05 | INFO |   save_most_recent: False
+2022-12-21,09:46:05 | INFO |   seed: 0
+2022-12-21,09:46:05 | INFO |   skip_scheduler: False
+2022-12-21,09:46:05 | INFO |   tensorboard: False
+2022-12-21,09:46:05 | INFO |   tensorboard_path: 
+2022-12-21,09:46:05 | INFO |   torchscript: False
+2022-12-21,09:46:05 | INFO |   trace: False
+2022-12-21,09:46:05 | INFO |   train_data: None
+2022-12-21,09:46:05 | INFO |   train_num_samples: None
+2022-12-21,09:46:05 | INFO |   use_bn_sync: False
+2022-12-21,09:46:05 | INFO |   val_data: None
+2022-12-21,09:46:05 | INFO |   val_frequency: 1
+2022-12-21,09:46:05 | INFO |   val_num_samples: None
+2022-12-21,09:46:05 | INFO |   wandb: False
+2022-12-21,09:46:05 | INFO |   wandb_notes: 
+2022-12-21,09:46:05 | INFO |   warmup: 10000
+2022-12-21,09:46:05 | INFO |   wd: 0.2
+2022-12-21,09:46:05 | INFO |   workers: 1
+2022-12-21,09:46:05 | INFO |   world_size: 1
+2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:46:05 | INFO | Building zero-shot classifier
+2022-12-21,09:46:23 | INFO | Using classifier
+2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5533	imagenet-zeroshot-val-top5: 0.8258
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e9f50341b60bb3259916d31bf14607f7d7387cf0
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_16.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_16.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9edaa26ccfd17d68cbbee27e592ed0b7c4b75729
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.55452, "imagenet-zeroshot-val-top5": 0.83016}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..0afd116bc0d85235880b585d84ec255a31cf9edb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt).
+2022-12-21,09:54:46 | INFO | Model:
+2022-12-21,09:54:46 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:46 | INFO | Params:
+2022-12-21,09:54:46 | INFO |   batch_size: 256
+2022-12-21,09:54:46 | INFO |   beta1: 0.9
+2022-12-21,09:54:46 | INFO |   beta2: 0.98
+2022-12-21,09:54:46 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/checkpoints
+2022-12-21,09:54:46 | INFO |   copy_codebase: False
+2022-12-21,09:54:46 | INFO |   csv_caption_key: title
+2022-12-21,09:54:46 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:46 | INFO |   csv_separator: 	
+2022-12-21,09:54:46 | INFO |   dataset_resampled: False
+2022-12-21,09:54:46 | INFO |   dataset_type: auto
+2022-12-21,09:54:46 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:46 | INFO |   debug: False
+2022-12-21,09:54:46 | INFO |   device: cuda:0
+2022-12-21,09:54:46 | INFO |   dist_backend: nccl
+2022-12-21,09:54:46 | INFO |   dist_url: env://
+2022-12-21,09:54:46 | INFO |   distributed: False
+2022-12-21,09:54:46 | INFO |   epochs: 32
+2022-12-21,09:54:46 | INFO |   eps: 1e-06
+2022-12-21,09:54:46 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:46 | INFO |   gather_with_grad: False
+2022-12-21,09:54:46 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:46 | INFO |   horovod: False
+2022-12-21,09:54:46 | INFO |   image_mean: None
+2022-12-21,09:54:46 | INFO |   image_std: None
+2022-12-21,09:54:46 | INFO |   imagenet_v2: None
+2022-12-21,09:54:46 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:46 | INFO |   local_loss: False
+2022-12-21,09:54:46 | INFO |   local_rank: 0
+2022-12-21,09:54:46 | INFO |   lock_image: False
+2022-12-21,09:54:46 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:46 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:46 | INFO |   log_level: 20
+2022-12-21,09:54:46 | INFO |   log_local: False
+2022-12-21,09:54:46 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/out.log
+2022-12-21,09:54:46 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:46 | INFO |   lr: 0.0005
+2022-12-21,09:54:46 | INFO |   model: ViT-B-16
+2022-12-21,09:54:46 | INFO |   name: epoch_17.pt
+2022-12-21,09:54:46 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:46 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:46 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:46 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt
+2022-12-21,09:54:46 | INFO |   pretrained_image: False
+2022-12-21,09:54:46 | INFO |   rank: 0
+2022-12-21,09:54:46 | INFO |   report_to: 
+2022-12-21,09:54:46 | INFO |   resume: None
+2022-12-21,09:54:46 | INFO |   save_frequency: 1
+2022-12-21,09:54:46 | INFO |   save_most_recent: False
+2022-12-21,09:54:46 | INFO |   seed: 0
+2022-12-21,09:54:46 | INFO |   skip_scheduler: False
+2022-12-21,09:54:46 | INFO |   tensorboard: False
+2022-12-21,09:54:46 | INFO |   tensorboard_path: 
+2022-12-21,09:54:46 | INFO |   torchscript: False
+2022-12-21,09:54:46 | INFO |   trace: False
+2022-12-21,09:54:46 | INFO |   train_data: None
+2022-12-21,09:54:46 | INFO |   train_num_samples: None
+2022-12-21,09:54:46 | INFO |   use_bn_sync: False
+2022-12-21,09:54:46 | INFO |   val_data: None
+2022-12-21,09:54:46 | INFO |   val_frequency: 1
+2022-12-21,09:54:46 | INFO |   val_num_samples: None
+2022-12-21,09:54:46 | INFO |   wandb: False
+2022-12-21,09:54:46 | INFO |   wandb_notes: 
+2022-12-21,09:54:46 | INFO |   warmup: 10000
+2022-12-21,09:54:46 | INFO |   wd: 0.2
+2022-12-21,09:54:46 | INFO |   workers: 1
+2022-12-21,09:54:46 | INFO |   world_size: 1
+2022-12-21,09:54:46 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:46 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:46 | INFO | Building zero-shot classifier
+2022-12-21,09:55:04 | INFO | Using classifier
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5545	imagenet-zeroshot-val-top5: 0.8302
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c4bb728705ef82bd5c6f612799f9a053e44a73c9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_17.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_17.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..237c210139bddeac9baf8f4686fd991df6f2c8f6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.55338, "imagenet-zeroshot-val-top5": 0.82778}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..39aaf09587ffdf428e62d47380ddee660336531c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt).
+2022-12-21,09:54:45 | INFO | Model:
+2022-12-21,09:54:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:45 | INFO | Params:
+2022-12-21,09:54:45 | INFO |   batch_size: 256
+2022-12-21,09:54:45 | INFO |   beta1: 0.9
+2022-12-21,09:54:45 | INFO |   beta2: 0.98
+2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/checkpoints
+2022-12-21,09:54:45 | INFO |   copy_codebase: False
+2022-12-21,09:54:45 | INFO |   csv_caption_key: title
+2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:45 | INFO |   csv_separator: 	
+2022-12-21,09:54:45 | INFO |   dataset_resampled: False
+2022-12-21,09:54:45 | INFO |   dataset_type: auto
+2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:45 | INFO |   debug: False
+2022-12-21,09:54:45 | INFO |   device: cuda:0
+2022-12-21,09:54:45 | INFO |   dist_backend: nccl
+2022-12-21,09:54:45 | INFO |   dist_url: env://
+2022-12-21,09:54:45 | INFO |   distributed: False
+2022-12-21,09:54:45 | INFO |   epochs: 32
+2022-12-21,09:54:45 | INFO |   eps: 1e-06
+2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:45 | INFO |   gather_with_grad: False
+2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:45 | INFO |   horovod: False
+2022-12-21,09:54:45 | INFO |   image_mean: None
+2022-12-21,09:54:45 | INFO |   image_std: None
+2022-12-21,09:54:45 | INFO |   imagenet_v2: None
+2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:45 | INFO |   local_loss: False
+2022-12-21,09:54:45 | INFO |   local_rank: 0
+2022-12-21,09:54:45 | INFO |   lock_image: False
+2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:45 | INFO |   log_level: 20
+2022-12-21,09:54:45 | INFO |   log_local: False
+2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/out.log
+2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:45 | INFO |   lr: 0.0005
+2022-12-21,09:54:45 | INFO |   model: ViT-B-16
+2022-12-21,09:54:45 | INFO |   name: epoch_18.pt
+2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt
+2022-12-21,09:54:45 | INFO |   pretrained_image: False
+2022-12-21,09:54:45 | INFO |   rank: 0
+2022-12-21,09:54:45 | INFO |   report_to: 
+2022-12-21,09:54:45 | INFO |   resume: None
+2022-12-21,09:54:45 | INFO |   save_frequency: 1
+2022-12-21,09:54:45 | INFO |   save_most_recent: False
+2022-12-21,09:54:45 | INFO |   seed: 0
+2022-12-21,09:54:45 | INFO |   skip_scheduler: False
+2022-12-21,09:54:45 | INFO |   tensorboard: False
+2022-12-21,09:54:45 | INFO |   tensorboard_path: 
+2022-12-21,09:54:45 | INFO |   torchscript: False
+2022-12-21,09:54:45 | INFO |   trace: False
+2022-12-21,09:54:45 | INFO |   train_data: None
+2022-12-21,09:54:45 | INFO |   train_num_samples: None
+2022-12-21,09:54:45 | INFO |   use_bn_sync: False
+2022-12-21,09:54:45 | INFO |   val_data: None
+2022-12-21,09:54:45 | INFO |   val_frequency: 1
+2022-12-21,09:54:45 | INFO |   val_num_samples: None
+2022-12-21,09:54:45 | INFO |   wandb: False
+2022-12-21,09:54:45 | INFO |   wandb_notes: 
+2022-12-21,09:54:45 | INFO |   warmup: 10000
+2022-12-21,09:54:45 | INFO |   wd: 0.2
+2022-12-21,09:54:45 | INFO |   workers: 1
+2022-12-21,09:54:45 | INFO |   world_size: 1
+2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:45 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:45 | INFO | Building zero-shot classifier
+2022-12-21,09:55:03 | INFO | Using classifier
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5534	imagenet-zeroshot-val-top5: 0.8278
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1eceeea5c9d73e7774aef4ba33813e7a5b71aa30
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_18.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_18.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5631b7a9be9609f71db156bdb16fbd8015e415c6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.56036, "imagenet-zeroshot-val-top5": 0.83126}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..945079c5bf248430a512d5aeba4809cc8cf1a465
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt).
+2022-12-21,09:54:45 | INFO | Model:
+2022-12-21,09:54:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:45 | INFO | Params:
+2022-12-21,09:54:45 | INFO |   batch_size: 256
+2022-12-21,09:54:45 | INFO |   beta1: 0.9
+2022-12-21,09:54:45 | INFO |   beta2: 0.98
+2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/checkpoints
+2022-12-21,09:54:45 | INFO |   copy_codebase: False
+2022-12-21,09:54:45 | INFO |   csv_caption_key: title
+2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:45 | INFO |   csv_separator: 	
+2022-12-21,09:54:45 | INFO |   dataset_resampled: False
+2022-12-21,09:54:45 | INFO |   dataset_type: auto
+2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:45 | INFO |   debug: False
+2022-12-21,09:54:45 | INFO |   device: cuda:0
+2022-12-21,09:54:45 | INFO |   dist_backend: nccl
+2022-12-21,09:54:45 | INFO |   dist_url: env://
+2022-12-21,09:54:45 | INFO |   distributed: False
+2022-12-21,09:54:45 | INFO |   epochs: 32
+2022-12-21,09:54:45 | INFO |   eps: 1e-06
+2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:45 | INFO |   gather_with_grad: False
+2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:45 | INFO |   horovod: False
+2022-12-21,09:54:45 | INFO |   image_mean: None
+2022-12-21,09:54:45 | INFO |   image_std: None
+2022-12-21,09:54:45 | INFO |   imagenet_v2: None
+2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:45 | INFO |   local_loss: False
+2022-12-21,09:54:45 | INFO |   local_rank: 0
+2022-12-21,09:54:45 | INFO |   lock_image: False
+2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:45 | INFO |   log_level: 20
+2022-12-21,09:54:45 | INFO |   log_local: False
+2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/out.log
+2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:45 | INFO |   lr: 0.0005
+2022-12-21,09:54:45 | INFO |   model: ViT-B-16
+2022-12-21,09:54:45 | INFO |   name: epoch_19.pt
+2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt
+2022-12-21,09:54:45 | INFO |   pretrained_image: False
+2022-12-21,09:54:45 | INFO |   rank: 0
+2022-12-21,09:54:45 | INFO |   report_to: 
+2022-12-21,09:54:45 | INFO |   resume: None
+2022-12-21,09:54:45 | INFO |   save_frequency: 1
+2022-12-21,09:54:45 | INFO |   save_most_recent: False
+2022-12-21,09:54:45 | INFO |   seed: 0
+2022-12-21,09:54:45 | INFO |   skip_scheduler: False
+2022-12-21,09:54:45 | INFO |   tensorboard: False
+2022-12-21,09:54:45 | INFO |   tensorboard_path: 
+2022-12-21,09:54:45 | INFO |   torchscript: False
+2022-12-21,09:54:45 | INFO |   trace: False
+2022-12-21,09:54:45 | INFO |   train_data: None
+2022-12-21,09:54:45 | INFO |   train_num_samples: None
+2022-12-21,09:54:45 | INFO |   use_bn_sync: False
+2022-12-21,09:54:45 | INFO |   val_data: None
+2022-12-21,09:54:45 | INFO |   val_frequency: 1
+2022-12-21,09:54:45 | INFO |   val_num_samples: None
+2022-12-21,09:54:45 | INFO |   wandb: False
+2022-12-21,09:54:45 | INFO |   wandb_notes: 
+2022-12-21,09:54:45 | INFO |   warmup: 10000
+2022-12-21,09:54:45 | INFO |   wd: 0.2
+2022-12-21,09:54:45 | INFO |   workers: 1
+2022-12-21,09:54:45 | INFO |   world_size: 1
+2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:46 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:46 | INFO | Building zero-shot classifier
+2022-12-21,09:55:04 | INFO | Using classifier
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5604	imagenet-zeroshot-val-top5: 0.8313
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2f3d754854f8b980d26680d6167a4da4eb628d71
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_19.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_19.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..47f953d0001acb1b2e92fde1e553a02ea027b8bd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.26644, "imagenet-zeroshot-val-top5": 0.5124}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..6e5ad5c599b500fb668e9d151579a8af0bb30708
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
+2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt).
+2022-12-21,09:54:45 | INFO | Model:
+2022-12-21,09:54:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,09:54:45 | INFO | Params:
+2022-12-21,09:54:45 | INFO |   batch_size: 256
+2022-12-21,09:54:45 | INFO |   beta1: 0.9
+2022-12-21,09:54:45 | INFO |   beta2: 0.98
+2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/checkpoints
+2022-12-21,09:54:45 | INFO |   copy_codebase: False
+2022-12-21,09:54:45 | INFO |   csv_caption_key: title
+2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
+2022-12-21,09:54:45 | INFO |   csv_separator: 	
+2022-12-21,09:54:45 | INFO |   dataset_resampled: False
+2022-12-21,09:54:45 | INFO |   dataset_type: auto
+2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
+2022-12-21,09:54:45 | INFO |   debug: False
+2022-12-21,09:54:45 | INFO |   device: cuda:0
+2022-12-21,09:54:45 | INFO |   dist_backend: nccl
+2022-12-21,09:54:45 | INFO |   dist_url: env://
+2022-12-21,09:54:45 | INFO |   distributed: False
+2022-12-21,09:54:45 | INFO |   epochs: 32
+2022-12-21,09:54:45 | INFO |   eps: 1e-06
+2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
+2022-12-21,09:54:45 | INFO |   gather_with_grad: False
+2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
+2022-12-21,09:54:45 | INFO |   horovod: False
+2022-12-21,09:54:45 | INFO |   image_mean: None
+2022-12-21,09:54:45 | INFO |   image_std: None
+2022-12-21,09:54:45 | INFO |   imagenet_v2: None
+2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,09:54:45 | INFO |   local_loss: False
+2022-12-21,09:54:45 | INFO |   local_rank: 0
+2022-12-21,09:54:45 | INFO |   lock_image: False
+2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,09:54:45 | INFO |   log_level: 20
+2022-12-21,09:54:45 | INFO |   log_local: False
+2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/out.log
+2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,09:54:45 | INFO |   lr: 0.0005
+2022-12-21,09:54:45 | INFO |   model: ViT-B-16
+2022-12-21,09:54:45 | INFO |   name: epoch_2.pt
+2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
+2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
+2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
+2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt
+2022-12-21,09:54:45 | INFO |   pretrained_image: False
+2022-12-21,09:54:45 | INFO |   rank: 0
+2022-12-21,09:54:45 | INFO |   report_to: 
+2022-12-21,09:54:45 | INFO |   resume: None
+2022-12-21,09:54:45 | INFO |   save_frequency: 1
+2022-12-21,09:54:45 | INFO |   save_most_recent: False
+2022-12-21,09:54:45 | INFO |   seed: 0
+2022-12-21,09:54:45 | INFO |   skip_scheduler: False
+2022-12-21,09:54:45 | INFO |   tensorboard: False
+2022-12-21,09:54:45 | INFO |   tensorboard_path: 
+2022-12-21,09:54:45 | INFO |   torchscript: False
+2022-12-21,09:54:45 | INFO |   trace: False
+2022-12-21,09:54:45 | INFO |   train_data: None
+2022-12-21,09:54:45 | INFO |   train_num_samples: None
+2022-12-21,09:54:45 | INFO |   use_bn_sync: False
+2022-12-21,09:54:45 | INFO |   val_data: None
+2022-12-21,09:54:45 | INFO |   val_frequency: 1
+2022-12-21,09:54:45 | INFO |   val_num_samples: None
+2022-12-21,09:54:45 | INFO |   wandb: False
+2022-12-21,09:54:45 | INFO |   wandb_notes: 
+2022-12-21,09:54:45 | INFO |   warmup: 10000
+2022-12-21,09:54:45 | INFO |   wd: 0.2
+2022-12-21,09:54:45 | INFO |   workers: 1
+2022-12-21,09:54:45 | INFO |   world_size: 1
+2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,09:54:45 | INFO | Starting zero-shot imagenet.
+2022-12-21,09:54:45 | INFO | Building zero-shot classifier
+2022-12-21,09:55:03 | INFO | Using classifier
+2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.2664	imagenet-zeroshot-val-top5: 0.5124
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..73fe7f46c58b16a250a66a3c44effaae1875aea0
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_2.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_2.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7b8e2197bd3c01d10f77ef80ffaa9731b6dc0283
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.56448, "imagenet-zeroshot-val-top5": 0.83528}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..a8581f2af4854c285a029da184a5d783f3640e51
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt).
+2022-12-21,10:01:05 | INFO | Model:
+2022-12-21,10:01:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:05 | INFO | Params:
+2022-12-21,10:01:05 | INFO |   batch_size: 256
+2022-12-21,10:01:05 | INFO |   beta1: 0.9
+2022-12-21,10:01:05 | INFO |   beta2: 0.98
+2022-12-21,10:01:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/checkpoints
+2022-12-21,10:01:05 | INFO |   copy_codebase: False
+2022-12-21,10:01:05 | INFO |   csv_caption_key: title
+2022-12-21,10:01:05 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:05 | INFO |   csv_separator: 	
+2022-12-21,10:01:05 | INFO |   dataset_resampled: False
+2022-12-21,10:01:05 | INFO |   dataset_type: auto
+2022-12-21,10:01:05 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:05 | INFO |   debug: False
+2022-12-21,10:01:05 | INFO |   device: cuda:0
+2022-12-21,10:01:05 | INFO |   dist_backend: nccl
+2022-12-21,10:01:05 | INFO |   dist_url: env://
+2022-12-21,10:01:05 | INFO |   distributed: False
+2022-12-21,10:01:05 | INFO |   epochs: 32
+2022-12-21,10:01:05 | INFO |   eps: 1e-06
+2022-12-21,10:01:05 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:05 | INFO |   gather_with_grad: False
+2022-12-21,10:01:05 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:05 | INFO |   horovod: False
+2022-12-21,10:01:05 | INFO |   image_mean: None
+2022-12-21,10:01:05 | INFO |   image_std: None
+2022-12-21,10:01:05 | INFO |   imagenet_v2: None
+2022-12-21,10:01:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:05 | INFO |   local_loss: False
+2022-12-21,10:01:05 | INFO |   local_rank: 0
+2022-12-21,10:01:05 | INFO |   lock_image: False
+2022-12-21,10:01:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:05 | INFO |   log_level: 20
+2022-12-21,10:01:05 | INFO |   log_local: False
+2022-12-21,10:01:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/out.log
+2022-12-21,10:01:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:05 | INFO |   lr: 0.0005
+2022-12-21,10:01:05 | INFO |   model: ViT-B-16
+2022-12-21,10:01:05 | INFO |   name: epoch_20.pt
+2022-12-21,10:01:05 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:05 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:05 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt
+2022-12-21,10:01:05 | INFO |   pretrained_image: False
+2022-12-21,10:01:05 | INFO |   rank: 0
+2022-12-21,10:01:05 | INFO |   report_to: 
+2022-12-21,10:01:05 | INFO |   resume: None
+2022-12-21,10:01:05 | INFO |   save_frequency: 1
+2022-12-21,10:01:05 | INFO |   save_most_recent: False
+2022-12-21,10:01:05 | INFO |   seed: 0
+2022-12-21,10:01:05 | INFO |   skip_scheduler: False
+2022-12-21,10:01:05 | INFO |   tensorboard: False
+2022-12-21,10:01:05 | INFO |   tensorboard_path: 
+2022-12-21,10:01:05 | INFO |   torchscript: False
+2022-12-21,10:01:05 | INFO |   trace: False
+2022-12-21,10:01:05 | INFO |   train_data: None
+2022-12-21,10:01:05 | INFO |   train_num_samples: None
+2022-12-21,10:01:05 | INFO |   use_bn_sync: False
+2022-12-21,10:01:05 | INFO |   val_data: None
+2022-12-21,10:01:05 | INFO |   val_frequency: 1
+2022-12-21,10:01:05 | INFO |   val_num_samples: None
+2022-12-21,10:01:05 | INFO |   wandb: False
+2022-12-21,10:01:05 | INFO |   wandb_notes: 
+2022-12-21,10:01:05 | INFO |   warmup: 10000
+2022-12-21,10:01:05 | INFO |   wd: 0.2
+2022-12-21,10:01:05 | INFO |   workers: 1
+2022-12-21,10:01:05 | INFO |   world_size: 1
+2022-12-21,10:01:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:05 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:05 | INFO | Building zero-shot classifier
+2022-12-21,10:01:23 | INFO | Using classifier
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5645	imagenet-zeroshot-val-top5: 0.8353
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..af76aa6a3bb1921ce39394c2ad22f8d0cb52f061
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_20.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_20.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..da890b8575e40390c3570615666663602e7fa85f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5724, "imagenet-zeroshot-val-top5": 0.84056}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..83c247e79ad1f78fea9b9ad7410dd42503533887
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt).
+2022-12-21,10:01:07 | INFO | Model:
+2022-12-21,10:01:07 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:07 | INFO | Params:
+2022-12-21,10:01:07 | INFO |   batch_size: 256
+2022-12-21,10:01:07 | INFO |   beta1: 0.9
+2022-12-21,10:01:07 | INFO |   beta2: 0.98
+2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/checkpoints
+2022-12-21,10:01:07 | INFO |   copy_codebase: False
+2022-12-21,10:01:07 | INFO |   csv_caption_key: title
+2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:07 | INFO |   csv_separator: 	
+2022-12-21,10:01:07 | INFO |   dataset_resampled: False
+2022-12-21,10:01:07 | INFO |   dataset_type: auto
+2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:07 | INFO |   debug: False
+2022-12-21,10:01:07 | INFO |   device: cuda:0
+2022-12-21,10:01:07 | INFO |   dist_backend: nccl
+2022-12-21,10:01:07 | INFO |   dist_url: env://
+2022-12-21,10:01:07 | INFO |   distributed: False
+2022-12-21,10:01:07 | INFO |   epochs: 32
+2022-12-21,10:01:07 | INFO |   eps: 1e-06
+2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:07 | INFO |   gather_with_grad: False
+2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:07 | INFO |   horovod: False
+2022-12-21,10:01:07 | INFO |   image_mean: None
+2022-12-21,10:01:07 | INFO |   image_std: None
+2022-12-21,10:01:07 | INFO |   imagenet_v2: None
+2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:07 | INFO |   local_loss: False
+2022-12-21,10:01:07 | INFO |   local_rank: 0
+2022-12-21,10:01:07 | INFO |   lock_image: False
+2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:07 | INFO |   log_level: 20
+2022-12-21,10:01:07 | INFO |   log_local: False
+2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/out.log
+2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:07 | INFO |   lr: 0.0005
+2022-12-21,10:01:07 | INFO |   model: ViT-B-16
+2022-12-21,10:01:07 | INFO |   name: epoch_21.pt
+2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt
+2022-12-21,10:01:07 | INFO |   pretrained_image: False
+2022-12-21,10:01:07 | INFO |   rank: 0
+2022-12-21,10:01:07 | INFO |   report_to: 
+2022-12-21,10:01:07 | INFO |   resume: None
+2022-12-21,10:01:07 | INFO |   save_frequency: 1
+2022-12-21,10:01:07 | INFO |   save_most_recent: False
+2022-12-21,10:01:07 | INFO |   seed: 0
+2022-12-21,10:01:07 | INFO |   skip_scheduler: False
+2022-12-21,10:01:07 | INFO |   tensorboard: False
+2022-12-21,10:01:07 | INFO |   tensorboard_path: 
+2022-12-21,10:01:07 | INFO |   torchscript: False
+2022-12-21,10:01:07 | INFO |   trace: False
+2022-12-21,10:01:07 | INFO |   train_data: None
+2022-12-21,10:01:07 | INFO |   train_num_samples: None
+2022-12-21,10:01:07 | INFO |   use_bn_sync: False
+2022-12-21,10:01:07 | INFO |   val_data: None
+2022-12-21,10:01:07 | INFO |   val_frequency: 1
+2022-12-21,10:01:07 | INFO |   val_num_samples: None
+2022-12-21,10:01:07 | INFO |   wandb: False
+2022-12-21,10:01:07 | INFO |   wandb_notes: 
+2022-12-21,10:01:07 | INFO |   warmup: 10000
+2022-12-21,10:01:07 | INFO |   wd: 0.2
+2022-12-21,10:01:07 | INFO |   workers: 1
+2022-12-21,10:01:07 | INFO |   world_size: 1
+2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:07 | INFO | Building zero-shot classifier
+2022-12-21,10:01:25 | INFO | Using classifier
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5724	imagenet-zeroshot-val-top5: 0.8406
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..681ff34a9e5a08a87c37c303e7d9b6d48e26c602
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_21.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_21.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f4f7d222c0408346220a334f5db8fdc3e0e9ac49
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.56542, "imagenet-zeroshot-val-top5": 0.83776}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..935850d636b503e10464e49a6c4e32625a59dd17
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:00:58 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:58 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt).
+2022-12-21,10:01:07 | INFO | Model:
+2022-12-21,10:01:07 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:07 | INFO | Params:
+2022-12-21,10:01:07 | INFO |   batch_size: 256
+2022-12-21,10:01:07 | INFO |   beta1: 0.9
+2022-12-21,10:01:07 | INFO |   beta2: 0.98
+2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/checkpoints
+2022-12-21,10:01:07 | INFO |   copy_codebase: False
+2022-12-21,10:01:07 | INFO |   csv_caption_key: title
+2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:07 | INFO |   csv_separator: 	
+2022-12-21,10:01:07 | INFO |   dataset_resampled: False
+2022-12-21,10:01:07 | INFO |   dataset_type: auto
+2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:07 | INFO |   debug: False
+2022-12-21,10:01:07 | INFO |   device: cuda:0
+2022-12-21,10:01:07 | INFO |   dist_backend: nccl
+2022-12-21,10:01:07 | INFO |   dist_url: env://
+2022-12-21,10:01:07 | INFO |   distributed: False
+2022-12-21,10:01:07 | INFO |   epochs: 32
+2022-12-21,10:01:07 | INFO |   eps: 1e-06
+2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:07 | INFO |   gather_with_grad: False
+2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:07 | INFO |   horovod: False
+2022-12-21,10:01:07 | INFO |   image_mean: None
+2022-12-21,10:01:07 | INFO |   image_std: None
+2022-12-21,10:01:07 | INFO |   imagenet_v2: None
+2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:07 | INFO |   local_loss: False
+2022-12-21,10:01:07 | INFO |   local_rank: 0
+2022-12-21,10:01:07 | INFO |   lock_image: False
+2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:07 | INFO |   log_level: 20
+2022-12-21,10:01:07 | INFO |   log_local: False
+2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/out.log
+2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:07 | INFO |   lr: 0.0005
+2022-12-21,10:01:07 | INFO |   model: ViT-B-16
+2022-12-21,10:01:07 | INFO |   name: epoch_22.pt
+2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt
+2022-12-21,10:01:07 | INFO |   pretrained_image: False
+2022-12-21,10:01:07 | INFO |   rank: 0
+2022-12-21,10:01:07 | INFO |   report_to: 
+2022-12-21,10:01:07 | INFO |   resume: None
+2022-12-21,10:01:07 | INFO |   save_frequency: 1
+2022-12-21,10:01:07 | INFO |   save_most_recent: False
+2022-12-21,10:01:07 | INFO |   seed: 0
+2022-12-21,10:01:07 | INFO |   skip_scheduler: False
+2022-12-21,10:01:07 | INFO |   tensorboard: False
+2022-12-21,10:01:07 | INFO |   tensorboard_path: 
+2022-12-21,10:01:07 | INFO |   torchscript: False
+2022-12-21,10:01:07 | INFO |   trace: False
+2022-12-21,10:01:07 | INFO |   train_data: None
+2022-12-21,10:01:07 | INFO |   train_num_samples: None
+2022-12-21,10:01:07 | INFO |   use_bn_sync: False
+2022-12-21,10:01:07 | INFO |   val_data: None
+2022-12-21,10:01:07 | INFO |   val_frequency: 1
+2022-12-21,10:01:07 | INFO |   val_num_samples: None
+2022-12-21,10:01:07 | INFO |   wandb: False
+2022-12-21,10:01:07 | INFO |   wandb_notes: 
+2022-12-21,10:01:07 | INFO |   warmup: 10000
+2022-12-21,10:01:07 | INFO |   wd: 0.2
+2022-12-21,10:01:07 | INFO |   workers: 1
+2022-12-21,10:01:07 | INFO |   world_size: 1
+2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:07 | INFO | Building zero-shot classifier
+2022-12-21,10:01:25 | INFO | Using classifier
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5654	imagenet-zeroshot-val-top5: 0.8378
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..261fd0cbec71eeb9f9bac74b4a69c5e8858546f8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_22.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_22.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ab8d3ff2f5383f29fd0c31a7cfa4ccf89325f8e5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5731, "imagenet-zeroshot-val-top5": 0.84312}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..85d6dcba7e1f4ee68829a0a50b1890c3d513f2b8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt).
+2022-12-21,10:01:07 | INFO | Model:
+2022-12-21,10:01:07 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:01:07 | INFO | Params:
+2022-12-21,10:01:07 | INFO |   batch_size: 256
+2022-12-21,10:01:07 | INFO |   beta1: 0.9
+2022-12-21,10:01:07 | INFO |   beta2: 0.98
+2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/checkpoints
+2022-12-21,10:01:07 | INFO |   copy_codebase: False
+2022-12-21,10:01:07 | INFO |   csv_caption_key: title
+2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
+2022-12-21,10:01:07 | INFO |   csv_separator: 	
+2022-12-21,10:01:07 | INFO |   dataset_resampled: False
+2022-12-21,10:01:07 | INFO |   dataset_type: auto
+2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
+2022-12-21,10:01:07 | INFO |   debug: False
+2022-12-21,10:01:07 | INFO |   device: cuda:0
+2022-12-21,10:01:07 | INFO |   dist_backend: nccl
+2022-12-21,10:01:07 | INFO |   dist_url: env://
+2022-12-21,10:01:07 | INFO |   distributed: False
+2022-12-21,10:01:07 | INFO |   epochs: 32
+2022-12-21,10:01:07 | INFO |   eps: 1e-06
+2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
+2022-12-21,10:01:07 | INFO |   gather_with_grad: False
+2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
+2022-12-21,10:01:07 | INFO |   horovod: False
+2022-12-21,10:01:07 | INFO |   image_mean: None
+2022-12-21,10:01:07 | INFO |   image_std: None
+2022-12-21,10:01:07 | INFO |   imagenet_v2: None
+2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:01:07 | INFO |   local_loss: False
+2022-12-21,10:01:07 | INFO |   local_rank: 0
+2022-12-21,10:01:07 | INFO |   lock_image: False
+2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:01:07 | INFO |   log_level: 20
+2022-12-21,10:01:07 | INFO |   log_local: False
+2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/out.log
+2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:01:07 | INFO |   lr: 0.0005
+2022-12-21,10:01:07 | INFO |   model: ViT-B-16
+2022-12-21,10:01:07 | INFO |   name: epoch_23.pt
+2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
+2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
+2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
+2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt
+2022-12-21,10:01:07 | INFO |   pretrained_image: False
+2022-12-21,10:01:07 | INFO |   rank: 0
+2022-12-21,10:01:07 | INFO |   report_to: 
+2022-12-21,10:01:07 | INFO |   resume: None
+2022-12-21,10:01:07 | INFO |   save_frequency: 1
+2022-12-21,10:01:07 | INFO |   save_most_recent: False
+2022-12-21,10:01:07 | INFO |   seed: 0
+2022-12-21,10:01:07 | INFO |   skip_scheduler: False
+2022-12-21,10:01:07 | INFO |   tensorboard: False
+2022-12-21,10:01:07 | INFO |   tensorboard_path: 
+2022-12-21,10:01:07 | INFO |   torchscript: False
+2022-12-21,10:01:07 | INFO |   trace: False
+2022-12-21,10:01:07 | INFO |   train_data: None
+2022-12-21,10:01:07 | INFO |   train_num_samples: None
+2022-12-21,10:01:07 | INFO |   use_bn_sync: False
+2022-12-21,10:01:07 | INFO |   val_data: None
+2022-12-21,10:01:07 | INFO |   val_frequency: 1
+2022-12-21,10:01:07 | INFO |   val_num_samples: None
+2022-12-21,10:01:07 | INFO |   wandb: False
+2022-12-21,10:01:07 | INFO |   wandb_notes: 
+2022-12-21,10:01:07 | INFO |   warmup: 10000
+2022-12-21,10:01:07 | INFO |   wd: 0.2
+2022-12-21,10:01:07 | INFO |   workers: 1
+2022-12-21,10:01:07 | INFO |   world_size: 1
+2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:01:07 | INFO | Building zero-shot classifier
+2022-12-21,10:01:25 | INFO | Using classifier
+2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5731	imagenet-zeroshot-val-top5: 0.8431
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ca449d7b14a03f94bc5e38cd3080089f20915297
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_23.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_23.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..32d13b1995218fd01adeb10f3552e8717b48a077
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.57296, "imagenet-zeroshot-val-top5": 0.84176}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..4d36d3810eb8400f5a532a11571e910b3000d401
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt).
+2022-12-21,10:07:24 | INFO | Model:
+2022-12-21,10:07:24 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:24 | INFO | Params:
+2022-12-21,10:07:24 | INFO |   batch_size: 256
+2022-12-21,10:07:24 | INFO |   beta1: 0.9
+2022-12-21,10:07:24 | INFO |   beta2: 0.98
+2022-12-21,10:07:24 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/checkpoints
+2022-12-21,10:07:24 | INFO |   copy_codebase: False
+2022-12-21,10:07:24 | INFO |   csv_caption_key: title
+2022-12-21,10:07:24 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:24 | INFO |   csv_separator: 	
+2022-12-21,10:07:24 | INFO |   dataset_resampled: False
+2022-12-21,10:07:24 | INFO |   dataset_type: auto
+2022-12-21,10:07:24 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:24 | INFO |   debug: False
+2022-12-21,10:07:24 | INFO |   device: cuda:0
+2022-12-21,10:07:24 | INFO |   dist_backend: nccl
+2022-12-21,10:07:24 | INFO |   dist_url: env://
+2022-12-21,10:07:24 | INFO |   distributed: False
+2022-12-21,10:07:24 | INFO |   epochs: 32
+2022-12-21,10:07:24 | INFO |   eps: 1e-06
+2022-12-21,10:07:24 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:24 | INFO |   gather_with_grad: False
+2022-12-21,10:07:24 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:24 | INFO |   horovod: False
+2022-12-21,10:07:24 | INFO |   image_mean: None
+2022-12-21,10:07:24 | INFO |   image_std: None
+2022-12-21,10:07:24 | INFO |   imagenet_v2: None
+2022-12-21,10:07:24 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:24 | INFO |   local_loss: False
+2022-12-21,10:07:24 | INFO |   local_rank: 0
+2022-12-21,10:07:24 | INFO |   lock_image: False
+2022-12-21,10:07:24 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:24 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:24 | INFO |   log_level: 20
+2022-12-21,10:07:24 | INFO |   log_local: False
+2022-12-21,10:07:24 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/out.log
+2022-12-21,10:07:24 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:24 | INFO |   lr: 0.0005
+2022-12-21,10:07:24 | INFO |   model: ViT-B-16
+2022-12-21,10:07:24 | INFO |   name: epoch_24.pt
+2022-12-21,10:07:24 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:24 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:24 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:24 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt
+2022-12-21,10:07:24 | INFO |   pretrained_image: False
+2022-12-21,10:07:24 | INFO |   rank: 0
+2022-12-21,10:07:24 | INFO |   report_to: 
+2022-12-21,10:07:24 | INFO |   resume: None
+2022-12-21,10:07:24 | INFO |   save_frequency: 1
+2022-12-21,10:07:24 | INFO |   save_most_recent: False
+2022-12-21,10:07:24 | INFO |   seed: 0
+2022-12-21,10:07:24 | INFO |   skip_scheduler: False
+2022-12-21,10:07:24 | INFO |   tensorboard: False
+2022-12-21,10:07:24 | INFO |   tensorboard_path: 
+2022-12-21,10:07:24 | INFO |   torchscript: False
+2022-12-21,10:07:24 | INFO |   trace: False
+2022-12-21,10:07:24 | INFO |   train_data: None
+2022-12-21,10:07:24 | INFO |   train_num_samples: None
+2022-12-21,10:07:24 | INFO |   use_bn_sync: False
+2022-12-21,10:07:24 | INFO |   val_data: None
+2022-12-21,10:07:24 | INFO |   val_frequency: 1
+2022-12-21,10:07:24 | INFO |   val_num_samples: None
+2022-12-21,10:07:24 | INFO |   wandb: False
+2022-12-21,10:07:24 | INFO |   wandb_notes: 
+2022-12-21,10:07:24 | INFO |   warmup: 10000
+2022-12-21,10:07:24 | INFO |   wd: 0.2
+2022-12-21,10:07:24 | INFO |   workers: 1
+2022-12-21,10:07:24 | INFO |   world_size: 1
+2022-12-21,10:07:24 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:25 | INFO | Building zero-shot classifier
+2022-12-21,10:07:43 | INFO | Using classifier
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5730	imagenet-zeroshot-val-top5: 0.8418
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..57f9126bc61f6dfde7ff66f45866aaf42770301c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_24.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_24.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d6b673a2a506bf5a45c2c51423b3cdef5b9b4196
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.57334, "imagenet-zeroshot-val-top5": 0.84134}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..370acc0a1cbc38a57f00f1f3b1a21467ee0f251c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt).
+2022-12-21,10:07:24 | INFO | Model:
+2022-12-21,10:07:24 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:24 | INFO | Params:
+2022-12-21,10:07:24 | INFO |   batch_size: 256
+2022-12-21,10:07:24 | INFO |   beta1: 0.9
+2022-12-21,10:07:24 | INFO |   beta2: 0.98
+2022-12-21,10:07:24 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/checkpoints
+2022-12-21,10:07:24 | INFO |   copy_codebase: False
+2022-12-21,10:07:24 | INFO |   csv_caption_key: title
+2022-12-21,10:07:24 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:24 | INFO |   csv_separator: 	
+2022-12-21,10:07:24 | INFO |   dataset_resampled: False
+2022-12-21,10:07:24 | INFO |   dataset_type: auto
+2022-12-21,10:07:24 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:24 | INFO |   debug: False
+2022-12-21,10:07:24 | INFO |   device: cuda:0
+2022-12-21,10:07:24 | INFO |   dist_backend: nccl
+2022-12-21,10:07:24 | INFO |   dist_url: env://
+2022-12-21,10:07:24 | INFO |   distributed: False
+2022-12-21,10:07:24 | INFO |   epochs: 32
+2022-12-21,10:07:24 | INFO |   eps: 1e-06
+2022-12-21,10:07:24 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:24 | INFO |   gather_with_grad: False
+2022-12-21,10:07:24 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:24 | INFO |   horovod: False
+2022-12-21,10:07:24 | INFO |   image_mean: None
+2022-12-21,10:07:24 | INFO |   image_std: None
+2022-12-21,10:07:24 | INFO |   imagenet_v2: None
+2022-12-21,10:07:24 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:24 | INFO |   local_loss: False
+2022-12-21,10:07:24 | INFO |   local_rank: 0
+2022-12-21,10:07:24 | INFO |   lock_image: False
+2022-12-21,10:07:24 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:24 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:24 | INFO |   log_level: 20
+2022-12-21,10:07:24 | INFO |   log_local: False
+2022-12-21,10:07:24 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/out.log
+2022-12-21,10:07:24 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:24 | INFO |   lr: 0.0005
+2022-12-21,10:07:24 | INFO |   model: ViT-B-16
+2022-12-21,10:07:24 | INFO |   name: epoch_25.pt
+2022-12-21,10:07:24 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:24 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:24 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:24 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt
+2022-12-21,10:07:24 | INFO |   pretrained_image: False
+2022-12-21,10:07:24 | INFO |   rank: 0
+2022-12-21,10:07:24 | INFO |   report_to: 
+2022-12-21,10:07:24 | INFO |   resume: None
+2022-12-21,10:07:24 | INFO |   save_frequency: 1
+2022-12-21,10:07:24 | INFO |   save_most_recent: False
+2022-12-21,10:07:24 | INFO |   seed: 0
+2022-12-21,10:07:24 | INFO |   skip_scheduler: False
+2022-12-21,10:07:24 | INFO |   tensorboard: False
+2022-12-21,10:07:24 | INFO |   tensorboard_path: 
+2022-12-21,10:07:24 | INFO |   torchscript: False
+2022-12-21,10:07:24 | INFO |   trace: False
+2022-12-21,10:07:24 | INFO |   train_data: None
+2022-12-21,10:07:24 | INFO |   train_num_samples: None
+2022-12-21,10:07:24 | INFO |   use_bn_sync: False
+2022-12-21,10:07:24 | INFO |   val_data: None
+2022-12-21,10:07:24 | INFO |   val_frequency: 1
+2022-12-21,10:07:24 | INFO |   val_num_samples: None
+2022-12-21,10:07:24 | INFO |   wandb: False
+2022-12-21,10:07:24 | INFO |   wandb_notes: 
+2022-12-21,10:07:24 | INFO |   warmup: 10000
+2022-12-21,10:07:24 | INFO |   wd: 0.2
+2022-12-21,10:07:24 | INFO |   workers: 1
+2022-12-21,10:07:24 | INFO |   world_size: 1
+2022-12-21,10:07:24 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:24 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:24 | INFO | Building zero-shot classifier
+2022-12-21,10:07:42 | INFO | Using classifier
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5733	imagenet-zeroshot-val-top5: 0.8413
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f5662720be25bbfa98eb596f267ca954a4004e7f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_25.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_25.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d28164394d98f14cdd4203b51e24ec6f14ad43d3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.57476, "imagenet-zeroshot-val-top5": 0.84552}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..2cdde3c4705c9816a19bef6d81c95d591e92e809
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt).
+2022-12-21,10:07:25 | INFO | Model:
+2022-12-21,10:07:25 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:25 | INFO | Params:
+2022-12-21,10:07:25 | INFO |   batch_size: 256
+2022-12-21,10:07:25 | INFO |   beta1: 0.9
+2022-12-21,10:07:25 | INFO |   beta2: 0.98
+2022-12-21,10:07:25 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/checkpoints
+2022-12-21,10:07:25 | INFO |   copy_codebase: False
+2022-12-21,10:07:25 | INFO |   csv_caption_key: title
+2022-12-21,10:07:25 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:25 | INFO |   csv_separator: 	
+2022-12-21,10:07:25 | INFO |   dataset_resampled: False
+2022-12-21,10:07:25 | INFO |   dataset_type: auto
+2022-12-21,10:07:25 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:25 | INFO |   debug: False
+2022-12-21,10:07:25 | INFO |   device: cuda:0
+2022-12-21,10:07:25 | INFO |   dist_backend: nccl
+2022-12-21,10:07:25 | INFO |   dist_url: env://
+2022-12-21,10:07:25 | INFO |   distributed: False
+2022-12-21,10:07:25 | INFO |   epochs: 32
+2022-12-21,10:07:25 | INFO |   eps: 1e-06
+2022-12-21,10:07:25 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:25 | INFO |   gather_with_grad: False
+2022-12-21,10:07:25 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:25 | INFO |   horovod: False
+2022-12-21,10:07:25 | INFO |   image_mean: None
+2022-12-21,10:07:25 | INFO |   image_std: None
+2022-12-21,10:07:25 | INFO |   imagenet_v2: None
+2022-12-21,10:07:25 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:25 | INFO |   local_loss: False
+2022-12-21,10:07:25 | INFO |   local_rank: 0
+2022-12-21,10:07:25 | INFO |   lock_image: False
+2022-12-21,10:07:25 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:25 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:25 | INFO |   log_level: 20
+2022-12-21,10:07:25 | INFO |   log_local: False
+2022-12-21,10:07:25 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/out.log
+2022-12-21,10:07:25 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:25 | INFO |   lr: 0.0005
+2022-12-21,10:07:25 | INFO |   model: ViT-B-16
+2022-12-21,10:07:25 | INFO |   name: epoch_26.pt
+2022-12-21,10:07:25 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:25 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:25 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:25 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt
+2022-12-21,10:07:25 | INFO |   pretrained_image: False
+2022-12-21,10:07:25 | INFO |   rank: 0
+2022-12-21,10:07:25 | INFO |   report_to: 
+2022-12-21,10:07:25 | INFO |   resume: None
+2022-12-21,10:07:25 | INFO |   save_frequency: 1
+2022-12-21,10:07:25 | INFO |   save_most_recent: False
+2022-12-21,10:07:25 | INFO |   seed: 0
+2022-12-21,10:07:25 | INFO |   skip_scheduler: False
+2022-12-21,10:07:25 | INFO |   tensorboard: False
+2022-12-21,10:07:25 | INFO |   tensorboard_path: 
+2022-12-21,10:07:25 | INFO |   torchscript: False
+2022-12-21,10:07:25 | INFO |   trace: False
+2022-12-21,10:07:25 | INFO |   train_data: None
+2022-12-21,10:07:25 | INFO |   train_num_samples: None
+2022-12-21,10:07:25 | INFO |   use_bn_sync: False
+2022-12-21,10:07:25 | INFO |   val_data: None
+2022-12-21,10:07:25 | INFO |   val_frequency: 1
+2022-12-21,10:07:25 | INFO |   val_num_samples: None
+2022-12-21,10:07:25 | INFO |   wandb: False
+2022-12-21,10:07:25 | INFO |   wandb_notes: 
+2022-12-21,10:07:25 | INFO |   warmup: 10000
+2022-12-21,10:07:25 | INFO |   wd: 0.2
+2022-12-21,10:07:25 | INFO |   workers: 1
+2022-12-21,10:07:25 | INFO |   world_size: 1
+2022-12-21,10:07:25 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:25 | INFO | Building zero-shot classifier
+2022-12-21,10:07:43 | INFO | Using classifier
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5748	imagenet-zeroshot-val-top5: 0.8455
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..591c9e1242072bde154f5716477767ca7b56106c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_26.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_26.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9d35a11896170766bce05447d131da877bb67016
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5813, "imagenet-zeroshot-val-top5": 0.84764}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..90afa8b801fde6216ed59397cdeac6c611692f6d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt).
+2022-12-21,10:07:25 | INFO | Model:
+2022-12-21,10:07:25 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:07:25 | INFO | Params:
+2022-12-21,10:07:25 | INFO |   batch_size: 256
+2022-12-21,10:07:25 | INFO |   beta1: 0.9
+2022-12-21,10:07:25 | INFO |   beta2: 0.98
+2022-12-21,10:07:25 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/checkpoints
+2022-12-21,10:07:25 | INFO |   copy_codebase: False
+2022-12-21,10:07:25 | INFO |   csv_caption_key: title
+2022-12-21,10:07:25 | INFO |   csv_img_key: filepath
+2022-12-21,10:07:25 | INFO |   csv_separator: 	
+2022-12-21,10:07:25 | INFO |   dataset_resampled: False
+2022-12-21,10:07:25 | INFO |   dataset_type: auto
+2022-12-21,10:07:25 | INFO |   ddp_static_graph: False
+2022-12-21,10:07:25 | INFO |   debug: False
+2022-12-21,10:07:25 | INFO |   device: cuda:0
+2022-12-21,10:07:25 | INFO |   dist_backend: nccl
+2022-12-21,10:07:25 | INFO |   dist_url: env://
+2022-12-21,10:07:25 | INFO |   distributed: False
+2022-12-21,10:07:25 | INFO |   epochs: 32
+2022-12-21,10:07:25 | INFO |   eps: 1e-06
+2022-12-21,10:07:25 | INFO |   force_quick_gelu: False
+2022-12-21,10:07:25 | INFO |   gather_with_grad: False
+2022-12-21,10:07:25 | INFO |   grad_checkpointing: False
+2022-12-21,10:07:25 | INFO |   horovod: False
+2022-12-21,10:07:25 | INFO |   image_mean: None
+2022-12-21,10:07:25 | INFO |   image_std: None
+2022-12-21,10:07:25 | INFO |   imagenet_v2: None
+2022-12-21,10:07:25 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:07:25 | INFO |   local_loss: False
+2022-12-21,10:07:25 | INFO |   local_rank: 0
+2022-12-21,10:07:25 | INFO |   lock_image: False
+2022-12-21,10:07:25 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:07:25 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:07:25 | INFO |   log_level: 20
+2022-12-21,10:07:25 | INFO |   log_local: False
+2022-12-21,10:07:25 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/out.log
+2022-12-21,10:07:25 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:07:25 | INFO |   lr: 0.0005
+2022-12-21,10:07:25 | INFO |   model: ViT-B-16
+2022-12-21,10:07:25 | INFO |   name: epoch_27.pt
+2022-12-21,10:07:25 | INFO |   no_set_device_rank: False
+2022-12-21,10:07:25 | INFO |   norm_gradient_clip: None
+2022-12-21,10:07:25 | INFO |   precision: amp_bfloat16
+2022-12-21,10:07:25 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt
+2022-12-21,10:07:25 | INFO |   pretrained_image: False
+2022-12-21,10:07:25 | INFO |   rank: 0
+2022-12-21,10:07:25 | INFO |   report_to: 
+2022-12-21,10:07:25 | INFO |   resume: None
+2022-12-21,10:07:25 | INFO |   save_frequency: 1
+2022-12-21,10:07:25 | INFO |   save_most_recent: False
+2022-12-21,10:07:25 | INFO |   seed: 0
+2022-12-21,10:07:25 | INFO |   skip_scheduler: False
+2022-12-21,10:07:25 | INFO |   tensorboard: False
+2022-12-21,10:07:25 | INFO |   tensorboard_path: 
+2022-12-21,10:07:25 | INFO |   torchscript: False
+2022-12-21,10:07:25 | INFO |   trace: False
+2022-12-21,10:07:25 | INFO |   train_data: None
+2022-12-21,10:07:25 | INFO |   train_num_samples: None
+2022-12-21,10:07:25 | INFO |   use_bn_sync: False
+2022-12-21,10:07:25 | INFO |   val_data: None
+2022-12-21,10:07:25 | INFO |   val_frequency: 1
+2022-12-21,10:07:25 | INFO |   val_num_samples: None
+2022-12-21,10:07:25 | INFO |   wandb: False
+2022-12-21,10:07:25 | INFO |   wandb_notes: 
+2022-12-21,10:07:25 | INFO |   warmup: 10000
+2022-12-21,10:07:25 | INFO |   wd: 0.2
+2022-12-21,10:07:25 | INFO |   workers: 1
+2022-12-21,10:07:25 | INFO |   world_size: 1
+2022-12-21,10:07:25 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:07:25 | INFO | Building zero-shot classifier
+2022-12-21,10:07:43 | INFO | Using classifier
+2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5813	imagenet-zeroshot-val-top5: 0.8476
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..3089eb44b45a690ac2643a0d524334f2352368e0
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_27.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_27.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1e90869c1da9e2dd07d5561f66d811e075522c11
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5849, "imagenet-zeroshot-val-top5": 0.85174}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..12d6c9f0415e2b048ce73dcce3559c10502588d3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:13:33 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:33 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:34 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt).
+2022-12-21,10:13:41 | INFO | Model:
+2022-12-21,10:13:41 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:42 | INFO | Params:
+2022-12-21,10:13:42 | INFO |   batch_size: 256
+2022-12-21,10:13:42 | INFO |   beta1: 0.9
+2022-12-21,10:13:42 | INFO |   beta2: 0.98
+2022-12-21,10:13:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/checkpoints
+2022-12-21,10:13:42 | INFO |   copy_codebase: False
+2022-12-21,10:13:42 | INFO |   csv_caption_key: title
+2022-12-21,10:13:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:42 | INFO |   csv_separator: 	
+2022-12-21,10:13:42 | INFO |   dataset_resampled: False
+2022-12-21,10:13:42 | INFO |   dataset_type: auto
+2022-12-21,10:13:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:42 | INFO |   debug: False
+2022-12-21,10:13:42 | INFO |   device: cuda:0
+2022-12-21,10:13:42 | INFO |   dist_backend: nccl
+2022-12-21,10:13:42 | INFO |   dist_url: env://
+2022-12-21,10:13:42 | INFO |   distributed: False
+2022-12-21,10:13:42 | INFO |   epochs: 32
+2022-12-21,10:13:42 | INFO |   eps: 1e-06
+2022-12-21,10:13:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:42 | INFO |   gather_with_grad: False
+2022-12-21,10:13:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:42 | INFO |   horovod: False
+2022-12-21,10:13:42 | INFO |   image_mean: None
+2022-12-21,10:13:42 | INFO |   image_std: None
+2022-12-21,10:13:42 | INFO |   imagenet_v2: None
+2022-12-21,10:13:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:42 | INFO |   local_loss: False
+2022-12-21,10:13:42 | INFO |   local_rank: 0
+2022-12-21,10:13:42 | INFO |   lock_image: False
+2022-12-21,10:13:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:42 | INFO |   log_level: 20
+2022-12-21,10:13:42 | INFO |   log_local: False
+2022-12-21,10:13:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/out.log
+2022-12-21,10:13:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:42 | INFO |   lr: 0.0005
+2022-12-21,10:13:42 | INFO |   model: ViT-B-16
+2022-12-21,10:13:42 | INFO |   name: epoch_28.pt
+2022-12-21,10:13:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt
+2022-12-21,10:13:42 | INFO |   pretrained_image: False
+2022-12-21,10:13:42 | INFO |   rank: 0
+2022-12-21,10:13:42 | INFO |   report_to: 
+2022-12-21,10:13:42 | INFO |   resume: None
+2022-12-21,10:13:42 | INFO |   save_frequency: 1
+2022-12-21,10:13:42 | INFO |   save_most_recent: False
+2022-12-21,10:13:42 | INFO |   seed: 0
+2022-12-21,10:13:42 | INFO |   skip_scheduler: False
+2022-12-21,10:13:42 | INFO |   tensorboard: False
+2022-12-21,10:13:42 | INFO |   tensorboard_path: 
+2022-12-21,10:13:42 | INFO |   torchscript: False
+2022-12-21,10:13:42 | INFO |   trace: False
+2022-12-21,10:13:42 | INFO |   train_data: None
+2022-12-21,10:13:42 | INFO |   train_num_samples: None
+2022-12-21,10:13:42 | INFO |   use_bn_sync: False
+2022-12-21,10:13:42 | INFO |   val_data: None
+2022-12-21,10:13:42 | INFO |   val_frequency: 1
+2022-12-21,10:13:42 | INFO |   val_num_samples: None
+2022-12-21,10:13:42 | INFO |   wandb: False
+2022-12-21,10:13:42 | INFO |   wandb_notes: 
+2022-12-21,10:13:42 | INFO |   warmup: 10000
+2022-12-21,10:13:42 | INFO |   wd: 0.2
+2022-12-21,10:13:42 | INFO |   workers: 1
+2022-12-21,10:13:42 | INFO |   world_size: 1
+2022-12-21,10:13:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:42 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:42 | INFO | Building zero-shot classifier
+2022-12-21,10:14:00 | INFO | Using classifier
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5849	imagenet-zeroshot-val-top5: 0.8517
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..70bd1c051949526183342799948b58f9e6985ddd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_28.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_28.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a7f35d61987489afb4e819da8e04398276bebecc
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.57956, "imagenet-zeroshot-val-top5": 0.8498}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..0d12d0531a8c59ff234d7b3b7294460b449cd438
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt).
+2022-12-21,10:13:40 | INFO | Model:
+2022-12-21,10:13:40 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:40 | INFO | Params:
+2022-12-21,10:13:40 | INFO |   batch_size: 256
+2022-12-21,10:13:40 | INFO |   beta1: 0.9
+2022-12-21,10:13:40 | INFO |   beta2: 0.98
+2022-12-21,10:13:40 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/checkpoints
+2022-12-21,10:13:40 | INFO |   copy_codebase: False
+2022-12-21,10:13:40 | INFO |   csv_caption_key: title
+2022-12-21,10:13:40 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:40 | INFO |   csv_separator: 	
+2022-12-21,10:13:40 | INFO |   dataset_resampled: False
+2022-12-21,10:13:40 | INFO |   dataset_type: auto
+2022-12-21,10:13:40 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:40 | INFO |   debug: False
+2022-12-21,10:13:40 | INFO |   device: cuda:0
+2022-12-21,10:13:40 | INFO |   dist_backend: nccl
+2022-12-21,10:13:40 | INFO |   dist_url: env://
+2022-12-21,10:13:40 | INFO |   distributed: False
+2022-12-21,10:13:40 | INFO |   epochs: 32
+2022-12-21,10:13:40 | INFO |   eps: 1e-06
+2022-12-21,10:13:40 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:40 | INFO |   gather_with_grad: False
+2022-12-21,10:13:40 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:40 | INFO |   horovod: False
+2022-12-21,10:13:40 | INFO |   image_mean: None
+2022-12-21,10:13:40 | INFO |   image_std: None
+2022-12-21,10:13:40 | INFO |   imagenet_v2: None
+2022-12-21,10:13:40 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:40 | INFO |   local_loss: False
+2022-12-21,10:13:40 | INFO |   local_rank: 0
+2022-12-21,10:13:40 | INFO |   lock_image: False
+2022-12-21,10:13:40 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:40 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:40 | INFO |   log_level: 20
+2022-12-21,10:13:40 | INFO |   log_local: False
+2022-12-21,10:13:40 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/out.log
+2022-12-21,10:13:40 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:40 | INFO |   lr: 0.0005
+2022-12-21,10:13:40 | INFO |   model: ViT-B-16
+2022-12-21,10:13:40 | INFO |   name: epoch_29.pt
+2022-12-21,10:13:40 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:40 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:40 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:40 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt
+2022-12-21,10:13:40 | INFO |   pretrained_image: False
+2022-12-21,10:13:40 | INFO |   rank: 0
+2022-12-21,10:13:40 | INFO |   report_to: 
+2022-12-21,10:13:40 | INFO |   resume: None
+2022-12-21,10:13:40 | INFO |   save_frequency: 1
+2022-12-21,10:13:40 | INFO |   save_most_recent: False
+2022-12-21,10:13:40 | INFO |   seed: 0
+2022-12-21,10:13:40 | INFO |   skip_scheduler: False
+2022-12-21,10:13:40 | INFO |   tensorboard: False
+2022-12-21,10:13:40 | INFO |   tensorboard_path: 
+2022-12-21,10:13:40 | INFO |   torchscript: False
+2022-12-21,10:13:40 | INFO |   trace: False
+2022-12-21,10:13:40 | INFO |   train_data: None
+2022-12-21,10:13:40 | INFO |   train_num_samples: None
+2022-12-21,10:13:40 | INFO |   use_bn_sync: False
+2022-12-21,10:13:40 | INFO |   val_data: None
+2022-12-21,10:13:40 | INFO |   val_frequency: 1
+2022-12-21,10:13:40 | INFO |   val_num_samples: None
+2022-12-21,10:13:40 | INFO |   wandb: False
+2022-12-21,10:13:40 | INFO |   wandb_notes: 
+2022-12-21,10:13:40 | INFO |   warmup: 10000
+2022-12-21,10:13:40 | INFO |   wd: 0.2
+2022-12-21,10:13:40 | INFO |   workers: 1
+2022-12-21,10:13:40 | INFO |   world_size: 1
+2022-12-21,10:13:40 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:40 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:40 | INFO | Building zero-shot classifier
+2022-12-21,10:13:58 | INFO | Using classifier
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5796	imagenet-zeroshot-val-top5: 0.8498
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..1daa8e6a789c6631518b5b94dd4f3c4cba6dfcca
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_29.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_29.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..64e4a05a3aac351917c5ba97f90ca87fd1f6e31f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.35972, "imagenet-zeroshot-val-top5": 0.6317}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..50d5e576057be1e8a694049f1e8d64c7a5cde5af
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt).
+2022-12-21,10:13:41 | INFO | Model:
+2022-12-21,10:13:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:42 | INFO | Params:
+2022-12-21,10:13:42 | INFO |   batch_size: 256
+2022-12-21,10:13:42 | INFO |   beta1: 0.9
+2022-12-21,10:13:42 | INFO |   beta2: 0.98
+2022-12-21,10:13:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/checkpoints
+2022-12-21,10:13:42 | INFO |   copy_codebase: False
+2022-12-21,10:13:42 | INFO |   csv_caption_key: title
+2022-12-21,10:13:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:42 | INFO |   csv_separator: 	
+2022-12-21,10:13:42 | INFO |   dataset_resampled: False
+2022-12-21,10:13:42 | INFO |   dataset_type: auto
+2022-12-21,10:13:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:42 | INFO |   debug: False
+2022-12-21,10:13:42 | INFO |   device: cuda:0
+2022-12-21,10:13:42 | INFO |   dist_backend: nccl
+2022-12-21,10:13:42 | INFO |   dist_url: env://
+2022-12-21,10:13:42 | INFO |   distributed: False
+2022-12-21,10:13:42 | INFO |   epochs: 32
+2022-12-21,10:13:42 | INFO |   eps: 1e-06
+2022-12-21,10:13:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:42 | INFO |   gather_with_grad: False
+2022-12-21,10:13:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:42 | INFO |   horovod: False
+2022-12-21,10:13:42 | INFO |   image_mean: None
+2022-12-21,10:13:42 | INFO |   image_std: None
+2022-12-21,10:13:42 | INFO |   imagenet_v2: None
+2022-12-21,10:13:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:42 | INFO |   local_loss: False
+2022-12-21,10:13:42 | INFO |   local_rank: 0
+2022-12-21,10:13:42 | INFO |   lock_image: False
+2022-12-21,10:13:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:42 | INFO |   log_level: 20
+2022-12-21,10:13:42 | INFO |   log_local: False
+2022-12-21,10:13:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/out.log
+2022-12-21,10:13:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:42 | INFO |   lr: 0.0005
+2022-12-21,10:13:42 | INFO |   model: ViT-B-16
+2022-12-21,10:13:42 | INFO |   name: epoch_3.pt
+2022-12-21,10:13:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt
+2022-12-21,10:13:42 | INFO |   pretrained_image: False
+2022-12-21,10:13:42 | INFO |   rank: 0
+2022-12-21,10:13:42 | INFO |   report_to: 
+2022-12-21,10:13:42 | INFO |   resume: None
+2022-12-21,10:13:42 | INFO |   save_frequency: 1
+2022-12-21,10:13:42 | INFO |   save_most_recent: False
+2022-12-21,10:13:42 | INFO |   seed: 0
+2022-12-21,10:13:42 | INFO |   skip_scheduler: False
+2022-12-21,10:13:42 | INFO |   tensorboard: False
+2022-12-21,10:13:42 | INFO |   tensorboard_path: 
+2022-12-21,10:13:42 | INFO |   torchscript: False
+2022-12-21,10:13:42 | INFO |   trace: False
+2022-12-21,10:13:42 | INFO |   train_data: None
+2022-12-21,10:13:42 | INFO |   train_num_samples: None
+2022-12-21,10:13:42 | INFO |   use_bn_sync: False
+2022-12-21,10:13:42 | INFO |   val_data: None
+2022-12-21,10:13:42 | INFO |   val_frequency: 1
+2022-12-21,10:13:42 | INFO |   val_num_samples: None
+2022-12-21,10:13:42 | INFO |   wandb: False
+2022-12-21,10:13:42 | INFO |   wandb_notes: 
+2022-12-21,10:13:42 | INFO |   warmup: 10000
+2022-12-21,10:13:42 | INFO |   wd: 0.2
+2022-12-21,10:13:42 | INFO |   workers: 1
+2022-12-21,10:13:42 | INFO |   world_size: 1
+2022-12-21,10:13:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:42 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:42 | INFO | Building zero-shot classifier
+2022-12-21,10:14:00 | INFO | Using classifier
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.3597	imagenet-zeroshot-val-top5: 0.6317
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e3f15ea12fc4e4340fa4f50a91cc85d069794235
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_3.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_3.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..930c500a7b956c3770f524879dc71db52b75b476
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.54598, "imagenet-zeroshot-val-top5": 0.82246}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..722393871129ed0381acd2df6b85eb83eeb8e21e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt).
+2022-12-21,10:13:41 | INFO | Model:
+2022-12-21,10:13:41 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:13:41 | INFO | Params:
+2022-12-21,10:13:41 | INFO |   batch_size: 256
+2022-12-21,10:13:41 | INFO |   beta1: 0.9
+2022-12-21,10:13:41 | INFO |   beta2: 0.98
+2022-12-21,10:13:41 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/checkpoints
+2022-12-21,10:13:41 | INFO |   copy_codebase: False
+2022-12-21,10:13:41 | INFO |   csv_caption_key: title
+2022-12-21,10:13:41 | INFO |   csv_img_key: filepath
+2022-12-21,10:13:41 | INFO |   csv_separator: 	
+2022-12-21,10:13:41 | INFO |   dataset_resampled: False
+2022-12-21,10:13:41 | INFO |   dataset_type: auto
+2022-12-21,10:13:41 | INFO |   ddp_static_graph: False
+2022-12-21,10:13:41 | INFO |   debug: False
+2022-12-21,10:13:41 | INFO |   device: cuda:0
+2022-12-21,10:13:41 | INFO |   dist_backend: nccl
+2022-12-21,10:13:41 | INFO |   dist_url: env://
+2022-12-21,10:13:41 | INFO |   distributed: False
+2022-12-21,10:13:41 | INFO |   epochs: 32
+2022-12-21,10:13:41 | INFO |   eps: 1e-06
+2022-12-21,10:13:41 | INFO |   force_quick_gelu: False
+2022-12-21,10:13:41 | INFO |   gather_with_grad: False
+2022-12-21,10:13:41 | INFO |   grad_checkpointing: False
+2022-12-21,10:13:41 | INFO |   horovod: False
+2022-12-21,10:13:41 | INFO |   image_mean: None
+2022-12-21,10:13:41 | INFO |   image_std: None
+2022-12-21,10:13:41 | INFO |   imagenet_v2: None
+2022-12-21,10:13:41 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:13:41 | INFO |   local_loss: False
+2022-12-21,10:13:41 | INFO |   local_rank: 0
+2022-12-21,10:13:41 | INFO |   lock_image: False
+2022-12-21,10:13:41 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:13:41 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:13:41 | INFO |   log_level: 20
+2022-12-21,10:13:41 | INFO |   log_local: False
+2022-12-21,10:13:41 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/out.log
+2022-12-21,10:13:41 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:13:41 | INFO |   lr: 0.0005
+2022-12-21,10:13:41 | INFO |   model: ViT-B-16
+2022-12-21,10:13:41 | INFO |   name: epoch_30.pt
+2022-12-21,10:13:41 | INFO |   no_set_device_rank: False
+2022-12-21,10:13:41 | INFO |   norm_gradient_clip: None
+2022-12-21,10:13:41 | INFO |   precision: amp_bfloat16
+2022-12-21,10:13:41 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt
+2022-12-21,10:13:41 | INFO |   pretrained_image: False
+2022-12-21,10:13:41 | INFO |   rank: 0
+2022-12-21,10:13:41 | INFO |   report_to: 
+2022-12-21,10:13:41 | INFO |   resume: None
+2022-12-21,10:13:41 | INFO |   save_frequency: 1
+2022-12-21,10:13:41 | INFO |   save_most_recent: False
+2022-12-21,10:13:41 | INFO |   seed: 0
+2022-12-21,10:13:41 | INFO |   skip_scheduler: False
+2022-12-21,10:13:41 | INFO |   tensorboard: False
+2022-12-21,10:13:41 | INFO |   tensorboard_path: 
+2022-12-21,10:13:41 | INFO |   torchscript: False
+2022-12-21,10:13:41 | INFO |   trace: False
+2022-12-21,10:13:41 | INFO |   train_data: None
+2022-12-21,10:13:41 | INFO |   train_num_samples: None
+2022-12-21,10:13:41 | INFO |   use_bn_sync: False
+2022-12-21,10:13:41 | INFO |   val_data: None
+2022-12-21,10:13:41 | INFO |   val_frequency: 1
+2022-12-21,10:13:41 | INFO |   val_num_samples: None
+2022-12-21,10:13:41 | INFO |   wandb: False
+2022-12-21,10:13:41 | INFO |   wandb_notes: 
+2022-12-21,10:13:41 | INFO |   warmup: 10000
+2022-12-21,10:13:41 | INFO |   wd: 0.2
+2022-12-21,10:13:41 | INFO |   workers: 1
+2022-12-21,10:13:41 | INFO |   world_size: 1
+2022-12-21,10:13:41 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:13:41 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:13:41 | INFO | Building zero-shot classifier
+2022-12-21,10:13:59 | INFO | Using classifier
+2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5460	imagenet-zeroshot-val-top5: 0.8225
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..bc3b817367820fb79cf7d7f03bf832bb22e00931
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_30.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_30.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..acfe35f8d994925171f2c503bb765e1f73c1baa8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.58834, "imagenet-zeroshot-val-top5": 0.85554}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..2640c413fbf3c48e36556e62b1db1a786d883ab9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt).
+2022-12-21,10:19:58 | INFO | Model:
+2022-12-21,10:19:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:58 | INFO | Params:
+2022-12-21,10:19:58 | INFO |   batch_size: 256
+2022-12-21,10:19:58 | INFO |   beta1: 0.9
+2022-12-21,10:19:58 | INFO |   beta2: 0.98
+2022-12-21,10:19:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/checkpoints
+2022-12-21,10:19:58 | INFO |   copy_codebase: False
+2022-12-21,10:19:58 | INFO |   csv_caption_key: title
+2022-12-21,10:19:58 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:58 | INFO |   csv_separator: 	
+2022-12-21,10:19:58 | INFO |   dataset_resampled: False
+2022-12-21,10:19:58 | INFO |   dataset_type: auto
+2022-12-21,10:19:58 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:58 | INFO |   debug: False
+2022-12-21,10:19:58 | INFO |   device: cuda:0
+2022-12-21,10:19:58 | INFO |   dist_backend: nccl
+2022-12-21,10:19:58 | INFO |   dist_url: env://
+2022-12-21,10:19:58 | INFO |   distributed: False
+2022-12-21,10:19:58 | INFO |   epochs: 32
+2022-12-21,10:19:58 | INFO |   eps: 1e-06
+2022-12-21,10:19:58 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:58 | INFO |   gather_with_grad: False
+2022-12-21,10:19:58 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:58 | INFO |   horovod: False
+2022-12-21,10:19:58 | INFO |   image_mean: None
+2022-12-21,10:19:58 | INFO |   image_std: None
+2022-12-21,10:19:58 | INFO |   imagenet_v2: None
+2022-12-21,10:19:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:58 | INFO |   local_loss: False
+2022-12-21,10:19:58 | INFO |   local_rank: 0
+2022-12-21,10:19:58 | INFO |   lock_image: False
+2022-12-21,10:19:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:58 | INFO |   log_level: 20
+2022-12-21,10:19:58 | INFO |   log_local: False
+2022-12-21,10:19:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/out.log
+2022-12-21,10:19:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:58 | INFO |   lr: 0.0005
+2022-12-21,10:19:58 | INFO |   model: ViT-B-16
+2022-12-21,10:19:58 | INFO |   name: epoch_31.pt
+2022-12-21,10:19:58 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:58 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:58 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt
+2022-12-21,10:19:58 | INFO |   pretrained_image: False
+2022-12-21,10:19:58 | INFO |   rank: 0
+2022-12-21,10:19:58 | INFO |   report_to: 
+2022-12-21,10:19:58 | INFO |   resume: None
+2022-12-21,10:19:58 | INFO |   save_frequency: 1
+2022-12-21,10:19:58 | INFO |   save_most_recent: False
+2022-12-21,10:19:58 | INFO |   seed: 0
+2022-12-21,10:19:58 | INFO |   skip_scheduler: False
+2022-12-21,10:19:58 | INFO |   tensorboard: False
+2022-12-21,10:19:58 | INFO |   tensorboard_path: 
+2022-12-21,10:19:58 | INFO |   torchscript: False
+2022-12-21,10:19:58 | INFO |   trace: False
+2022-12-21,10:19:58 | INFO |   train_data: None
+2022-12-21,10:19:58 | INFO |   train_num_samples: None
+2022-12-21,10:19:58 | INFO |   use_bn_sync: False
+2022-12-21,10:19:58 | INFO |   val_data: None
+2022-12-21,10:19:58 | INFO |   val_frequency: 1
+2022-12-21,10:19:58 | INFO |   val_num_samples: None
+2022-12-21,10:19:58 | INFO |   wandb: False
+2022-12-21,10:19:58 | INFO |   wandb_notes: 
+2022-12-21,10:19:58 | INFO |   warmup: 10000
+2022-12-21,10:19:58 | INFO |   wd: 0.2
+2022-12-21,10:19:58 | INFO |   workers: 1
+2022-12-21,10:19:58 | INFO |   world_size: 1
+2022-12-21,10:19:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:58 | INFO | Building zero-shot classifier
+2022-12-21,10:20:16 | INFO | Using classifier
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5883	imagenet-zeroshot-val-top5: 0.8555
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2f962c5acb3b365b69d095f4263daeaf40550711
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_31.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_31.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..8d7d632467abbfd020fcff64ce3c9bb86d557c6b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.59066, "imagenet-zeroshot-val-top5": 0.85842}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..8c70699652adecd5f7bf715367d9400c442590cb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt).
+2022-12-21,10:19:58 | INFO | Model:
+2022-12-21,10:19:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:58 | INFO | Params:
+2022-12-21,10:19:58 | INFO |   batch_size: 256
+2022-12-21,10:19:58 | INFO |   beta1: 0.9
+2022-12-21,10:19:58 | INFO |   beta2: 0.98
+2022-12-21,10:19:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/checkpoints
+2022-12-21,10:19:58 | INFO |   copy_codebase: False
+2022-12-21,10:19:58 | INFO |   csv_caption_key: title
+2022-12-21,10:19:58 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:58 | INFO |   csv_separator: 	
+2022-12-21,10:19:58 | INFO |   dataset_resampled: False
+2022-12-21,10:19:58 | INFO |   dataset_type: auto
+2022-12-21,10:19:58 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:58 | INFO |   debug: False
+2022-12-21,10:19:58 | INFO |   device: cuda:0
+2022-12-21,10:19:58 | INFO |   dist_backend: nccl
+2022-12-21,10:19:58 | INFO |   dist_url: env://
+2022-12-21,10:19:58 | INFO |   distributed: False
+2022-12-21,10:19:58 | INFO |   epochs: 32
+2022-12-21,10:19:58 | INFO |   eps: 1e-06
+2022-12-21,10:19:58 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:58 | INFO |   gather_with_grad: False
+2022-12-21,10:19:58 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:58 | INFO |   horovod: False
+2022-12-21,10:19:58 | INFO |   image_mean: None
+2022-12-21,10:19:58 | INFO |   image_std: None
+2022-12-21,10:19:58 | INFO |   imagenet_v2: None
+2022-12-21,10:19:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:58 | INFO |   local_loss: False
+2022-12-21,10:19:58 | INFO |   local_rank: 0
+2022-12-21,10:19:58 | INFO |   lock_image: False
+2022-12-21,10:19:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:58 | INFO |   log_level: 20
+2022-12-21,10:19:58 | INFO |   log_local: False
+2022-12-21,10:19:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/out.log
+2022-12-21,10:19:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:58 | INFO |   lr: 0.0005
+2022-12-21,10:19:58 | INFO |   model: ViT-B-16
+2022-12-21,10:19:58 | INFO |   name: epoch_32.pt
+2022-12-21,10:19:58 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:58 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:58 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt
+2022-12-21,10:19:58 | INFO |   pretrained_image: False
+2022-12-21,10:19:58 | INFO |   rank: 0
+2022-12-21,10:19:58 | INFO |   report_to: 
+2022-12-21,10:19:58 | INFO |   resume: None
+2022-12-21,10:19:58 | INFO |   save_frequency: 1
+2022-12-21,10:19:58 | INFO |   save_most_recent: False
+2022-12-21,10:19:58 | INFO |   seed: 0
+2022-12-21,10:19:58 | INFO |   skip_scheduler: False
+2022-12-21,10:19:58 | INFO |   tensorboard: False
+2022-12-21,10:19:58 | INFO |   tensorboard_path: 
+2022-12-21,10:19:58 | INFO |   torchscript: False
+2022-12-21,10:19:58 | INFO |   trace: False
+2022-12-21,10:19:58 | INFO |   train_data: None
+2022-12-21,10:19:58 | INFO |   train_num_samples: None
+2022-12-21,10:19:58 | INFO |   use_bn_sync: False
+2022-12-21,10:19:58 | INFO |   val_data: None
+2022-12-21,10:19:58 | INFO |   val_frequency: 1
+2022-12-21,10:19:58 | INFO |   val_num_samples: None
+2022-12-21,10:19:58 | INFO |   wandb: False
+2022-12-21,10:19:58 | INFO |   wandb_notes: 
+2022-12-21,10:19:58 | INFO |   warmup: 10000
+2022-12-21,10:19:58 | INFO |   wd: 0.2
+2022-12-21,10:19:58 | INFO |   workers: 1
+2022-12-21,10:19:58 | INFO |   world_size: 1
+2022-12-21,10:19:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:58 | INFO | Building zero-shot classifier
+2022-12-21,10:20:16 | INFO | Using classifier
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5907	imagenet-zeroshot-val-top5: 0.8584
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..475fdd9a6223bd710f8926c281a529db5f65f3b2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_32.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_32.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..daaab276bd4a225b0fddd2c45ea34cbc178bcdaa
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5897, "imagenet-zeroshot-val-top5": 0.85856}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..b0c4284e9661bdd439a98336e34125241414e12e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt).
+2022-12-21,10:19:59 | INFO | Model:
+2022-12-21,10:19:59 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:59 | INFO | Params:
+2022-12-21,10:19:59 | INFO |   batch_size: 256
+2022-12-21,10:19:59 | INFO |   beta1: 0.9
+2022-12-21,10:19:59 | INFO |   beta2: 0.98
+2022-12-21,10:19:59 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/checkpoints
+2022-12-21,10:19:59 | INFO |   copy_codebase: False
+2022-12-21,10:19:59 | INFO |   csv_caption_key: title
+2022-12-21,10:19:59 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:59 | INFO |   csv_separator: 	
+2022-12-21,10:19:59 | INFO |   dataset_resampled: False
+2022-12-21,10:19:59 | INFO |   dataset_type: auto
+2022-12-21,10:19:59 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:59 | INFO |   debug: False
+2022-12-21,10:19:59 | INFO |   device: cuda:0
+2022-12-21,10:19:59 | INFO |   dist_backend: nccl
+2022-12-21,10:19:59 | INFO |   dist_url: env://
+2022-12-21,10:19:59 | INFO |   distributed: False
+2022-12-21,10:19:59 | INFO |   epochs: 32
+2022-12-21,10:19:59 | INFO |   eps: 1e-06
+2022-12-21,10:19:59 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:59 | INFO |   gather_with_grad: False
+2022-12-21,10:19:59 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:59 | INFO |   horovod: False
+2022-12-21,10:19:59 | INFO |   image_mean: None
+2022-12-21,10:19:59 | INFO |   image_std: None
+2022-12-21,10:19:59 | INFO |   imagenet_v2: None
+2022-12-21,10:19:59 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:59 | INFO |   local_loss: False
+2022-12-21,10:19:59 | INFO |   local_rank: 0
+2022-12-21,10:19:59 | INFO |   lock_image: False
+2022-12-21,10:19:59 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:59 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:59 | INFO |   log_level: 20
+2022-12-21,10:19:59 | INFO |   log_local: False
+2022-12-21,10:19:59 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/out.log
+2022-12-21,10:19:59 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:59 | INFO |   lr: 0.0005
+2022-12-21,10:19:59 | INFO |   model: ViT-B-16
+2022-12-21,10:19:59 | INFO |   name: epoch_33.pt
+2022-12-21,10:19:59 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:59 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:59 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:59 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt
+2022-12-21,10:19:59 | INFO |   pretrained_image: False
+2022-12-21,10:19:59 | INFO |   rank: 0
+2022-12-21,10:19:59 | INFO |   report_to: 
+2022-12-21,10:19:59 | INFO |   resume: None
+2022-12-21,10:19:59 | INFO |   save_frequency: 1
+2022-12-21,10:19:59 | INFO |   save_most_recent: False
+2022-12-21,10:19:59 | INFO |   seed: 0
+2022-12-21,10:19:59 | INFO |   skip_scheduler: False
+2022-12-21,10:19:59 | INFO |   tensorboard: False
+2022-12-21,10:19:59 | INFO |   tensorboard_path: 
+2022-12-21,10:19:59 | INFO |   torchscript: False
+2022-12-21,10:19:59 | INFO |   trace: False
+2022-12-21,10:19:59 | INFO |   train_data: None
+2022-12-21,10:19:59 | INFO |   train_num_samples: None
+2022-12-21,10:19:59 | INFO |   use_bn_sync: False
+2022-12-21,10:19:59 | INFO |   val_data: None
+2022-12-21,10:19:59 | INFO |   val_frequency: 1
+2022-12-21,10:19:59 | INFO |   val_num_samples: None
+2022-12-21,10:19:59 | INFO |   wandb: False
+2022-12-21,10:19:59 | INFO |   wandb_notes: 
+2022-12-21,10:19:59 | INFO |   warmup: 10000
+2022-12-21,10:19:59 | INFO |   wd: 0.2
+2022-12-21,10:19:59 | INFO |   workers: 1
+2022-12-21,10:19:59 | INFO |   world_size: 1
+2022-12-21,10:19:59 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:59 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:59 | INFO | Building zero-shot classifier
+2022-12-21,10:20:17 | INFO | Using classifier
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5897	imagenet-zeroshot-val-top5: 0.8586
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..54177191620cc6bcae398ef160a917ff2bf1a317
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_33.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_33.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..eb83ba87f9849b9d5b812d20e1592bc3cee7b64f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.59294, "imagenet-zeroshot-val-top5": 0.8605}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..396784b60b47e24bea586ea1c6a3e135dc59b0ff
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt).
+2022-12-21,10:19:59 | INFO | Model:
+2022-12-21,10:19:59 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:19:59 | INFO | Params:
+2022-12-21,10:19:59 | INFO |   batch_size: 256
+2022-12-21,10:19:59 | INFO |   beta1: 0.9
+2022-12-21,10:19:59 | INFO |   beta2: 0.98
+2022-12-21,10:19:59 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/checkpoints
+2022-12-21,10:19:59 | INFO |   copy_codebase: False
+2022-12-21,10:19:59 | INFO |   csv_caption_key: title
+2022-12-21,10:19:59 | INFO |   csv_img_key: filepath
+2022-12-21,10:19:59 | INFO |   csv_separator: 	
+2022-12-21,10:19:59 | INFO |   dataset_resampled: False
+2022-12-21,10:19:59 | INFO |   dataset_type: auto
+2022-12-21,10:19:59 | INFO |   ddp_static_graph: False
+2022-12-21,10:19:59 | INFO |   debug: False
+2022-12-21,10:19:59 | INFO |   device: cuda:0
+2022-12-21,10:19:59 | INFO |   dist_backend: nccl
+2022-12-21,10:19:59 | INFO |   dist_url: env://
+2022-12-21,10:19:59 | INFO |   distributed: False
+2022-12-21,10:19:59 | INFO |   epochs: 32
+2022-12-21,10:19:59 | INFO |   eps: 1e-06
+2022-12-21,10:19:59 | INFO |   force_quick_gelu: False
+2022-12-21,10:19:59 | INFO |   gather_with_grad: False
+2022-12-21,10:19:59 | INFO |   grad_checkpointing: False
+2022-12-21,10:19:59 | INFO |   horovod: False
+2022-12-21,10:19:59 | INFO |   image_mean: None
+2022-12-21,10:19:59 | INFO |   image_std: None
+2022-12-21,10:19:59 | INFO |   imagenet_v2: None
+2022-12-21,10:19:59 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:19:59 | INFO |   local_loss: False
+2022-12-21,10:19:59 | INFO |   local_rank: 0
+2022-12-21,10:19:59 | INFO |   lock_image: False
+2022-12-21,10:19:59 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:19:59 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:19:59 | INFO |   log_level: 20
+2022-12-21,10:19:59 | INFO |   log_local: False
+2022-12-21,10:19:59 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/out.log
+2022-12-21,10:19:59 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:19:59 | INFO |   lr: 0.0005
+2022-12-21,10:19:59 | INFO |   model: ViT-B-16
+2022-12-21,10:19:59 | INFO |   name: epoch_34.pt
+2022-12-21,10:19:59 | INFO |   no_set_device_rank: False
+2022-12-21,10:19:59 | INFO |   norm_gradient_clip: None
+2022-12-21,10:19:59 | INFO |   precision: amp_bfloat16
+2022-12-21,10:19:59 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt
+2022-12-21,10:19:59 | INFO |   pretrained_image: False
+2022-12-21,10:19:59 | INFO |   rank: 0
+2022-12-21,10:19:59 | INFO |   report_to: 
+2022-12-21,10:19:59 | INFO |   resume: None
+2022-12-21,10:19:59 | INFO |   save_frequency: 1
+2022-12-21,10:19:59 | INFO |   save_most_recent: False
+2022-12-21,10:19:59 | INFO |   seed: 0
+2022-12-21,10:19:59 | INFO |   skip_scheduler: False
+2022-12-21,10:19:59 | INFO |   tensorboard: False
+2022-12-21,10:19:59 | INFO |   tensorboard_path: 
+2022-12-21,10:19:59 | INFO |   torchscript: False
+2022-12-21,10:19:59 | INFO |   trace: False
+2022-12-21,10:19:59 | INFO |   train_data: None
+2022-12-21,10:19:59 | INFO |   train_num_samples: None
+2022-12-21,10:19:59 | INFO |   use_bn_sync: False
+2022-12-21,10:19:59 | INFO |   val_data: None
+2022-12-21,10:19:59 | INFO |   val_frequency: 1
+2022-12-21,10:19:59 | INFO |   val_num_samples: None
+2022-12-21,10:19:59 | INFO |   wandb: False
+2022-12-21,10:19:59 | INFO |   wandb_notes: 
+2022-12-21,10:19:59 | INFO |   warmup: 10000
+2022-12-21,10:19:59 | INFO |   wd: 0.2
+2022-12-21,10:19:59 | INFO |   workers: 1
+2022-12-21,10:19:59 | INFO |   world_size: 1
+2022-12-21,10:19:59 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:19:59 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:19:59 | INFO | Building zero-shot classifier
+2022-12-21,10:20:17 | INFO | Using classifier
+2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5929	imagenet-zeroshot-val-top5: 0.8605
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..47f77a874d0716656b8275e4c3e84cfa74d179cb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_34.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_34.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4f79add7ed115246a894903e9e5e462cafaa59af
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.59976, "imagenet-zeroshot-val-top5": 0.8633}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..3cd93395034b78c93cab7bffbbac96fa8b964462
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt).
+2022-12-21,10:27:26 | INFO | Model:
+2022-12-21,10:27:26 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:26 | INFO | Params:
+2022-12-21,10:27:26 | INFO |   batch_size: 256
+2022-12-21,10:27:26 | INFO |   beta1: 0.9
+2022-12-21,10:27:26 | INFO |   beta2: 0.98
+2022-12-21,10:27:26 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/checkpoints
+2022-12-21,10:27:26 | INFO |   copy_codebase: False
+2022-12-21,10:27:26 | INFO |   csv_caption_key: title
+2022-12-21,10:27:26 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:26 | INFO |   csv_separator: 	
+2022-12-21,10:27:26 | INFO |   dataset_resampled: False
+2022-12-21,10:27:26 | INFO |   dataset_type: auto
+2022-12-21,10:27:26 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:26 | INFO |   debug: False
+2022-12-21,10:27:26 | INFO |   device: cuda:0
+2022-12-21,10:27:26 | INFO |   dist_backend: nccl
+2022-12-21,10:27:26 | INFO |   dist_url: env://
+2022-12-21,10:27:26 | INFO |   distributed: False
+2022-12-21,10:27:26 | INFO |   epochs: 32
+2022-12-21,10:27:26 | INFO |   eps: 1e-06
+2022-12-21,10:27:26 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:26 | INFO |   gather_with_grad: False
+2022-12-21,10:27:26 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:26 | INFO |   horovod: False
+2022-12-21,10:27:26 | INFO |   image_mean: None
+2022-12-21,10:27:26 | INFO |   image_std: None
+2022-12-21,10:27:26 | INFO |   imagenet_v2: None
+2022-12-21,10:27:26 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:26 | INFO |   local_loss: False
+2022-12-21,10:27:26 | INFO |   local_rank: 0
+2022-12-21,10:27:26 | INFO |   lock_image: False
+2022-12-21,10:27:26 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:26 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:26 | INFO |   log_level: 20
+2022-12-21,10:27:26 | INFO |   log_local: False
+2022-12-21,10:27:26 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/out.log
+2022-12-21,10:27:26 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:26 | INFO |   lr: 0.0005
+2022-12-21,10:27:26 | INFO |   model: ViT-B-16
+2022-12-21,10:27:26 | INFO |   name: epoch_35.pt
+2022-12-21,10:27:26 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:26 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:26 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:26 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt
+2022-12-21,10:27:26 | INFO |   pretrained_image: False
+2022-12-21,10:27:26 | INFO |   rank: 0
+2022-12-21,10:27:26 | INFO |   report_to: 
+2022-12-21,10:27:26 | INFO |   resume: None
+2022-12-21,10:27:26 | INFO |   save_frequency: 1
+2022-12-21,10:27:26 | INFO |   save_most_recent: False
+2022-12-21,10:27:26 | INFO |   seed: 0
+2022-12-21,10:27:26 | INFO |   skip_scheduler: False
+2022-12-21,10:27:26 | INFO |   tensorboard: False
+2022-12-21,10:27:26 | INFO |   tensorboard_path: 
+2022-12-21,10:27:26 | INFO |   torchscript: False
+2022-12-21,10:27:26 | INFO |   trace: False
+2022-12-21,10:27:26 | INFO |   train_data: None
+2022-12-21,10:27:26 | INFO |   train_num_samples: None
+2022-12-21,10:27:26 | INFO |   use_bn_sync: False
+2022-12-21,10:27:26 | INFO |   val_data: None
+2022-12-21,10:27:26 | INFO |   val_frequency: 1
+2022-12-21,10:27:26 | INFO |   val_num_samples: None
+2022-12-21,10:27:26 | INFO |   wandb: False
+2022-12-21,10:27:26 | INFO |   wandb_notes: 
+2022-12-21,10:27:26 | INFO |   warmup: 10000
+2022-12-21,10:27:26 | INFO |   wd: 0.2
+2022-12-21,10:27:26 | INFO |   workers: 1
+2022-12-21,10:27:26 | INFO |   world_size: 1
+2022-12-21,10:27:26 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:26 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:26 | INFO | Building zero-shot classifier
+2022-12-21,10:27:44 | INFO | Using classifier
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5998	imagenet-zeroshot-val-top5: 0.8633
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ef0c1be1776b15fa5c63525ccf825ed4a8649a16
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_35.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_35.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2f39fed4a15696eb4310bc3f277134917105a724
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.60484, "imagenet-zeroshot-val-top5": 0.86404}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..2e90247875bb70e4064926a629a3133a365b8579
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt).
+2022-12-21,10:27:27 | INFO | Model:
+2022-12-21,10:27:27 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:27 | INFO | Params:
+2022-12-21,10:27:27 | INFO |   batch_size: 256
+2022-12-21,10:27:27 | INFO |   beta1: 0.9
+2022-12-21,10:27:27 | INFO |   beta2: 0.98
+2022-12-21,10:27:27 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/checkpoints
+2022-12-21,10:27:27 | INFO |   copy_codebase: False
+2022-12-21,10:27:27 | INFO |   csv_caption_key: title
+2022-12-21,10:27:27 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:27 | INFO |   csv_separator: 	
+2022-12-21,10:27:27 | INFO |   dataset_resampled: False
+2022-12-21,10:27:27 | INFO |   dataset_type: auto
+2022-12-21,10:27:27 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:27 | INFO |   debug: False
+2022-12-21,10:27:27 | INFO |   device: cuda:0
+2022-12-21,10:27:27 | INFO |   dist_backend: nccl
+2022-12-21,10:27:27 | INFO |   dist_url: env://
+2022-12-21,10:27:27 | INFO |   distributed: False
+2022-12-21,10:27:27 | INFO |   epochs: 32
+2022-12-21,10:27:27 | INFO |   eps: 1e-06
+2022-12-21,10:27:27 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:27 | INFO |   gather_with_grad: False
+2022-12-21,10:27:27 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:27 | INFO |   horovod: False
+2022-12-21,10:27:27 | INFO |   image_mean: None
+2022-12-21,10:27:27 | INFO |   image_std: None
+2022-12-21,10:27:27 | INFO |   imagenet_v2: None
+2022-12-21,10:27:27 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:27 | INFO |   local_loss: False
+2022-12-21,10:27:27 | INFO |   local_rank: 0
+2022-12-21,10:27:27 | INFO |   lock_image: False
+2022-12-21,10:27:27 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:27 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:27 | INFO |   log_level: 20
+2022-12-21,10:27:27 | INFO |   log_local: False
+2022-12-21,10:27:27 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/out.log
+2022-12-21,10:27:27 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:27 | INFO |   lr: 0.0005
+2022-12-21,10:27:27 | INFO |   model: ViT-B-16
+2022-12-21,10:27:27 | INFO |   name: epoch_36.pt
+2022-12-21,10:27:27 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:27 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:27 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:27 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt
+2022-12-21,10:27:27 | INFO |   pretrained_image: False
+2022-12-21,10:27:27 | INFO |   rank: 0
+2022-12-21,10:27:27 | INFO |   report_to: 
+2022-12-21,10:27:27 | INFO |   resume: None
+2022-12-21,10:27:27 | INFO |   save_frequency: 1
+2022-12-21,10:27:27 | INFO |   save_most_recent: False
+2022-12-21,10:27:27 | INFO |   seed: 0
+2022-12-21,10:27:27 | INFO |   skip_scheduler: False
+2022-12-21,10:27:27 | INFO |   tensorboard: False
+2022-12-21,10:27:27 | INFO |   tensorboard_path: 
+2022-12-21,10:27:27 | INFO |   torchscript: False
+2022-12-21,10:27:27 | INFO |   trace: False
+2022-12-21,10:27:27 | INFO |   train_data: None
+2022-12-21,10:27:27 | INFO |   train_num_samples: None
+2022-12-21,10:27:27 | INFO |   use_bn_sync: False
+2022-12-21,10:27:27 | INFO |   val_data: None
+2022-12-21,10:27:27 | INFO |   val_frequency: 1
+2022-12-21,10:27:27 | INFO |   val_num_samples: None
+2022-12-21,10:27:27 | INFO |   wandb: False
+2022-12-21,10:27:27 | INFO |   wandb_notes: 
+2022-12-21,10:27:27 | INFO |   warmup: 10000
+2022-12-21,10:27:27 | INFO |   wd: 0.2
+2022-12-21,10:27:27 | INFO |   workers: 1
+2022-12-21,10:27:27 | INFO |   world_size: 1
+2022-12-21,10:27:27 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:27 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:27 | INFO | Building zero-shot classifier
+2022-12-21,10:27:45 | INFO | Using classifier
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6048	imagenet-zeroshot-val-top5: 0.8640
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..de06446ecfbebdb99eb1e49a2d8fe6dc3be179cc
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_36.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_36.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..dd057d01f680cbda6162794414916ce72310f6c2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.60312, "imagenet-zeroshot-val-top5": 0.8665}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..87013d193446d2d57ded1eb08243515ef6d0e874
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt).
+2022-12-21,10:27:26 | INFO | Model:
+2022-12-21,10:27:26 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:26 | INFO | Params:
+2022-12-21,10:27:26 | INFO |   batch_size: 256
+2022-12-21,10:27:26 | INFO |   beta1: 0.9
+2022-12-21,10:27:26 | INFO |   beta2: 0.98
+2022-12-21,10:27:26 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/checkpoints
+2022-12-21,10:27:26 | INFO |   copy_codebase: False
+2022-12-21,10:27:26 | INFO |   csv_caption_key: title
+2022-12-21,10:27:26 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:26 | INFO |   csv_separator: 	
+2022-12-21,10:27:26 | INFO |   dataset_resampled: False
+2022-12-21,10:27:26 | INFO |   dataset_type: auto
+2022-12-21,10:27:26 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:26 | INFO |   debug: False
+2022-12-21,10:27:26 | INFO |   device: cuda:0
+2022-12-21,10:27:26 | INFO |   dist_backend: nccl
+2022-12-21,10:27:26 | INFO |   dist_url: env://
+2022-12-21,10:27:26 | INFO |   distributed: False
+2022-12-21,10:27:26 | INFO |   epochs: 32
+2022-12-21,10:27:26 | INFO |   eps: 1e-06
+2022-12-21,10:27:26 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:26 | INFO |   gather_with_grad: False
+2022-12-21,10:27:26 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:26 | INFO |   horovod: False
+2022-12-21,10:27:26 | INFO |   image_mean: None
+2022-12-21,10:27:26 | INFO |   image_std: None
+2022-12-21,10:27:26 | INFO |   imagenet_v2: None
+2022-12-21,10:27:26 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:26 | INFO |   local_loss: False
+2022-12-21,10:27:26 | INFO |   local_rank: 0
+2022-12-21,10:27:26 | INFO |   lock_image: False
+2022-12-21,10:27:26 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:26 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:26 | INFO |   log_level: 20
+2022-12-21,10:27:26 | INFO |   log_local: False
+2022-12-21,10:27:26 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/out.log
+2022-12-21,10:27:26 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:26 | INFO |   lr: 0.0005
+2022-12-21,10:27:26 | INFO |   model: ViT-B-16
+2022-12-21,10:27:26 | INFO |   name: epoch_37.pt
+2022-12-21,10:27:26 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:26 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:26 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:26 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt
+2022-12-21,10:27:26 | INFO |   pretrained_image: False
+2022-12-21,10:27:26 | INFO |   rank: 0
+2022-12-21,10:27:26 | INFO |   report_to: 
+2022-12-21,10:27:26 | INFO |   resume: None
+2022-12-21,10:27:26 | INFO |   save_frequency: 1
+2022-12-21,10:27:26 | INFO |   save_most_recent: False
+2022-12-21,10:27:26 | INFO |   seed: 0
+2022-12-21,10:27:26 | INFO |   skip_scheduler: False
+2022-12-21,10:27:26 | INFO |   tensorboard: False
+2022-12-21,10:27:26 | INFO |   tensorboard_path: 
+2022-12-21,10:27:26 | INFO |   torchscript: False
+2022-12-21,10:27:26 | INFO |   trace: False
+2022-12-21,10:27:26 | INFO |   train_data: None
+2022-12-21,10:27:26 | INFO |   train_num_samples: None
+2022-12-21,10:27:26 | INFO |   use_bn_sync: False
+2022-12-21,10:27:26 | INFO |   val_data: None
+2022-12-21,10:27:26 | INFO |   val_frequency: 1
+2022-12-21,10:27:26 | INFO |   val_num_samples: None
+2022-12-21,10:27:26 | INFO |   wandb: False
+2022-12-21,10:27:26 | INFO |   wandb_notes: 
+2022-12-21,10:27:26 | INFO |   warmup: 10000
+2022-12-21,10:27:26 | INFO |   wd: 0.2
+2022-12-21,10:27:26 | INFO |   workers: 1
+2022-12-21,10:27:26 | INFO |   world_size: 1
+2022-12-21,10:27:26 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:26 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:26 | INFO | Building zero-shot classifier
+2022-12-21,10:27:44 | INFO | Using classifier
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6031	imagenet-zeroshot-val-top5: 0.8665
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..75717afb7740b2908ae7064764d2ac2cbdd9d817
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_37.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_37.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..35c951ef5932a0f063ecf72fa6eac35cfa5283c5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.61014, "imagenet-zeroshot-val-top5": 0.86966}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..07c40d0cc70e2bdbf329f7db44a101468cb34105
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt).
+2022-12-21,10:27:27 | INFO | Model:
+2022-12-21,10:27:27 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:27:27 | INFO | Params:
+2022-12-21,10:27:27 | INFO |   batch_size: 256
+2022-12-21,10:27:27 | INFO |   beta1: 0.9
+2022-12-21,10:27:27 | INFO |   beta2: 0.98
+2022-12-21,10:27:27 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/checkpoints
+2022-12-21,10:27:27 | INFO |   copy_codebase: False
+2022-12-21,10:27:27 | INFO |   csv_caption_key: title
+2022-12-21,10:27:27 | INFO |   csv_img_key: filepath
+2022-12-21,10:27:27 | INFO |   csv_separator: 	
+2022-12-21,10:27:27 | INFO |   dataset_resampled: False
+2022-12-21,10:27:27 | INFO |   dataset_type: auto
+2022-12-21,10:27:27 | INFO |   ddp_static_graph: False
+2022-12-21,10:27:27 | INFO |   debug: False
+2022-12-21,10:27:27 | INFO |   device: cuda:0
+2022-12-21,10:27:27 | INFO |   dist_backend: nccl
+2022-12-21,10:27:27 | INFO |   dist_url: env://
+2022-12-21,10:27:27 | INFO |   distributed: False
+2022-12-21,10:27:27 | INFO |   epochs: 32
+2022-12-21,10:27:27 | INFO |   eps: 1e-06
+2022-12-21,10:27:27 | INFO |   force_quick_gelu: False
+2022-12-21,10:27:27 | INFO |   gather_with_grad: False
+2022-12-21,10:27:27 | INFO |   grad_checkpointing: False
+2022-12-21,10:27:27 | INFO |   horovod: False
+2022-12-21,10:27:27 | INFO |   image_mean: None
+2022-12-21,10:27:27 | INFO |   image_std: None
+2022-12-21,10:27:27 | INFO |   imagenet_v2: None
+2022-12-21,10:27:27 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:27:27 | INFO |   local_loss: False
+2022-12-21,10:27:27 | INFO |   local_rank: 0
+2022-12-21,10:27:27 | INFO |   lock_image: False
+2022-12-21,10:27:27 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:27:27 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:27:27 | INFO |   log_level: 20
+2022-12-21,10:27:27 | INFO |   log_local: False
+2022-12-21,10:27:27 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/out.log
+2022-12-21,10:27:27 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:27:27 | INFO |   lr: 0.0005
+2022-12-21,10:27:27 | INFO |   model: ViT-B-16
+2022-12-21,10:27:27 | INFO |   name: epoch_38.pt
+2022-12-21,10:27:27 | INFO |   no_set_device_rank: False
+2022-12-21,10:27:27 | INFO |   norm_gradient_clip: None
+2022-12-21,10:27:27 | INFO |   precision: amp_bfloat16
+2022-12-21,10:27:27 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt
+2022-12-21,10:27:27 | INFO |   pretrained_image: False
+2022-12-21,10:27:27 | INFO |   rank: 0
+2022-12-21,10:27:27 | INFO |   report_to: 
+2022-12-21,10:27:27 | INFO |   resume: None
+2022-12-21,10:27:27 | INFO |   save_frequency: 1
+2022-12-21,10:27:27 | INFO |   save_most_recent: False
+2022-12-21,10:27:27 | INFO |   seed: 0
+2022-12-21,10:27:27 | INFO |   skip_scheduler: False
+2022-12-21,10:27:27 | INFO |   tensorboard: False
+2022-12-21,10:27:27 | INFO |   tensorboard_path: 
+2022-12-21,10:27:27 | INFO |   torchscript: False
+2022-12-21,10:27:27 | INFO |   trace: False
+2022-12-21,10:27:27 | INFO |   train_data: None
+2022-12-21,10:27:27 | INFO |   train_num_samples: None
+2022-12-21,10:27:27 | INFO |   use_bn_sync: False
+2022-12-21,10:27:27 | INFO |   val_data: None
+2022-12-21,10:27:27 | INFO |   val_frequency: 1
+2022-12-21,10:27:27 | INFO |   val_num_samples: None
+2022-12-21,10:27:27 | INFO |   wandb: False
+2022-12-21,10:27:27 | INFO |   wandb_notes: 
+2022-12-21,10:27:27 | INFO |   warmup: 10000
+2022-12-21,10:27:27 | INFO |   wd: 0.2
+2022-12-21,10:27:27 | INFO |   workers: 1
+2022-12-21,10:27:27 | INFO |   world_size: 1
+2022-12-21,10:27:27 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:27:27 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:27:27 | INFO | Building zero-shot classifier
+2022-12-21,10:27:45 | INFO | Using classifier
+2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6101	imagenet-zeroshot-val-top5: 0.8697
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..6eaa25122070788900a1cca1b7ed45c78a5a88f6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_38.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_38.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..cc4129994d1226ac06d5fa892bbbe9d26aeea872
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.61158, "imagenet-zeroshot-val-top5": 0.86902}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..c6540095882c2085fa86cc40c17ca08e5c2e750d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_39.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:42 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:42 | INFO | Building zero-shot classifier
+2022-12-21,10:36:00 | INFO | Using classifier
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6116	imagenet-zeroshot-val-top5: 0.8690
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..465cec5e9546f7e2975b1ceb16daa0de6f6e4e23
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_39.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_39.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d9b0fcd170d1d72f995965700b00353481e4a227
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.42494, "imagenet-zeroshot-val-top5": 0.70486}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..1cca4bec1cb2880ad1ccdde37caa75dff5447126
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:34 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_4.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:43 | INFO | Building zero-shot classifier
+2022-12-21,10:36:01 | INFO | Using classifier
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4249	imagenet-zeroshot-val-top5: 0.7049
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b4b991420578e3c3a6db56dee513b879bd6f4596
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_4.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_4.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..fdadf6efc811f9aec566155ae5f8ba5d155d5c87
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.61046, "imagenet-zeroshot-val-top5": 0.86926}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..bc9e7fe676bd53283458fffe81b3103b5d53581e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_40.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:43 | INFO | Building zero-shot classifier
+2022-12-21,10:36:01 | INFO | Using classifier
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6105	imagenet-zeroshot-val-top5: 0.8693
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4f3ccc40ba29dcd6b5cc0213ea2e8439ee87a266
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_40.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_40.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..7be21d4d5752fbdb89d88e7b4a11d692e93d9056
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.6177, "imagenet-zeroshot-val-top5": 0.87412}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..53805a7d1fec9b2a5f9be804772eb0293c97a1f2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt).
+2022-12-21,10:35:42 | INFO | Model:
+2022-12-21,10:35:42 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:35:42 | INFO | Params:
+2022-12-21,10:35:42 | INFO |   batch_size: 256
+2022-12-21,10:35:42 | INFO |   beta1: 0.9
+2022-12-21,10:35:42 | INFO |   beta2: 0.98
+2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/checkpoints
+2022-12-21,10:35:42 | INFO |   copy_codebase: False
+2022-12-21,10:35:42 | INFO |   csv_caption_key: title
+2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
+2022-12-21,10:35:42 | INFO |   csv_separator: 	
+2022-12-21,10:35:42 | INFO |   dataset_resampled: False
+2022-12-21,10:35:42 | INFO |   dataset_type: auto
+2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
+2022-12-21,10:35:42 | INFO |   debug: False
+2022-12-21,10:35:42 | INFO |   device: cuda:0
+2022-12-21,10:35:42 | INFO |   dist_backend: nccl
+2022-12-21,10:35:42 | INFO |   dist_url: env://
+2022-12-21,10:35:42 | INFO |   distributed: False
+2022-12-21,10:35:42 | INFO |   epochs: 32
+2022-12-21,10:35:42 | INFO |   eps: 1e-06
+2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
+2022-12-21,10:35:42 | INFO |   gather_with_grad: False
+2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
+2022-12-21,10:35:42 | INFO |   horovod: False
+2022-12-21,10:35:42 | INFO |   image_mean: None
+2022-12-21,10:35:42 | INFO |   image_std: None
+2022-12-21,10:35:42 | INFO |   imagenet_v2: None
+2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:35:42 | INFO |   local_loss: False
+2022-12-21,10:35:42 | INFO |   local_rank: 0
+2022-12-21,10:35:42 | INFO |   lock_image: False
+2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:35:42 | INFO |   log_level: 20
+2022-12-21,10:35:42 | INFO |   log_local: False
+2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/out.log
+2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:35:42 | INFO |   lr: 0.0005
+2022-12-21,10:35:42 | INFO |   model: ViT-B-16
+2022-12-21,10:35:42 | INFO |   name: epoch_41.pt
+2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
+2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
+2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
+2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt
+2022-12-21,10:35:42 | INFO |   pretrained_image: False
+2022-12-21,10:35:42 | INFO |   rank: 0
+2022-12-21,10:35:42 | INFO |   report_to: 
+2022-12-21,10:35:42 | INFO |   resume: None
+2022-12-21,10:35:42 | INFO |   save_frequency: 1
+2022-12-21,10:35:42 | INFO |   save_most_recent: False
+2022-12-21,10:35:42 | INFO |   seed: 0
+2022-12-21,10:35:42 | INFO |   skip_scheduler: False
+2022-12-21,10:35:42 | INFO |   tensorboard: False
+2022-12-21,10:35:42 | INFO |   tensorboard_path: 
+2022-12-21,10:35:42 | INFO |   torchscript: False
+2022-12-21,10:35:42 | INFO |   trace: False
+2022-12-21,10:35:42 | INFO |   train_data: None
+2022-12-21,10:35:42 | INFO |   train_num_samples: None
+2022-12-21,10:35:42 | INFO |   use_bn_sync: False
+2022-12-21,10:35:42 | INFO |   val_data: None
+2022-12-21,10:35:42 | INFO |   val_frequency: 1
+2022-12-21,10:35:42 | INFO |   val_num_samples: None
+2022-12-21,10:35:42 | INFO |   wandb: False
+2022-12-21,10:35:42 | INFO |   wandb_notes: 
+2022-12-21,10:35:42 | INFO |   warmup: 10000
+2022-12-21,10:35:42 | INFO |   wd: 0.2
+2022-12-21,10:35:42 | INFO |   workers: 1
+2022-12-21,10:35:42 | INFO |   world_size: 1
+2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:35:43 | INFO | Building zero-shot classifier
+2022-12-21,10:36:01 | INFO | Using classifier
+2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6177	imagenet-zeroshot-val-top5: 0.8741
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9cb68f5f035ce748c89d16d10ff93e34ea4c83b1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_41.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_41.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..49d7298ee9c2e12a51d801e9936515b22dc7f8ef
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.61692, "imagenet-zeroshot-val-top5": 0.8739}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..fce2c177f0d30c69f3fc008cb931ce5a2af2b0f7
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt).
+2022-12-21,10:44:38 | INFO | Model:
+2022-12-21,10:44:38 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:38 | INFO | Params:
+2022-12-21,10:44:38 | INFO |   batch_size: 256
+2022-12-21,10:44:38 | INFO |   beta1: 0.9
+2022-12-21,10:44:38 | INFO |   beta2: 0.98
+2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/checkpoints
+2022-12-21,10:44:38 | INFO |   copy_codebase: False
+2022-12-21,10:44:38 | INFO |   csv_caption_key: title
+2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:38 | INFO |   csv_separator: 	
+2022-12-21,10:44:38 | INFO |   dataset_resampled: False
+2022-12-21,10:44:38 | INFO |   dataset_type: auto
+2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:38 | INFO |   debug: False
+2022-12-21,10:44:38 | INFO |   device: cuda:0
+2022-12-21,10:44:38 | INFO |   dist_backend: nccl
+2022-12-21,10:44:38 | INFO |   dist_url: env://
+2022-12-21,10:44:38 | INFO |   distributed: False
+2022-12-21,10:44:38 | INFO |   epochs: 32
+2022-12-21,10:44:38 | INFO |   eps: 1e-06
+2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:38 | INFO |   gather_with_grad: False
+2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:38 | INFO |   horovod: False
+2022-12-21,10:44:38 | INFO |   image_mean: None
+2022-12-21,10:44:38 | INFO |   image_std: None
+2022-12-21,10:44:38 | INFO |   imagenet_v2: None
+2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:38 | INFO |   local_loss: False
+2022-12-21,10:44:38 | INFO |   local_rank: 0
+2022-12-21,10:44:38 | INFO |   lock_image: False
+2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:38 | INFO |   log_level: 20
+2022-12-21,10:44:38 | INFO |   log_local: False
+2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/out.log
+2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:38 | INFO |   lr: 0.0005
+2022-12-21,10:44:38 | INFO |   model: ViT-B-16
+2022-12-21,10:44:38 | INFO |   name: epoch_42.pt
+2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt
+2022-12-21,10:44:38 | INFO |   pretrained_image: False
+2022-12-21,10:44:38 | INFO |   rank: 0
+2022-12-21,10:44:38 | INFO |   report_to: 
+2022-12-21,10:44:38 | INFO |   resume: None
+2022-12-21,10:44:38 | INFO |   save_frequency: 1
+2022-12-21,10:44:38 | INFO |   save_most_recent: False
+2022-12-21,10:44:38 | INFO |   seed: 0
+2022-12-21,10:44:38 | INFO |   skip_scheduler: False
+2022-12-21,10:44:38 | INFO |   tensorboard: False
+2022-12-21,10:44:38 | INFO |   tensorboard_path: 
+2022-12-21,10:44:38 | INFO |   torchscript: False
+2022-12-21,10:44:38 | INFO |   trace: False
+2022-12-21,10:44:38 | INFO |   train_data: None
+2022-12-21,10:44:38 | INFO |   train_num_samples: None
+2022-12-21,10:44:38 | INFO |   use_bn_sync: False
+2022-12-21,10:44:38 | INFO |   val_data: None
+2022-12-21,10:44:38 | INFO |   val_frequency: 1
+2022-12-21,10:44:38 | INFO |   val_num_samples: None
+2022-12-21,10:44:38 | INFO |   wandb: False
+2022-12-21,10:44:38 | INFO |   wandb_notes: 
+2022-12-21,10:44:38 | INFO |   warmup: 10000
+2022-12-21,10:44:38 | INFO |   wd: 0.2
+2022-12-21,10:44:38 | INFO |   workers: 1
+2022-12-21,10:44:38 | INFO |   world_size: 1
+2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+2022-12-21,10:44:56 | INFO | Using classifier
+2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6169	imagenet-zeroshot-val-top5: 0.8739
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e49779f92132b8eada428996db3961fbfb75767d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_42.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_42.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..d1017b01bbd94ce4dcf09d8108025bf80d0c252f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.6219, "imagenet-zeroshot-val-top5": 0.87622}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..ad1f92657351ead498214ec5156109bef7f7379b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt).
+2022-12-21,10:44:38 | INFO | Model:
+2022-12-21,10:44:38 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:38 | INFO | Params:
+2022-12-21,10:44:38 | INFO |   batch_size: 256
+2022-12-21,10:44:38 | INFO |   beta1: 0.9
+2022-12-21,10:44:38 | INFO |   beta2: 0.98
+2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/checkpoints
+2022-12-21,10:44:38 | INFO |   copy_codebase: False
+2022-12-21,10:44:38 | INFO |   csv_caption_key: title
+2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:38 | INFO |   csv_separator: 	
+2022-12-21,10:44:38 | INFO |   dataset_resampled: False
+2022-12-21,10:44:38 | INFO |   dataset_type: auto
+2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:38 | INFO |   debug: False
+2022-12-21,10:44:38 | INFO |   device: cuda:0
+2022-12-21,10:44:38 | INFO |   dist_backend: nccl
+2022-12-21,10:44:38 | INFO |   dist_url: env://
+2022-12-21,10:44:38 | INFO |   distributed: False
+2022-12-21,10:44:38 | INFO |   epochs: 32
+2022-12-21,10:44:38 | INFO |   eps: 1e-06
+2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:38 | INFO |   gather_with_grad: False
+2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:38 | INFO |   horovod: False
+2022-12-21,10:44:38 | INFO |   image_mean: None
+2022-12-21,10:44:38 | INFO |   image_std: None
+2022-12-21,10:44:38 | INFO |   imagenet_v2: None
+2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:38 | INFO |   local_loss: False
+2022-12-21,10:44:38 | INFO |   local_rank: 0
+2022-12-21,10:44:38 | INFO |   lock_image: False
+2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:38 | INFO |   log_level: 20
+2022-12-21,10:44:38 | INFO |   log_local: False
+2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/out.log
+2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:38 | INFO |   lr: 0.0005
+2022-12-21,10:44:38 | INFO |   model: ViT-B-16
+2022-12-21,10:44:38 | INFO |   name: epoch_43.pt
+2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt
+2022-12-21,10:44:38 | INFO |   pretrained_image: False
+2022-12-21,10:44:38 | INFO |   rank: 0
+2022-12-21,10:44:38 | INFO |   report_to: 
+2022-12-21,10:44:38 | INFO |   resume: None
+2022-12-21,10:44:38 | INFO |   save_frequency: 1
+2022-12-21,10:44:38 | INFO |   save_most_recent: False
+2022-12-21,10:44:38 | INFO |   seed: 0
+2022-12-21,10:44:38 | INFO |   skip_scheduler: False
+2022-12-21,10:44:38 | INFO |   tensorboard: False
+2022-12-21,10:44:38 | INFO |   tensorboard_path: 
+2022-12-21,10:44:38 | INFO |   torchscript: False
+2022-12-21,10:44:38 | INFO |   trace: False
+2022-12-21,10:44:38 | INFO |   train_data: None
+2022-12-21,10:44:38 | INFO |   train_num_samples: None
+2022-12-21,10:44:38 | INFO |   use_bn_sync: False
+2022-12-21,10:44:38 | INFO |   val_data: None
+2022-12-21,10:44:38 | INFO |   val_frequency: 1
+2022-12-21,10:44:38 | INFO |   val_num_samples: None
+2022-12-21,10:44:38 | INFO |   wandb: False
+2022-12-21,10:44:38 | INFO |   wandb_notes: 
+2022-12-21,10:44:38 | INFO |   warmup: 10000
+2022-12-21,10:44:38 | INFO |   wd: 0.2
+2022-12-21,10:44:38 | INFO |   workers: 1
+2022-12-21,10:44:38 | INFO |   world_size: 1
+2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+2022-12-21,10:44:56 | INFO | Using classifier
+2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6219	imagenet-zeroshot-val-top5: 0.8762
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e20113559daab8d5a751246c0e797b7de4fe1cdf
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_43.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_43.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ba0a412efbee708c76e2e18dc5b67deae2497afe
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.62592, "imagenet-zeroshot-val-top5": 0.87888}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..acf64059db3262c76b30e4f0d6f4281d9a676a5c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt).
+2022-12-21,10:44:37 | INFO | Model:
+2022-12-21,10:44:37 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:37 | INFO | Params:
+2022-12-21,10:44:37 | INFO |   batch_size: 256
+2022-12-21,10:44:37 | INFO |   beta1: 0.9
+2022-12-21,10:44:37 | INFO |   beta2: 0.98
+2022-12-21,10:44:37 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/checkpoints
+2022-12-21,10:44:37 | INFO |   copy_codebase: False
+2022-12-21,10:44:37 | INFO |   csv_caption_key: title
+2022-12-21,10:44:37 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:37 | INFO |   csv_separator: 	
+2022-12-21,10:44:37 | INFO |   dataset_resampled: False
+2022-12-21,10:44:37 | INFO |   dataset_type: auto
+2022-12-21,10:44:37 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:37 | INFO |   debug: False
+2022-12-21,10:44:37 | INFO |   device: cuda:0
+2022-12-21,10:44:37 | INFO |   dist_backend: nccl
+2022-12-21,10:44:37 | INFO |   dist_url: env://
+2022-12-21,10:44:37 | INFO |   distributed: False
+2022-12-21,10:44:37 | INFO |   epochs: 32
+2022-12-21,10:44:37 | INFO |   eps: 1e-06
+2022-12-21,10:44:37 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:37 | INFO |   gather_with_grad: False
+2022-12-21,10:44:37 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:37 | INFO |   horovod: False
+2022-12-21,10:44:37 | INFO |   image_mean: None
+2022-12-21,10:44:37 | INFO |   image_std: None
+2022-12-21,10:44:37 | INFO |   imagenet_v2: None
+2022-12-21,10:44:37 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:37 | INFO |   local_loss: False
+2022-12-21,10:44:37 | INFO |   local_rank: 0
+2022-12-21,10:44:37 | INFO |   lock_image: False
+2022-12-21,10:44:37 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:37 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:37 | INFO |   log_level: 20
+2022-12-21,10:44:37 | INFO |   log_local: False
+2022-12-21,10:44:37 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/out.log
+2022-12-21,10:44:37 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:37 | INFO |   lr: 0.0005
+2022-12-21,10:44:37 | INFO |   model: ViT-B-16
+2022-12-21,10:44:37 | INFO |   name: epoch_44.pt
+2022-12-21,10:44:37 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:37 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:37 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:37 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt
+2022-12-21,10:44:37 | INFO |   pretrained_image: False
+2022-12-21,10:44:37 | INFO |   rank: 0
+2022-12-21,10:44:37 | INFO |   report_to: 
+2022-12-21,10:44:37 | INFO |   resume: None
+2022-12-21,10:44:37 | INFO |   save_frequency: 1
+2022-12-21,10:44:37 | INFO |   save_most_recent: False
+2022-12-21,10:44:37 | INFO |   seed: 0
+2022-12-21,10:44:37 | INFO |   skip_scheduler: False
+2022-12-21,10:44:37 | INFO |   tensorboard: False
+2022-12-21,10:44:37 | INFO |   tensorboard_path: 
+2022-12-21,10:44:37 | INFO |   torchscript: False
+2022-12-21,10:44:37 | INFO |   trace: False
+2022-12-21,10:44:37 | INFO |   train_data: None
+2022-12-21,10:44:37 | INFO |   train_num_samples: None
+2022-12-21,10:44:37 | INFO |   use_bn_sync: False
+2022-12-21,10:44:37 | INFO |   val_data: None
+2022-12-21,10:44:37 | INFO |   val_frequency: 1
+2022-12-21,10:44:37 | INFO |   val_num_samples: None
+2022-12-21,10:44:37 | INFO |   wandb: False
+2022-12-21,10:44:37 | INFO |   wandb_notes: 
+2022-12-21,10:44:37 | INFO |   warmup: 10000
+2022-12-21,10:44:37 | INFO |   wd: 0.2
+2022-12-21,10:44:37 | INFO |   workers: 1
+2022-12-21,10:44:37 | INFO |   world_size: 1
+2022-12-21,10:44:37 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+2022-12-21,10:44:55 | INFO | Using classifier
+2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6259	imagenet-zeroshot-val-top5: 0.8789
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..92e38387a3026d3ccc514dae0d9192a94f1c5b64
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_44.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_44.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2b935225cedd6ad4b989c61097e924d9c895fbeb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.62778, "imagenet-zeroshot-val-top5": 0.88026}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..1656f8789833373cdfbf62fd6462c39187986231
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt).
+2022-12-21,10:44:38 | INFO | Model:
+2022-12-21,10:44:38 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:44:38 | INFO | Params:
+2022-12-21,10:44:38 | INFO |   batch_size: 256
+2022-12-21,10:44:38 | INFO |   beta1: 0.9
+2022-12-21,10:44:38 | INFO |   beta2: 0.98
+2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/checkpoints
+2022-12-21,10:44:38 | INFO |   copy_codebase: False
+2022-12-21,10:44:38 | INFO |   csv_caption_key: title
+2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
+2022-12-21,10:44:38 | INFO |   csv_separator: 	
+2022-12-21,10:44:38 | INFO |   dataset_resampled: False
+2022-12-21,10:44:38 | INFO |   dataset_type: auto
+2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
+2022-12-21,10:44:38 | INFO |   debug: False
+2022-12-21,10:44:38 | INFO |   device: cuda:0
+2022-12-21,10:44:38 | INFO |   dist_backend: nccl
+2022-12-21,10:44:38 | INFO |   dist_url: env://
+2022-12-21,10:44:38 | INFO |   distributed: False
+2022-12-21,10:44:38 | INFO |   epochs: 32
+2022-12-21,10:44:38 | INFO |   eps: 1e-06
+2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
+2022-12-21,10:44:38 | INFO |   gather_with_grad: False
+2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
+2022-12-21,10:44:38 | INFO |   horovod: False
+2022-12-21,10:44:38 | INFO |   image_mean: None
+2022-12-21,10:44:38 | INFO |   image_std: None
+2022-12-21,10:44:38 | INFO |   imagenet_v2: None
+2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:44:38 | INFO |   local_loss: False
+2022-12-21,10:44:38 | INFO |   local_rank: 0
+2022-12-21,10:44:38 | INFO |   lock_image: False
+2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:44:38 | INFO |   log_level: 20
+2022-12-21,10:44:38 | INFO |   log_local: False
+2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/out.log
+2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:44:38 | INFO |   lr: 0.0005
+2022-12-21,10:44:38 | INFO |   model: ViT-B-16
+2022-12-21,10:44:38 | INFO |   name: epoch_45.pt
+2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
+2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
+2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
+2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt
+2022-12-21,10:44:38 | INFO |   pretrained_image: False
+2022-12-21,10:44:38 | INFO |   rank: 0
+2022-12-21,10:44:38 | INFO |   report_to: 
+2022-12-21,10:44:38 | INFO |   resume: None
+2022-12-21,10:44:38 | INFO |   save_frequency: 1
+2022-12-21,10:44:38 | INFO |   save_most_recent: False
+2022-12-21,10:44:38 | INFO |   seed: 0
+2022-12-21,10:44:38 | INFO |   skip_scheduler: False
+2022-12-21,10:44:38 | INFO |   tensorboard: False
+2022-12-21,10:44:38 | INFO |   tensorboard_path: 
+2022-12-21,10:44:38 | INFO |   torchscript: False
+2022-12-21,10:44:38 | INFO |   trace: False
+2022-12-21,10:44:38 | INFO |   train_data: None
+2022-12-21,10:44:38 | INFO |   train_num_samples: None
+2022-12-21,10:44:38 | INFO |   use_bn_sync: False
+2022-12-21,10:44:38 | INFO |   val_data: None
+2022-12-21,10:44:38 | INFO |   val_frequency: 1
+2022-12-21,10:44:38 | INFO |   val_num_samples: None
+2022-12-21,10:44:38 | INFO |   wandb: False
+2022-12-21,10:44:38 | INFO |   wandb_notes: 
+2022-12-21,10:44:38 | INFO |   warmup: 10000
+2022-12-21,10:44:38 | INFO |   wd: 0.2
+2022-12-21,10:44:38 | INFO |   workers: 1
+2022-12-21,10:44:38 | INFO |   world_size: 1
+2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:44:38 | INFO | Building zero-shot classifier
+2022-12-21,10:44:56 | INFO | Using classifier
+2022-12-21,10:53:05 | INFO | Finished zero-shot imagenet.
+2022-12-21,10:53:05 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6278	imagenet-zeroshot-val-top5: 0.8803
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c453cadfd8a547525b54307abed05ae8659c3dfd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_45.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_45.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..a7f2ea624e17255762c7752d5f7921e46f4f41bd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.62696, "imagenet-zeroshot-val-top5": 0.8812}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..ec09ed8cc57f561b7a34fa09e28b44960c74e1c9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt).
+2022-12-21,10:53:30 | INFO | Model:
+2022-12-21,10:53:30 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:30 | INFO | Params:
+2022-12-21,10:53:30 | INFO |   batch_size: 256
+2022-12-21,10:53:30 | INFO |   beta1: 0.9
+2022-12-21,10:53:30 | INFO |   beta2: 0.98
+2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/checkpoints
+2022-12-21,10:53:30 | INFO |   copy_codebase: False
+2022-12-21,10:53:30 | INFO |   csv_caption_key: title
+2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:30 | INFO |   csv_separator: 	
+2022-12-21,10:53:30 | INFO |   dataset_resampled: False
+2022-12-21,10:53:30 | INFO |   dataset_type: auto
+2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:30 | INFO |   debug: False
+2022-12-21,10:53:30 | INFO |   device: cuda:0
+2022-12-21,10:53:30 | INFO |   dist_backend: nccl
+2022-12-21,10:53:30 | INFO |   dist_url: env://
+2022-12-21,10:53:30 | INFO |   distributed: False
+2022-12-21,10:53:30 | INFO |   epochs: 32
+2022-12-21,10:53:30 | INFO |   eps: 1e-06
+2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:30 | INFO |   gather_with_grad: False
+2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:30 | INFO |   horovod: False
+2022-12-21,10:53:30 | INFO |   image_mean: None
+2022-12-21,10:53:30 | INFO |   image_std: None
+2022-12-21,10:53:30 | INFO |   imagenet_v2: None
+2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:30 | INFO |   local_loss: False
+2022-12-21,10:53:30 | INFO |   local_rank: 0
+2022-12-21,10:53:30 | INFO |   lock_image: False
+2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:30 | INFO |   log_level: 20
+2022-12-21,10:53:30 | INFO |   log_local: False
+2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/out.log
+2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:30 | INFO |   lr: 0.0005
+2022-12-21,10:53:30 | INFO |   model: ViT-B-16
+2022-12-21,10:53:30 | INFO |   name: epoch_46.pt
+2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt
+2022-12-21,10:53:30 | INFO |   pretrained_image: False
+2022-12-21,10:53:30 | INFO |   rank: 0
+2022-12-21,10:53:30 | INFO |   report_to: 
+2022-12-21,10:53:30 | INFO |   resume: None
+2022-12-21,10:53:30 | INFO |   save_frequency: 1
+2022-12-21,10:53:30 | INFO |   save_most_recent: False
+2022-12-21,10:53:30 | INFO |   seed: 0
+2022-12-21,10:53:30 | INFO |   skip_scheduler: False
+2022-12-21,10:53:30 | INFO |   tensorboard: False
+2022-12-21,10:53:30 | INFO |   tensorboard_path: 
+2022-12-21,10:53:30 | INFO |   torchscript: False
+2022-12-21,10:53:30 | INFO |   trace: False
+2022-12-21,10:53:30 | INFO |   train_data: None
+2022-12-21,10:53:30 | INFO |   train_num_samples: None
+2022-12-21,10:53:30 | INFO |   use_bn_sync: False
+2022-12-21,10:53:30 | INFO |   val_data: None
+2022-12-21,10:53:30 | INFO |   val_frequency: 1
+2022-12-21,10:53:30 | INFO |   val_num_samples: None
+2022-12-21,10:53:30 | INFO |   wandb: False
+2022-12-21,10:53:30 | INFO |   wandb_notes: 
+2022-12-21,10:53:30 | INFO |   warmup: 10000
+2022-12-21,10:53:30 | INFO |   wd: 0.2
+2022-12-21,10:53:30 | INFO |   workers: 1
+2022-12-21,10:53:30 | INFO |   world_size: 1
+2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+2022-12-21,10:53:48 | INFO | Using classifier
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6270	imagenet-zeroshot-val-top5: 0.8812
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..252d979d2cee1e43d4b4451700397d19881bc634
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_46.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_46.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c00e935b4869a0a941866b8c66d686b49a6a933a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.63194, "imagenet-zeroshot-val-top5": 0.88234}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..caaf08d6403d25c3186fedd423504aedd82708fc
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt).
+2022-12-21,10:53:30 | INFO | Model:
+2022-12-21,10:53:30 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:30 | INFO | Params:
+2022-12-21,10:53:30 | INFO |   batch_size: 256
+2022-12-21,10:53:30 | INFO |   beta1: 0.9
+2022-12-21,10:53:30 | INFO |   beta2: 0.98
+2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/checkpoints
+2022-12-21,10:53:30 | INFO |   copy_codebase: False
+2022-12-21,10:53:30 | INFO |   csv_caption_key: title
+2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:30 | INFO |   csv_separator: 	
+2022-12-21,10:53:30 | INFO |   dataset_resampled: False
+2022-12-21,10:53:30 | INFO |   dataset_type: auto
+2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:30 | INFO |   debug: False
+2022-12-21,10:53:30 | INFO |   device: cuda:0
+2022-12-21,10:53:30 | INFO |   dist_backend: nccl
+2022-12-21,10:53:30 | INFO |   dist_url: env://
+2022-12-21,10:53:30 | INFO |   distributed: False
+2022-12-21,10:53:30 | INFO |   epochs: 32
+2022-12-21,10:53:30 | INFO |   eps: 1e-06
+2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:30 | INFO |   gather_with_grad: False
+2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:30 | INFO |   horovod: False
+2022-12-21,10:53:30 | INFO |   image_mean: None
+2022-12-21,10:53:30 | INFO |   image_std: None
+2022-12-21,10:53:30 | INFO |   imagenet_v2: None
+2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:30 | INFO |   local_loss: False
+2022-12-21,10:53:30 | INFO |   local_rank: 0
+2022-12-21,10:53:30 | INFO |   lock_image: False
+2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:30 | INFO |   log_level: 20
+2022-12-21,10:53:30 | INFO |   log_local: False
+2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/out.log
+2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:30 | INFO |   lr: 0.0005
+2022-12-21,10:53:30 | INFO |   model: ViT-B-16
+2022-12-21,10:53:30 | INFO |   name: epoch_47.pt
+2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt
+2022-12-21,10:53:30 | INFO |   pretrained_image: False
+2022-12-21,10:53:30 | INFO |   rank: 0
+2022-12-21,10:53:30 | INFO |   report_to: 
+2022-12-21,10:53:30 | INFO |   resume: None
+2022-12-21,10:53:30 | INFO |   save_frequency: 1
+2022-12-21,10:53:30 | INFO |   save_most_recent: False
+2022-12-21,10:53:30 | INFO |   seed: 0
+2022-12-21,10:53:30 | INFO |   skip_scheduler: False
+2022-12-21,10:53:30 | INFO |   tensorboard: False
+2022-12-21,10:53:30 | INFO |   tensorboard_path: 
+2022-12-21,10:53:30 | INFO |   torchscript: False
+2022-12-21,10:53:30 | INFO |   trace: False
+2022-12-21,10:53:30 | INFO |   train_data: None
+2022-12-21,10:53:30 | INFO |   train_num_samples: None
+2022-12-21,10:53:30 | INFO |   use_bn_sync: False
+2022-12-21,10:53:30 | INFO |   val_data: None
+2022-12-21,10:53:30 | INFO |   val_frequency: 1
+2022-12-21,10:53:30 | INFO |   val_num_samples: None
+2022-12-21,10:53:30 | INFO |   wandb: False
+2022-12-21,10:53:30 | INFO |   wandb_notes: 
+2022-12-21,10:53:30 | INFO |   warmup: 10000
+2022-12-21,10:53:30 | INFO |   wd: 0.2
+2022-12-21,10:53:30 | INFO |   workers: 1
+2022-12-21,10:53:30 | INFO |   world_size: 1
+2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+2022-12-21,10:53:48 | INFO | Using classifier
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6319	imagenet-zeroshot-val-top5: 0.8823
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a020bdef738d5e5fdfb4fedf19ade569b8b525d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_47.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_47.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b9576e2b95abe91087b57d47541b96ce3aa841bd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.63868, "imagenet-zeroshot-val-top5": 0.88444}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..05472e22d6ba0c5a8820c99751743a453b373c57
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt).
+2022-12-21,10:53:29 | INFO | Model:
+2022-12-21,10:53:29 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:29 | INFO | Params:
+2022-12-21,10:53:29 | INFO |   batch_size: 256
+2022-12-21,10:53:29 | INFO |   beta1: 0.9
+2022-12-21,10:53:29 | INFO |   beta2: 0.98
+2022-12-21,10:53:29 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/checkpoints
+2022-12-21,10:53:29 | INFO |   copy_codebase: False
+2022-12-21,10:53:29 | INFO |   csv_caption_key: title
+2022-12-21,10:53:29 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:29 | INFO |   csv_separator: 	
+2022-12-21,10:53:29 | INFO |   dataset_resampled: False
+2022-12-21,10:53:29 | INFO |   dataset_type: auto
+2022-12-21,10:53:29 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:29 | INFO |   debug: False
+2022-12-21,10:53:29 | INFO |   device: cuda:0
+2022-12-21,10:53:29 | INFO |   dist_backend: nccl
+2022-12-21,10:53:29 | INFO |   dist_url: env://
+2022-12-21,10:53:29 | INFO |   distributed: False
+2022-12-21,10:53:29 | INFO |   epochs: 32
+2022-12-21,10:53:29 | INFO |   eps: 1e-06
+2022-12-21,10:53:29 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:29 | INFO |   gather_with_grad: False
+2022-12-21,10:53:29 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:29 | INFO |   horovod: False
+2022-12-21,10:53:29 | INFO |   image_mean: None
+2022-12-21,10:53:29 | INFO |   image_std: None
+2022-12-21,10:53:29 | INFO |   imagenet_v2: None
+2022-12-21,10:53:29 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:29 | INFO |   local_loss: False
+2022-12-21,10:53:29 | INFO |   local_rank: 0
+2022-12-21,10:53:29 | INFO |   lock_image: False
+2022-12-21,10:53:29 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:29 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:29 | INFO |   log_level: 20
+2022-12-21,10:53:29 | INFO |   log_local: False
+2022-12-21,10:53:29 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/out.log
+2022-12-21,10:53:29 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:29 | INFO |   lr: 0.0005
+2022-12-21,10:53:29 | INFO |   model: ViT-B-16
+2022-12-21,10:53:29 | INFO |   name: epoch_48.pt
+2022-12-21,10:53:29 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:29 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:29 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:29 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt
+2022-12-21,10:53:29 | INFO |   pretrained_image: False
+2022-12-21,10:53:29 | INFO |   rank: 0
+2022-12-21,10:53:29 | INFO |   report_to: 
+2022-12-21,10:53:29 | INFO |   resume: None
+2022-12-21,10:53:29 | INFO |   save_frequency: 1
+2022-12-21,10:53:29 | INFO |   save_most_recent: False
+2022-12-21,10:53:29 | INFO |   seed: 0
+2022-12-21,10:53:29 | INFO |   skip_scheduler: False
+2022-12-21,10:53:29 | INFO |   tensorboard: False
+2022-12-21,10:53:29 | INFO |   tensorboard_path: 
+2022-12-21,10:53:29 | INFO |   torchscript: False
+2022-12-21,10:53:29 | INFO |   trace: False
+2022-12-21,10:53:29 | INFO |   train_data: None
+2022-12-21,10:53:29 | INFO |   train_num_samples: None
+2022-12-21,10:53:29 | INFO |   use_bn_sync: False
+2022-12-21,10:53:29 | INFO |   val_data: None
+2022-12-21,10:53:29 | INFO |   val_frequency: 1
+2022-12-21,10:53:29 | INFO |   val_num_samples: None
+2022-12-21,10:53:29 | INFO |   wandb: False
+2022-12-21,10:53:29 | INFO |   wandb_notes: 
+2022-12-21,10:53:29 | INFO |   warmup: 10000
+2022-12-21,10:53:29 | INFO |   wd: 0.2
+2022-12-21,10:53:29 | INFO |   workers: 1
+2022-12-21,10:53:29 | INFO |   world_size: 1
+2022-12-21,10:53:29 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+2022-12-21,10:53:47 | INFO | Using classifier
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6387	imagenet-zeroshot-val-top5: 0.8844
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9261e8118eafceb5c5e0f260f74ccb548a53da00
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_48.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_48.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b5c410fcb61802deb3bcea35fc1d56837f58ca6f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.6437, "imagenet-zeroshot-val-top5": 0.8902}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..62f80c3b537199aa428dafb859e8974477cb9303
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,10:53:19 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,10:53:19 | INFO | Loading ViT-B-16 model config.
+2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt).
+2022-12-21,10:53:30 | INFO | Model:
+2022-12-21,10:53:30 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,10:53:30 | INFO | Params:
+2022-12-21,10:53:30 | INFO |   batch_size: 256
+2022-12-21,10:53:30 | INFO |   beta1: 0.9
+2022-12-21,10:53:30 | INFO |   beta2: 0.98
+2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/checkpoints
+2022-12-21,10:53:30 | INFO |   copy_codebase: False
+2022-12-21,10:53:30 | INFO |   csv_caption_key: title
+2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
+2022-12-21,10:53:30 | INFO |   csv_separator: 	
+2022-12-21,10:53:30 | INFO |   dataset_resampled: False
+2022-12-21,10:53:30 | INFO |   dataset_type: auto
+2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
+2022-12-21,10:53:30 | INFO |   debug: False
+2022-12-21,10:53:30 | INFO |   device: cuda:0
+2022-12-21,10:53:30 | INFO |   dist_backend: nccl
+2022-12-21,10:53:30 | INFO |   dist_url: env://
+2022-12-21,10:53:30 | INFO |   distributed: False
+2022-12-21,10:53:30 | INFO |   epochs: 32
+2022-12-21,10:53:30 | INFO |   eps: 1e-06
+2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
+2022-12-21,10:53:30 | INFO |   gather_with_grad: False
+2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
+2022-12-21,10:53:30 | INFO |   horovod: False
+2022-12-21,10:53:30 | INFO |   image_mean: None
+2022-12-21,10:53:30 | INFO |   image_std: None
+2022-12-21,10:53:30 | INFO |   imagenet_v2: None
+2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,10:53:30 | INFO |   local_loss: False
+2022-12-21,10:53:30 | INFO |   local_rank: 0
+2022-12-21,10:53:30 | INFO |   lock_image: False
+2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,10:53:30 | INFO |   log_level: 20
+2022-12-21,10:53:30 | INFO |   log_local: False
+2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/out.log
+2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,10:53:30 | INFO |   lr: 0.0005
+2022-12-21,10:53:30 | INFO |   model: ViT-B-16
+2022-12-21,10:53:30 | INFO |   name: epoch_49.pt
+2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
+2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
+2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
+2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt
+2022-12-21,10:53:30 | INFO |   pretrained_image: False
+2022-12-21,10:53:30 | INFO |   rank: 0
+2022-12-21,10:53:30 | INFO |   report_to: 
+2022-12-21,10:53:30 | INFO |   resume: None
+2022-12-21,10:53:30 | INFO |   save_frequency: 1
+2022-12-21,10:53:30 | INFO |   save_most_recent: False
+2022-12-21,10:53:30 | INFO |   seed: 0
+2022-12-21,10:53:30 | INFO |   skip_scheduler: False
+2022-12-21,10:53:30 | INFO |   tensorboard: False
+2022-12-21,10:53:30 | INFO |   tensorboard_path: 
+2022-12-21,10:53:30 | INFO |   torchscript: False
+2022-12-21,10:53:30 | INFO |   trace: False
+2022-12-21,10:53:30 | INFO |   train_data: None
+2022-12-21,10:53:30 | INFO |   train_num_samples: None
+2022-12-21,10:53:30 | INFO |   use_bn_sync: False
+2022-12-21,10:53:30 | INFO |   val_data: None
+2022-12-21,10:53:30 | INFO |   val_frequency: 1
+2022-12-21,10:53:30 | INFO |   val_num_samples: None
+2022-12-21,10:53:30 | INFO |   wandb: False
+2022-12-21,10:53:30 | INFO |   wandb_notes: 
+2022-12-21,10:53:30 | INFO |   warmup: 10000
+2022-12-21,10:53:30 | INFO |   wd: 0.2
+2022-12-21,10:53:30 | INFO |   workers: 1
+2022-12-21,10:53:30 | INFO |   world_size: 1
+2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
+2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
+2022-12-21,10:53:30 | INFO | Building zero-shot classifier
+2022-12-21,10:53:48 | INFO | Using classifier
+2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6437	imagenet-zeroshot-val-top5: 0.8902
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b6bf5c8ed58366582d27e82e4de18f580e1603fe
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_49.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_49.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..58c94b2ca398601387655c0278544cbe8eb04dfb
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.46722, "imagenet-zeroshot-val-top5": 0.7495}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..bf6843d9016d67d845ad5348c6c8c1978bd0080d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_5.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+2022-12-21,11:03:09 | INFO | Using classifier
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4672	imagenet-zeroshot-val-top5: 0.7495
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2bfbd45a5f34f9393be1533b2a92968a833548cd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_5.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_5.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..efe22992d7d3c610ea4fce556638d0f62fed9ee5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.64662, "imagenet-zeroshot-val-top5": 0.8905}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..7064b0ac11bfe6b4b6c12c5b427961adae61e8a9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_50.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+2022-12-21,11:03:09 | INFO | Using classifier
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6466	imagenet-zeroshot-val-top5: 0.8905
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..befd95d8ec9abb52124c71eca82537b8e6c9d17c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_50.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_50.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..4211f36c6baf3a96a725e14612f1c40c13078d51
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.65182, "imagenet-zeroshot-val-top5": 0.89216}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..c950a72cab3ee96718904d638eb33523ed7d9fde
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_51.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+2022-12-21,11:03:09 | INFO | Using classifier
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6518	imagenet-zeroshot-val-top5: 0.8922
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..07a43da66673a8e1b74d59e3b9c1896a7f388017
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_51.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_51.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1912a98577c5ddce628fe13ffdfef482c25bffb6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.64742, "imagenet-zeroshot-val-top5": 0.89092}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..4c350127c2715f67e7a619bd36ca77832086e609
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt).
+2022-12-21,11:02:51 | INFO | Model:
+2022-12-21,11:02:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:02:51 | INFO | Params:
+2022-12-21,11:02:51 | INFO |   batch_size: 256
+2022-12-21,11:02:51 | INFO |   beta1: 0.9
+2022-12-21,11:02:51 | INFO |   beta2: 0.98
+2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/checkpoints
+2022-12-21,11:02:51 | INFO |   copy_codebase: False
+2022-12-21,11:02:51 | INFO |   csv_caption_key: title
+2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:02:51 | INFO |   csv_separator: 	
+2022-12-21,11:02:51 | INFO |   dataset_resampled: False
+2022-12-21,11:02:51 | INFO |   dataset_type: auto
+2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:02:51 | INFO |   debug: False
+2022-12-21,11:02:51 | INFO |   device: cuda:0
+2022-12-21,11:02:51 | INFO |   dist_backend: nccl
+2022-12-21,11:02:51 | INFO |   dist_url: env://
+2022-12-21,11:02:51 | INFO |   distributed: False
+2022-12-21,11:02:51 | INFO |   epochs: 32
+2022-12-21,11:02:51 | INFO |   eps: 1e-06
+2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:02:51 | INFO |   gather_with_grad: False
+2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:02:51 | INFO |   horovod: False
+2022-12-21,11:02:51 | INFO |   image_mean: None
+2022-12-21,11:02:51 | INFO |   image_std: None
+2022-12-21,11:02:51 | INFO |   imagenet_v2: None
+2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:02:51 | INFO |   local_loss: False
+2022-12-21,11:02:51 | INFO |   local_rank: 0
+2022-12-21,11:02:51 | INFO |   lock_image: False
+2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:02:51 | INFO |   log_level: 20
+2022-12-21,11:02:51 | INFO |   log_local: False
+2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/out.log
+2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:02:51 | INFO |   lr: 0.0005
+2022-12-21,11:02:51 | INFO |   model: ViT-B-16
+2022-12-21,11:02:51 | INFO |   name: epoch_52.pt
+2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt
+2022-12-21,11:02:51 | INFO |   pretrained_image: False
+2022-12-21,11:02:51 | INFO |   rank: 0
+2022-12-21,11:02:51 | INFO |   report_to: 
+2022-12-21,11:02:51 | INFO |   resume: None
+2022-12-21,11:02:51 | INFO |   save_frequency: 1
+2022-12-21,11:02:51 | INFO |   save_most_recent: False
+2022-12-21,11:02:51 | INFO |   seed: 0
+2022-12-21,11:02:51 | INFO |   skip_scheduler: False
+2022-12-21,11:02:51 | INFO |   tensorboard: False
+2022-12-21,11:02:51 | INFO |   tensorboard_path: 
+2022-12-21,11:02:51 | INFO |   torchscript: False
+2022-12-21,11:02:51 | INFO |   trace: False
+2022-12-21,11:02:51 | INFO |   train_data: None
+2022-12-21,11:02:51 | INFO |   train_num_samples: None
+2022-12-21,11:02:51 | INFO |   use_bn_sync: False
+2022-12-21,11:02:51 | INFO |   val_data: None
+2022-12-21,11:02:51 | INFO |   val_frequency: 1
+2022-12-21,11:02:51 | INFO |   val_num_samples: None
+2022-12-21,11:02:51 | INFO |   wandb: False
+2022-12-21,11:02:51 | INFO |   wandb_notes: 
+2022-12-21,11:02:51 | INFO |   warmup: 10000
+2022-12-21,11:02:51 | INFO |   wd: 0.2
+2022-12-21,11:02:51 | INFO |   workers: 1
+2022-12-21,11:02:51 | INFO |   world_size: 1
+2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:02:51 | INFO | Building zero-shot classifier
+2022-12-21,11:03:10 | INFO | Using classifier
+2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6474	imagenet-zeroshot-val-top5: 0.8909
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..333a994a72e05c6912b9dd83ab937f6c33367fb6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_52.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_52.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..5efcf3de2cadcd50fca4729c7f11141c53cfa264
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.65454, "imagenet-zeroshot-val-top5": 0.8941}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..a5f60af20557393019e1fabb73b63072780a2453
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt).
+2022-12-21,11:11:47 | INFO | Model:
+2022-12-21,11:11:47 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:47 | INFO | Params:
+2022-12-21,11:11:47 | INFO |   batch_size: 256
+2022-12-21,11:11:47 | INFO |   beta1: 0.9
+2022-12-21,11:11:47 | INFO |   beta2: 0.98
+2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/checkpoints
+2022-12-21,11:11:47 | INFO |   copy_codebase: False
+2022-12-21,11:11:47 | INFO |   csv_caption_key: title
+2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:47 | INFO |   csv_separator: 	
+2022-12-21,11:11:47 | INFO |   dataset_resampled: False
+2022-12-21,11:11:47 | INFO |   dataset_type: auto
+2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:47 | INFO |   debug: False
+2022-12-21,11:11:47 | INFO |   device: cuda:0
+2022-12-21,11:11:47 | INFO |   dist_backend: nccl
+2022-12-21,11:11:47 | INFO |   dist_url: env://
+2022-12-21,11:11:47 | INFO |   distributed: False
+2022-12-21,11:11:47 | INFO |   epochs: 32
+2022-12-21,11:11:47 | INFO |   eps: 1e-06
+2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:47 | INFO |   gather_with_grad: False
+2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:47 | INFO |   horovod: False
+2022-12-21,11:11:47 | INFO |   image_mean: None
+2022-12-21,11:11:47 | INFO |   image_std: None
+2022-12-21,11:11:47 | INFO |   imagenet_v2: None
+2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:47 | INFO |   local_loss: False
+2022-12-21,11:11:47 | INFO |   local_rank: 0
+2022-12-21,11:11:47 | INFO |   lock_image: False
+2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:47 | INFO |   log_level: 20
+2022-12-21,11:11:47 | INFO |   log_local: False
+2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/out.log
+2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:47 | INFO |   lr: 0.0005
+2022-12-21,11:11:47 | INFO |   model: ViT-B-16
+2022-12-21,11:11:47 | INFO |   name: epoch_53.pt
+2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt
+2022-12-21,11:11:47 | INFO |   pretrained_image: False
+2022-12-21,11:11:47 | INFO |   rank: 0
+2022-12-21,11:11:47 | INFO |   report_to: 
+2022-12-21,11:11:47 | INFO |   resume: None
+2022-12-21,11:11:47 | INFO |   save_frequency: 1
+2022-12-21,11:11:47 | INFO |   save_most_recent: False
+2022-12-21,11:11:47 | INFO |   seed: 0
+2022-12-21,11:11:47 | INFO |   skip_scheduler: False
+2022-12-21,11:11:47 | INFO |   tensorboard: False
+2022-12-21,11:11:47 | INFO |   tensorboard_path: 
+2022-12-21,11:11:47 | INFO |   torchscript: False
+2022-12-21,11:11:47 | INFO |   trace: False
+2022-12-21,11:11:47 | INFO |   train_data: None
+2022-12-21,11:11:47 | INFO |   train_num_samples: None
+2022-12-21,11:11:47 | INFO |   use_bn_sync: False
+2022-12-21,11:11:47 | INFO |   val_data: None
+2022-12-21,11:11:47 | INFO |   val_frequency: 1
+2022-12-21,11:11:47 | INFO |   val_num_samples: None
+2022-12-21,11:11:47 | INFO |   wandb: False
+2022-12-21,11:11:47 | INFO |   wandb_notes: 
+2022-12-21,11:11:47 | INFO |   warmup: 10000
+2022-12-21,11:11:47 | INFO |   wd: 0.2
+2022-12-21,11:11:47 | INFO |   workers: 1
+2022-12-21,11:11:47 | INFO |   world_size: 1
+2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:47 | INFO | Building zero-shot classifier
+2022-12-21,11:12:05 | INFO | Using classifier
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6545	imagenet-zeroshot-val-top5: 0.8941
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..255e1cdbe9cfa4e884d9602cd9601373d6e31148
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_53.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_53.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..c9fc50876f0207c1cd05b948d3966f3396bded36
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.65888, "imagenet-zeroshot-val-top5": 0.89588}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..41ee3f9d6519d9e200a5d250bd83b6cdb78a3a03
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt).
+2022-12-21,11:11:45 | INFO | Model:
+2022-12-21,11:11:45 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:45 | INFO | Params:
+2022-12-21,11:11:45 | INFO |   batch_size: 256
+2022-12-21,11:11:45 | INFO |   beta1: 0.9
+2022-12-21,11:11:45 | INFO |   beta2: 0.98
+2022-12-21,11:11:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/checkpoints
+2022-12-21,11:11:45 | INFO |   copy_codebase: False
+2022-12-21,11:11:45 | INFO |   csv_caption_key: title
+2022-12-21,11:11:45 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:45 | INFO |   csv_separator: 	
+2022-12-21,11:11:45 | INFO |   dataset_resampled: False
+2022-12-21,11:11:45 | INFO |   dataset_type: auto
+2022-12-21,11:11:45 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:45 | INFO |   debug: False
+2022-12-21,11:11:45 | INFO |   device: cuda:0
+2022-12-21,11:11:45 | INFO |   dist_backend: nccl
+2022-12-21,11:11:45 | INFO |   dist_url: env://
+2022-12-21,11:11:45 | INFO |   distributed: False
+2022-12-21,11:11:45 | INFO |   epochs: 32
+2022-12-21,11:11:45 | INFO |   eps: 1e-06
+2022-12-21,11:11:45 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:45 | INFO |   gather_with_grad: False
+2022-12-21,11:11:45 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:45 | INFO |   horovod: False
+2022-12-21,11:11:45 | INFO |   image_mean: None
+2022-12-21,11:11:45 | INFO |   image_std: None
+2022-12-21,11:11:45 | INFO |   imagenet_v2: None
+2022-12-21,11:11:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:45 | INFO |   local_loss: False
+2022-12-21,11:11:45 | INFO |   local_rank: 0
+2022-12-21,11:11:45 | INFO |   lock_image: False
+2022-12-21,11:11:45 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:45 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:45 | INFO |   log_level: 20
+2022-12-21,11:11:45 | INFO |   log_local: False
+2022-12-21,11:11:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/out.log
+2022-12-21,11:11:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:45 | INFO |   lr: 0.0005
+2022-12-21,11:11:45 | INFO |   model: ViT-B-16
+2022-12-21,11:11:45 | INFO |   name: epoch_54.pt
+2022-12-21,11:11:45 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:45 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:45 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt
+2022-12-21,11:11:45 | INFO |   pretrained_image: False
+2022-12-21,11:11:45 | INFO |   rank: 0
+2022-12-21,11:11:45 | INFO |   report_to: 
+2022-12-21,11:11:45 | INFO |   resume: None
+2022-12-21,11:11:45 | INFO |   save_frequency: 1
+2022-12-21,11:11:45 | INFO |   save_most_recent: False
+2022-12-21,11:11:45 | INFO |   seed: 0
+2022-12-21,11:11:45 | INFO |   skip_scheduler: False
+2022-12-21,11:11:45 | INFO |   tensorboard: False
+2022-12-21,11:11:45 | INFO |   tensorboard_path: 
+2022-12-21,11:11:45 | INFO |   torchscript: False
+2022-12-21,11:11:45 | INFO |   trace: False
+2022-12-21,11:11:45 | INFO |   train_data: None
+2022-12-21,11:11:45 | INFO |   train_num_samples: None
+2022-12-21,11:11:45 | INFO |   use_bn_sync: False
+2022-12-21,11:11:45 | INFO |   val_data: None
+2022-12-21,11:11:45 | INFO |   val_frequency: 1
+2022-12-21,11:11:45 | INFO |   val_num_samples: None
+2022-12-21,11:11:45 | INFO |   wandb: False
+2022-12-21,11:11:45 | INFO |   wandb_notes: 
+2022-12-21,11:11:45 | INFO |   warmup: 10000
+2022-12-21,11:11:45 | INFO |   wd: 0.2
+2022-12-21,11:11:45 | INFO |   workers: 1
+2022-12-21,11:11:45 | INFO |   world_size: 1
+2022-12-21,11:11:45 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:46 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:46 | INFO | Building zero-shot classifier
+2022-12-21,11:12:04 | INFO | Using classifier
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6589	imagenet-zeroshot-val-top5: 0.8959
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..674558e8dc3657fedb2712167e2eea0316d7ee9b
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_54.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_54.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..022e41b2db2bea818fec304c1972a03e9b799685
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.6622, "imagenet-zeroshot-val-top5": 0.89786}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..91adca9ac48d535de719a20f82b83827d0e54de6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_55.pt).
+2022-12-21,11:11:47 | INFO | Model:
+2022-12-21,11:11:47 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:47 | INFO | Params:
+2022-12-21,11:11:47 | INFO |   batch_size: 256
+2022-12-21,11:11:47 | INFO |   beta1: 0.9
+2022-12-21,11:11:47 | INFO |   beta2: 0.98
+2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_55.pt/checkpoints
+2022-12-21,11:11:47 | INFO |   copy_codebase: False
+2022-12-21,11:11:47 | INFO |   csv_caption_key: title
+2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:47 | INFO |   csv_separator: 	
+2022-12-21,11:11:47 | INFO |   dataset_resampled: False
+2022-12-21,11:11:47 | INFO |   dataset_type: auto
+2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:47 | INFO |   debug: False
+2022-12-21,11:11:47 | INFO |   device: cuda:0
+2022-12-21,11:11:47 | INFO |   dist_backend: nccl
+2022-12-21,11:11:47 | INFO |   dist_url: env://
+2022-12-21,11:11:47 | INFO |   distributed: False
+2022-12-21,11:11:47 | INFO |   epochs: 32
+2022-12-21,11:11:47 | INFO |   eps: 1e-06
+2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:47 | INFO |   gather_with_grad: False
+2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:47 | INFO |   horovod: False
+2022-12-21,11:11:47 | INFO |   image_mean: None
+2022-12-21,11:11:47 | INFO |   image_std: None
+2022-12-21,11:11:47 | INFO |   imagenet_v2: None
+2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:47 | INFO |   local_loss: False
+2022-12-21,11:11:47 | INFO |   local_rank: 0
+2022-12-21,11:11:47 | INFO |   lock_image: False
+2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:47 | INFO |   log_level: 20
+2022-12-21,11:11:47 | INFO |   log_local: False
+2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_55.pt/out.log
+2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:47 | INFO |   lr: 0.0005
+2022-12-21,11:11:47 | INFO |   model: ViT-B-16
+2022-12-21,11:11:47 | INFO |   name: epoch_55.pt
+2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_55.pt
+2022-12-21,11:11:47 | INFO |   pretrained_image: False
+2022-12-21,11:11:47 | INFO |   rank: 0
+2022-12-21,11:11:47 | INFO |   report_to: 
+2022-12-21,11:11:47 | INFO |   resume: None
+2022-12-21,11:11:47 | INFO |   save_frequency: 1
+2022-12-21,11:11:47 | INFO |   save_most_recent: False
+2022-12-21,11:11:47 | INFO |   seed: 0
+2022-12-21,11:11:47 | INFO |   skip_scheduler: False
+2022-12-21,11:11:47 | INFO |   tensorboard: False
+2022-12-21,11:11:47 | INFO |   tensorboard_path: 
+2022-12-21,11:11:47 | INFO |   torchscript: False
+2022-12-21,11:11:47 | INFO |   trace: False
+2022-12-21,11:11:47 | INFO |   train_data: None
+2022-12-21,11:11:47 | INFO |   train_num_samples: None
+2022-12-21,11:11:47 | INFO |   use_bn_sync: False
+2022-12-21,11:11:47 | INFO |   val_data: None
+2022-12-21,11:11:47 | INFO |   val_frequency: 1
+2022-12-21,11:11:47 | INFO |   val_num_samples: None
+2022-12-21,11:11:47 | INFO |   wandb: False
+2022-12-21,11:11:47 | INFO |   wandb_notes: 
+2022-12-21,11:11:47 | INFO |   warmup: 10000
+2022-12-21,11:11:47 | INFO |   wd: 0.2
+2022-12-21,11:11:47 | INFO |   workers: 1
+2022-12-21,11:11:47 | INFO |   world_size: 1
+2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:47 | INFO | Building zero-shot classifier
+2022-12-21,11:12:05 | INFO | Using classifier
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6622	imagenet-zeroshot-val-top5: 0.8979
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..33065e0fc810a3c04ba5e34031de1cf5f9a1fdcd
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_55.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_55.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_55.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_55.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_55.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..da6dc606720f107a9be49493e10005a06b1e38b1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.66584, "imagenet-zeroshot-val-top5": 0.90012}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..fbb77c3fa83aec0cc2d1d8a0d033f6a78a94e707
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_56.pt).
+2022-12-21,11:11:47 | INFO | Model:
+2022-12-21,11:11:47 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:11:47 | INFO | Params:
+2022-12-21,11:11:47 | INFO |   batch_size: 256
+2022-12-21,11:11:47 | INFO |   beta1: 0.9
+2022-12-21,11:11:47 | INFO |   beta2: 0.98
+2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_56.pt/checkpoints
+2022-12-21,11:11:47 | INFO |   copy_codebase: False
+2022-12-21,11:11:47 | INFO |   csv_caption_key: title
+2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
+2022-12-21,11:11:47 | INFO |   csv_separator: 	
+2022-12-21,11:11:47 | INFO |   dataset_resampled: False
+2022-12-21,11:11:47 | INFO |   dataset_type: auto
+2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
+2022-12-21,11:11:47 | INFO |   debug: False
+2022-12-21,11:11:47 | INFO |   device: cuda:0
+2022-12-21,11:11:47 | INFO |   dist_backend: nccl
+2022-12-21,11:11:47 | INFO |   dist_url: env://
+2022-12-21,11:11:47 | INFO |   distributed: False
+2022-12-21,11:11:47 | INFO |   epochs: 32
+2022-12-21,11:11:47 | INFO |   eps: 1e-06
+2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
+2022-12-21,11:11:47 | INFO |   gather_with_grad: False
+2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
+2022-12-21,11:11:47 | INFO |   horovod: False
+2022-12-21,11:11:47 | INFO |   image_mean: None
+2022-12-21,11:11:47 | INFO |   image_std: None
+2022-12-21,11:11:47 | INFO |   imagenet_v2: None
+2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:11:47 | INFO |   local_loss: False
+2022-12-21,11:11:47 | INFO |   local_rank: 0
+2022-12-21,11:11:47 | INFO |   lock_image: False
+2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:11:47 | INFO |   log_level: 20
+2022-12-21,11:11:47 | INFO |   log_local: False
+2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_56.pt/out.log
+2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:11:47 | INFO |   lr: 0.0005
+2022-12-21,11:11:47 | INFO |   model: ViT-B-16
+2022-12-21,11:11:47 | INFO |   name: epoch_56.pt
+2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
+2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
+2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
+2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_56.pt
+2022-12-21,11:11:47 | INFO |   pretrained_image: False
+2022-12-21,11:11:47 | INFO |   rank: 0
+2022-12-21,11:11:47 | INFO |   report_to: 
+2022-12-21,11:11:47 | INFO |   resume: None
+2022-12-21,11:11:47 | INFO |   save_frequency: 1
+2022-12-21,11:11:47 | INFO |   save_most_recent: False
+2022-12-21,11:11:47 | INFO |   seed: 0
+2022-12-21,11:11:47 | INFO |   skip_scheduler: False
+2022-12-21,11:11:47 | INFO |   tensorboard: False
+2022-12-21,11:11:47 | INFO |   tensorboard_path: 
+2022-12-21,11:11:47 | INFO |   torchscript: False
+2022-12-21,11:11:47 | INFO |   trace: False
+2022-12-21,11:11:47 | INFO |   train_data: None
+2022-12-21,11:11:47 | INFO |   train_num_samples: None
+2022-12-21,11:11:47 | INFO |   use_bn_sync: False
+2022-12-21,11:11:47 | INFO |   val_data: None
+2022-12-21,11:11:47 | INFO |   val_frequency: 1
+2022-12-21,11:11:47 | INFO |   val_num_samples: None
+2022-12-21,11:11:47 | INFO |   wandb: False
+2022-12-21,11:11:47 | INFO |   wandb_notes: 
+2022-12-21,11:11:47 | INFO |   warmup: 10000
+2022-12-21,11:11:47 | INFO |   wd: 0.2
+2022-12-21,11:11:47 | INFO |   workers: 1
+2022-12-21,11:11:47 | INFO |   world_size: 1
+2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:11:47 | INFO | Building zero-shot classifier
+2022-12-21,11:12:05 | INFO | Using classifier
+2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6658	imagenet-zeroshot-val-top5: 0.9001
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..63d34910c863ce4b4f43af11a9fce70b07a4b144
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_56.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_56.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_56.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_56.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_56.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ffd85f713e4148444e1949ad50b1bf41727e2c27
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.66758, "imagenet-zeroshot-val-top5": 0.90222}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..e28e852c562c4d438416ae47c6f3148cb059a109
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_57.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_57.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_57.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_57.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_57.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:12 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:12 | INFO | Building zero-shot classifier
+2022-12-21,11:21:30 | INFO | Using classifier
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6676	imagenet-zeroshot-val-top5: 0.9022
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b618dadef09426c5c294b31a96782c179eb1b26f
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_57.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_57.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_57.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_57.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_57.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..1fc343eb27521038724a7667a0d5b2fdd5c863a2
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.67204, "imagenet-zeroshot-val-top5": 0.90514}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..f75e6d00343c4fb41761445ce8aaf293dc4ba14d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_58.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_58.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_58.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_58.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_58.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:12 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:12 | INFO | Building zero-shot classifier
+2022-12-21,11:21:29 | INFO | Using classifier
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6720	imagenet-zeroshot-val-top5: 0.9051
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..57b5e7fdf9b018a238d382e4b9a3a0075cbf698d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_58.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_58.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_58.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_58.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_58.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..0ce0826e4de6ea65657b66c0d2000c58858ab879
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.67176, "imagenet-zeroshot-val-top5": 0.90568}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..053357e7e3dc5724ecef2cfabeca16e60dffb6b3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_59.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_59.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_59.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_59.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_59.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:11 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:11 | INFO | Building zero-shot classifier
+2022-12-21,11:21:29 | INFO | Using classifier
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6718	imagenet-zeroshot-val-top5: 0.9057
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e6fca90a42632d06d497993e1fbff33c4704c289
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_59.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_59.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_59.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_59.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_59.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..69e4e001d959ec35e309b69da57df535f3be4ac6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.48708, "imagenet-zeroshot-val-top5": 0.769}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..e7a45f245ca635de29db8ec74410b79c9884e59c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:21:01 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:21:01 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:21:03 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_6.pt).
+2022-12-21,11:21:11 | INFO | Model:
+2022-12-21,11:21:11 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:21:11 | INFO | Params:
+2022-12-21,11:21:11 | INFO |   batch_size: 256
+2022-12-21,11:21:11 | INFO |   beta1: 0.9
+2022-12-21,11:21:11 | INFO |   beta2: 0.98
+2022-12-21,11:21:11 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_6.pt/checkpoints
+2022-12-21,11:21:11 | INFO |   copy_codebase: False
+2022-12-21,11:21:11 | INFO |   csv_caption_key: title
+2022-12-21,11:21:11 | INFO |   csv_img_key: filepath
+2022-12-21,11:21:11 | INFO |   csv_separator: 	
+2022-12-21,11:21:11 | INFO |   dataset_resampled: False
+2022-12-21,11:21:11 | INFO |   dataset_type: auto
+2022-12-21,11:21:11 | INFO |   ddp_static_graph: False
+2022-12-21,11:21:11 | INFO |   debug: False
+2022-12-21,11:21:11 | INFO |   device: cuda:0
+2022-12-21,11:21:11 | INFO |   dist_backend: nccl
+2022-12-21,11:21:11 | INFO |   dist_url: env://
+2022-12-21,11:21:11 | INFO |   distributed: False
+2022-12-21,11:21:11 | INFO |   epochs: 32
+2022-12-21,11:21:11 | INFO |   eps: 1e-06
+2022-12-21,11:21:11 | INFO |   force_quick_gelu: False
+2022-12-21,11:21:11 | INFO |   gather_with_grad: False
+2022-12-21,11:21:11 | INFO |   grad_checkpointing: False
+2022-12-21,11:21:11 | INFO |   horovod: False
+2022-12-21,11:21:11 | INFO |   image_mean: None
+2022-12-21,11:21:11 | INFO |   image_std: None
+2022-12-21,11:21:11 | INFO |   imagenet_v2: None
+2022-12-21,11:21:11 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:21:11 | INFO |   local_loss: False
+2022-12-21,11:21:11 | INFO |   local_rank: 0
+2022-12-21,11:21:11 | INFO |   lock_image: False
+2022-12-21,11:21:11 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:21:11 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:21:11 | INFO |   log_level: 20
+2022-12-21,11:21:11 | INFO |   log_local: False
+2022-12-21,11:21:11 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_6.pt/out.log
+2022-12-21,11:21:11 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:21:11 | INFO |   lr: 0.0005
+2022-12-21,11:21:11 | INFO |   model: ViT-B-16
+2022-12-21,11:21:11 | INFO |   name: epoch_6.pt
+2022-12-21,11:21:11 | INFO |   no_set_device_rank: False
+2022-12-21,11:21:11 | INFO |   norm_gradient_clip: None
+2022-12-21,11:21:11 | INFO |   precision: amp_bfloat16
+2022-12-21,11:21:11 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_6.pt
+2022-12-21,11:21:11 | INFO |   pretrained_image: False
+2022-12-21,11:21:11 | INFO |   rank: 0
+2022-12-21,11:21:11 | INFO |   report_to: 
+2022-12-21,11:21:11 | INFO |   resume: None
+2022-12-21,11:21:11 | INFO |   save_frequency: 1
+2022-12-21,11:21:11 | INFO |   save_most_recent: False
+2022-12-21,11:21:11 | INFO |   seed: 0
+2022-12-21,11:21:11 | INFO |   skip_scheduler: False
+2022-12-21,11:21:11 | INFO |   tensorboard: False
+2022-12-21,11:21:11 | INFO |   tensorboard_path: 
+2022-12-21,11:21:11 | INFO |   torchscript: False
+2022-12-21,11:21:11 | INFO |   trace: False
+2022-12-21,11:21:11 | INFO |   train_data: None
+2022-12-21,11:21:11 | INFO |   train_num_samples: None
+2022-12-21,11:21:11 | INFO |   use_bn_sync: False
+2022-12-21,11:21:11 | INFO |   val_data: None
+2022-12-21,11:21:11 | INFO |   val_frequency: 1
+2022-12-21,11:21:11 | INFO |   val_num_samples: None
+2022-12-21,11:21:11 | INFO |   wandb: False
+2022-12-21,11:21:11 | INFO |   wandb_notes: 
+2022-12-21,11:21:11 | INFO |   warmup: 10000
+2022-12-21,11:21:11 | INFO |   wd: 0.2
+2022-12-21,11:21:11 | INFO |   workers: 1
+2022-12-21,11:21:11 | INFO |   world_size: 1
+2022-12-21,11:21:11 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:21:11 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:21:11 | INFO | Building zero-shot classifier
+2022-12-21,11:21:29 | INFO | Using classifier
+2022-12-21,11:29:41 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:29:41 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4871	imagenet-zeroshot-val-top5: 0.7690
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..9c566b5ae417c14494dcd6c8cc378d4380e62d9d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_6.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_6.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_6.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_6.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_6.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..9b1ca9ae98a520ef6ca4ef088fe98a8123578b6a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.67582, "imagenet-zeroshot-val-top5": 0.90622}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..a2562ff73a9883c954d7c70131507ca7c71972f7
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_60.pt).
+2022-12-21,11:30:06 | INFO | Model:
+2022-12-21,11:30:06 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:06 | INFO | Params:
+2022-12-21,11:30:06 | INFO |   batch_size: 256
+2022-12-21,11:30:06 | INFO |   beta1: 0.9
+2022-12-21,11:30:06 | INFO |   beta2: 0.98
+2022-12-21,11:30:06 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_60.pt/checkpoints
+2022-12-21,11:30:06 | INFO |   copy_codebase: False
+2022-12-21,11:30:06 | INFO |   csv_caption_key: title
+2022-12-21,11:30:06 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:06 | INFO |   csv_separator: 	
+2022-12-21,11:30:06 | INFO |   dataset_resampled: False
+2022-12-21,11:30:06 | INFO |   dataset_type: auto
+2022-12-21,11:30:06 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:06 | INFO |   debug: False
+2022-12-21,11:30:06 | INFO |   device: cuda:0
+2022-12-21,11:30:06 | INFO |   dist_backend: nccl
+2022-12-21,11:30:06 | INFO |   dist_url: env://
+2022-12-21,11:30:06 | INFO |   distributed: False
+2022-12-21,11:30:06 | INFO |   epochs: 32
+2022-12-21,11:30:06 | INFO |   eps: 1e-06
+2022-12-21,11:30:06 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:06 | INFO |   gather_with_grad: False
+2022-12-21,11:30:06 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:06 | INFO |   horovod: False
+2022-12-21,11:30:06 | INFO |   image_mean: None
+2022-12-21,11:30:06 | INFO |   image_std: None
+2022-12-21,11:30:06 | INFO |   imagenet_v2: None
+2022-12-21,11:30:06 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:06 | INFO |   local_loss: False
+2022-12-21,11:30:06 | INFO |   local_rank: 0
+2022-12-21,11:30:06 | INFO |   lock_image: False
+2022-12-21,11:30:06 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:06 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:06 | INFO |   log_level: 20
+2022-12-21,11:30:06 | INFO |   log_local: False
+2022-12-21,11:30:06 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_60.pt/out.log
+2022-12-21,11:30:06 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:06 | INFO |   lr: 0.0005
+2022-12-21,11:30:06 | INFO |   model: ViT-B-16
+2022-12-21,11:30:06 | INFO |   name: epoch_60.pt
+2022-12-21,11:30:06 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:06 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:06 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:06 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_60.pt
+2022-12-21,11:30:06 | INFO |   pretrained_image: False
+2022-12-21,11:30:06 | INFO |   rank: 0
+2022-12-21,11:30:06 | INFO |   report_to: 
+2022-12-21,11:30:06 | INFO |   resume: None
+2022-12-21,11:30:06 | INFO |   save_frequency: 1
+2022-12-21,11:30:06 | INFO |   save_most_recent: False
+2022-12-21,11:30:06 | INFO |   seed: 0
+2022-12-21,11:30:06 | INFO |   skip_scheduler: False
+2022-12-21,11:30:06 | INFO |   tensorboard: False
+2022-12-21,11:30:06 | INFO |   tensorboard_path: 
+2022-12-21,11:30:06 | INFO |   torchscript: False
+2022-12-21,11:30:06 | INFO |   trace: False
+2022-12-21,11:30:06 | INFO |   train_data: None
+2022-12-21,11:30:06 | INFO |   train_num_samples: None
+2022-12-21,11:30:06 | INFO |   use_bn_sync: False
+2022-12-21,11:30:06 | INFO |   val_data: None
+2022-12-21,11:30:06 | INFO |   val_frequency: 1
+2022-12-21,11:30:06 | INFO |   val_num_samples: None
+2022-12-21,11:30:06 | INFO |   wandb: False
+2022-12-21,11:30:06 | INFO |   wandb_notes: 
+2022-12-21,11:30:06 | INFO |   warmup: 10000
+2022-12-21,11:30:06 | INFO |   wd: 0.2
+2022-12-21,11:30:06 | INFO |   workers: 1
+2022-12-21,11:30:06 | INFO |   world_size: 1
+2022-12-21,11:30:06 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:06 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:06 | INFO | Building zero-shot classifier
+2022-12-21,11:30:24 | INFO | Using classifier
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6758	imagenet-zeroshot-val-top5: 0.9062
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0cdc6e27857863d8c9003f42f4b0847d6b7b0cd4
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_60.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_60.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_60.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_60.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_60.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..bc330e89ee68496f2cffbd8d13f45e83506d400c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.67892, "imagenet-zeroshot-val-top5": 0.90768}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..d71f0b7f7c42eca797ab47bbcf7a9629bcc0f3a5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_61.pt).
+2022-12-21,11:30:06 | INFO | Model:
+2022-12-21,11:30:06 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:06 | INFO | Params:
+2022-12-21,11:30:06 | INFO |   batch_size: 256
+2022-12-21,11:30:06 | INFO |   beta1: 0.9
+2022-12-21,11:30:06 | INFO |   beta2: 0.98
+2022-12-21,11:30:06 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_61.pt/checkpoints
+2022-12-21,11:30:06 | INFO |   copy_codebase: False
+2022-12-21,11:30:06 | INFO |   csv_caption_key: title
+2022-12-21,11:30:06 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:06 | INFO |   csv_separator: 	
+2022-12-21,11:30:06 | INFO |   dataset_resampled: False
+2022-12-21,11:30:06 | INFO |   dataset_type: auto
+2022-12-21,11:30:06 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:06 | INFO |   debug: False
+2022-12-21,11:30:06 | INFO |   device: cuda:0
+2022-12-21,11:30:06 | INFO |   dist_backend: nccl
+2022-12-21,11:30:06 | INFO |   dist_url: env://
+2022-12-21,11:30:06 | INFO |   distributed: False
+2022-12-21,11:30:06 | INFO |   epochs: 32
+2022-12-21,11:30:06 | INFO |   eps: 1e-06
+2022-12-21,11:30:06 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:06 | INFO |   gather_with_grad: False
+2022-12-21,11:30:06 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:06 | INFO |   horovod: False
+2022-12-21,11:30:06 | INFO |   image_mean: None
+2022-12-21,11:30:06 | INFO |   image_std: None
+2022-12-21,11:30:06 | INFO |   imagenet_v2: None
+2022-12-21,11:30:06 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:06 | INFO |   local_loss: False
+2022-12-21,11:30:06 | INFO |   local_rank: 0
+2022-12-21,11:30:06 | INFO |   lock_image: False
+2022-12-21,11:30:06 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:06 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:06 | INFO |   log_level: 20
+2022-12-21,11:30:06 | INFO |   log_local: False
+2022-12-21,11:30:06 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_61.pt/out.log
+2022-12-21,11:30:06 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:06 | INFO |   lr: 0.0005
+2022-12-21,11:30:06 | INFO |   model: ViT-B-16
+2022-12-21,11:30:06 | INFO |   name: epoch_61.pt
+2022-12-21,11:30:06 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:06 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:06 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:06 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_61.pt
+2022-12-21,11:30:06 | INFO |   pretrained_image: False
+2022-12-21,11:30:06 | INFO |   rank: 0
+2022-12-21,11:30:06 | INFO |   report_to: 
+2022-12-21,11:30:06 | INFO |   resume: None
+2022-12-21,11:30:06 | INFO |   save_frequency: 1
+2022-12-21,11:30:06 | INFO |   save_most_recent: False
+2022-12-21,11:30:06 | INFO |   seed: 0
+2022-12-21,11:30:06 | INFO |   skip_scheduler: False
+2022-12-21,11:30:06 | INFO |   tensorboard: False
+2022-12-21,11:30:06 | INFO |   tensorboard_path: 
+2022-12-21,11:30:06 | INFO |   torchscript: False
+2022-12-21,11:30:06 | INFO |   trace: False
+2022-12-21,11:30:06 | INFO |   train_data: None
+2022-12-21,11:30:06 | INFO |   train_num_samples: None
+2022-12-21,11:30:06 | INFO |   use_bn_sync: False
+2022-12-21,11:30:06 | INFO |   val_data: None
+2022-12-21,11:30:06 | INFO |   val_frequency: 1
+2022-12-21,11:30:06 | INFO |   val_num_samples: None
+2022-12-21,11:30:06 | INFO |   wandb: False
+2022-12-21,11:30:06 | INFO |   wandb_notes: 
+2022-12-21,11:30:06 | INFO |   warmup: 10000
+2022-12-21,11:30:06 | INFO |   wd: 0.2
+2022-12-21,11:30:06 | INFO |   workers: 1
+2022-12-21,11:30:06 | INFO |   world_size: 1
+2022-12-21,11:30:06 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:06 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:06 | INFO | Building zero-shot classifier
+2022-12-21,11:30:24 | INFO | Using classifier
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6789	imagenet-zeroshot-val-top5: 0.9077
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f299d258dc33fd3862a0d009ef87be0f0e6a47b5
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_61.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_61.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_61.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_61.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_61.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..ad820565a8de22f851382b21d3d2260f993e96a9
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.67906, "imagenet-zeroshot-val-top5": 0.90796}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..41e7b6baa1265edfee36b79c747220996ab7a420
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_62.pt).
+2022-12-21,11:30:05 | INFO | Model:
+2022-12-21,11:30:05 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:05 | INFO | Params:
+2022-12-21,11:30:05 | INFO |   batch_size: 256
+2022-12-21,11:30:05 | INFO |   beta1: 0.9
+2022-12-21,11:30:05 | INFO |   beta2: 0.98
+2022-12-21,11:30:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_62.pt/checkpoints
+2022-12-21,11:30:05 | INFO |   copy_codebase: False
+2022-12-21,11:30:05 | INFO |   csv_caption_key: title
+2022-12-21,11:30:05 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:05 | INFO |   csv_separator: 	
+2022-12-21,11:30:05 | INFO |   dataset_resampled: False
+2022-12-21,11:30:05 | INFO |   dataset_type: auto
+2022-12-21,11:30:05 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:05 | INFO |   debug: False
+2022-12-21,11:30:05 | INFO |   device: cuda:0
+2022-12-21,11:30:05 | INFO |   dist_backend: nccl
+2022-12-21,11:30:05 | INFO |   dist_url: env://
+2022-12-21,11:30:05 | INFO |   distributed: False
+2022-12-21,11:30:05 | INFO |   epochs: 32
+2022-12-21,11:30:05 | INFO |   eps: 1e-06
+2022-12-21,11:30:05 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:05 | INFO |   gather_with_grad: False
+2022-12-21,11:30:05 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:05 | INFO |   horovod: False
+2022-12-21,11:30:05 | INFO |   image_mean: None
+2022-12-21,11:30:05 | INFO |   image_std: None
+2022-12-21,11:30:05 | INFO |   imagenet_v2: None
+2022-12-21,11:30:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:05 | INFO |   local_loss: False
+2022-12-21,11:30:05 | INFO |   local_rank: 0
+2022-12-21,11:30:05 | INFO |   lock_image: False
+2022-12-21,11:30:05 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:05 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:05 | INFO |   log_level: 20
+2022-12-21,11:30:05 | INFO |   log_local: False
+2022-12-21,11:30:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_62.pt/out.log
+2022-12-21,11:30:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:05 | INFO |   lr: 0.0005
+2022-12-21,11:30:05 | INFO |   model: ViT-B-16
+2022-12-21,11:30:05 | INFO |   name: epoch_62.pt
+2022-12-21,11:30:05 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:05 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:05 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_62.pt
+2022-12-21,11:30:05 | INFO |   pretrained_image: False
+2022-12-21,11:30:05 | INFO |   rank: 0
+2022-12-21,11:30:05 | INFO |   report_to: 
+2022-12-21,11:30:05 | INFO |   resume: None
+2022-12-21,11:30:05 | INFO |   save_frequency: 1
+2022-12-21,11:30:05 | INFO |   save_most_recent: False
+2022-12-21,11:30:05 | INFO |   seed: 0
+2022-12-21,11:30:05 | INFO |   skip_scheduler: False
+2022-12-21,11:30:05 | INFO |   tensorboard: False
+2022-12-21,11:30:05 | INFO |   tensorboard_path: 
+2022-12-21,11:30:05 | INFO |   torchscript: False
+2022-12-21,11:30:05 | INFO |   trace: False
+2022-12-21,11:30:05 | INFO |   train_data: None
+2022-12-21,11:30:05 | INFO |   train_num_samples: None
+2022-12-21,11:30:05 | INFO |   use_bn_sync: False
+2022-12-21,11:30:05 | INFO |   val_data: None
+2022-12-21,11:30:05 | INFO |   val_frequency: 1
+2022-12-21,11:30:05 | INFO |   val_num_samples: None
+2022-12-21,11:30:05 | INFO |   wandb: False
+2022-12-21,11:30:05 | INFO |   wandb_notes: 
+2022-12-21,11:30:05 | INFO |   warmup: 10000
+2022-12-21,11:30:05 | INFO |   wd: 0.2
+2022-12-21,11:30:05 | INFO |   workers: 1
+2022-12-21,11:30:05 | INFO |   world_size: 1
+2022-12-21,11:30:05 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:06 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:06 | INFO | Building zero-shot classifier
+2022-12-21,11:30:24 | INFO | Using classifier
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6791	imagenet-zeroshot-val-top5: 0.9080
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..35e2cfa819c3022fb3545715cd849786728dbfe3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_62.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_62.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_62.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_62.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_62.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..b3f04ed5e7d8ab42efa06d956f2ec72b7f330cc6
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.68066, "imagenet-zeroshot-val-top5": 0.9083}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..1602ca3afee194250807d443b41c95c17270d841
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:29:56 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:29:56 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:29:57 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_63.pt).
+2022-12-21,11:30:04 | INFO | Model:
+2022-12-21,11:30:04 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:30:04 | INFO | Params:
+2022-12-21,11:30:04 | INFO |   batch_size: 256
+2022-12-21,11:30:04 | INFO |   beta1: 0.9
+2022-12-21,11:30:04 | INFO |   beta2: 0.98
+2022-12-21,11:30:04 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_63.pt/checkpoints
+2022-12-21,11:30:04 | INFO |   copy_codebase: False
+2022-12-21,11:30:04 | INFO |   csv_caption_key: title
+2022-12-21,11:30:04 | INFO |   csv_img_key: filepath
+2022-12-21,11:30:04 | INFO |   csv_separator: 	
+2022-12-21,11:30:04 | INFO |   dataset_resampled: False
+2022-12-21,11:30:04 | INFO |   dataset_type: auto
+2022-12-21,11:30:04 | INFO |   ddp_static_graph: False
+2022-12-21,11:30:04 | INFO |   debug: False
+2022-12-21,11:30:04 | INFO |   device: cuda:0
+2022-12-21,11:30:04 | INFO |   dist_backend: nccl
+2022-12-21,11:30:04 | INFO |   dist_url: env://
+2022-12-21,11:30:04 | INFO |   distributed: False
+2022-12-21,11:30:04 | INFO |   epochs: 32
+2022-12-21,11:30:04 | INFO |   eps: 1e-06
+2022-12-21,11:30:04 | INFO |   force_quick_gelu: False
+2022-12-21,11:30:04 | INFO |   gather_with_grad: False
+2022-12-21,11:30:04 | INFO |   grad_checkpointing: False
+2022-12-21,11:30:04 | INFO |   horovod: False
+2022-12-21,11:30:04 | INFO |   image_mean: None
+2022-12-21,11:30:04 | INFO |   image_std: None
+2022-12-21,11:30:04 | INFO |   imagenet_v2: None
+2022-12-21,11:30:04 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:30:04 | INFO |   local_loss: False
+2022-12-21,11:30:04 | INFO |   local_rank: 0
+2022-12-21,11:30:04 | INFO |   lock_image: False
+2022-12-21,11:30:04 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:30:04 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:30:04 | INFO |   log_level: 20
+2022-12-21,11:30:04 | INFO |   log_local: False
+2022-12-21,11:30:04 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_63.pt/out.log
+2022-12-21,11:30:04 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:30:04 | INFO |   lr: 0.0005
+2022-12-21,11:30:04 | INFO |   model: ViT-B-16
+2022-12-21,11:30:04 | INFO |   name: epoch_63.pt
+2022-12-21,11:30:04 | INFO |   no_set_device_rank: False
+2022-12-21,11:30:04 | INFO |   norm_gradient_clip: None
+2022-12-21,11:30:04 | INFO |   precision: amp_bfloat16
+2022-12-21,11:30:04 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_63.pt
+2022-12-21,11:30:04 | INFO |   pretrained_image: False
+2022-12-21,11:30:04 | INFO |   rank: 0
+2022-12-21,11:30:04 | INFO |   report_to: 
+2022-12-21,11:30:04 | INFO |   resume: None
+2022-12-21,11:30:04 | INFO |   save_frequency: 1
+2022-12-21,11:30:04 | INFO |   save_most_recent: False
+2022-12-21,11:30:04 | INFO |   seed: 0
+2022-12-21,11:30:04 | INFO |   skip_scheduler: False
+2022-12-21,11:30:04 | INFO |   tensorboard: False
+2022-12-21,11:30:04 | INFO |   tensorboard_path: 
+2022-12-21,11:30:04 | INFO |   torchscript: False
+2022-12-21,11:30:04 | INFO |   trace: False
+2022-12-21,11:30:04 | INFO |   train_data: None
+2022-12-21,11:30:04 | INFO |   train_num_samples: None
+2022-12-21,11:30:04 | INFO |   use_bn_sync: False
+2022-12-21,11:30:04 | INFO |   val_data: None
+2022-12-21,11:30:04 | INFO |   val_frequency: 1
+2022-12-21,11:30:04 | INFO |   val_num_samples: None
+2022-12-21,11:30:04 | INFO |   wandb: False
+2022-12-21,11:30:04 | INFO |   wandb_notes: 
+2022-12-21,11:30:04 | INFO |   warmup: 10000
+2022-12-21,11:30:04 | INFO |   wd: 0.2
+2022-12-21,11:30:04 | INFO |   workers: 1
+2022-12-21,11:30:04 | INFO |   world_size: 1
+2022-12-21,11:30:04 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:30:04 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:30:04 | INFO | Building zero-shot classifier
+2022-12-21,11:30:22 | INFO | Using classifier
+2022-12-21,11:38:33 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:38:33 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6807	imagenet-zeroshot-val-top5: 0.9083
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0c5fb196981c61ded051b3d5b497f2af9c764d38
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_63.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_63.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_63.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_63.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_63.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..3765444f46f92117611e49930641fe0d06daf6c3
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.68092, "imagenet-zeroshot-val-top5": 0.9087}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..8e850ca9e0cfd850bb254608ba35d4e1cd9979f1
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_64.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_64.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_64.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_64.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_64.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+2022-12-21,11:39:16 | INFO | Using classifier
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6809	imagenet-zeroshot-val-top5: 0.9087
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..885826c40355b8f039849e8b5f128c112454083d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_64.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_64.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_64.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_64.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_64.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f9696442911bc60b6d4911d817fa671e4126229a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.68124, "imagenet-zeroshot-val-top5": 0.90862}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..09b7708c7c63bc99a61518c55aee66c1d525d3f8
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_65.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_65.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_65.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_65.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_65.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+2022-12-21,11:39:16 | INFO | Using classifier
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6812	imagenet-zeroshot-val-top5: 0.9086
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f6bcc0a7829a49193f725320c007078239c6e478
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_65.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_65.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_65.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_65.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_65.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f2d66f7e68614069292e7031381fab9dfbfeac63
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.5052, "imagenet-zeroshot-val-top5": 0.78206}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..ef3db8b63e45346ee8ccd1b8b9e8a00f15716d02
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_7.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_7.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_7.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_7.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_7.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+2022-12-21,11:39:17 | INFO | Using classifier
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5052	imagenet-zeroshot-val-top5: 0.7821
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..70385acd17c65d7a27a475c90941250548697a99
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_7.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_7.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_7.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_7.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_7.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..2a899437adc03df6a788a888f84457f5d875f007
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.50924, "imagenet-zeroshot-val-top5": 0.79092}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..aedcea04f71bd10a20abf99d32506849f5001802
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:38:48 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:38:48 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:38:49 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_8.pt).
+2022-12-21,11:38:58 | INFO | Model:
+2022-12-21,11:38:58 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:38:58 | INFO | Params:
+2022-12-21,11:38:58 | INFO |   batch_size: 256
+2022-12-21,11:38:58 | INFO |   beta1: 0.9
+2022-12-21,11:38:58 | INFO |   beta2: 0.98
+2022-12-21,11:38:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_8.pt/checkpoints
+2022-12-21,11:38:58 | INFO |   copy_codebase: False
+2022-12-21,11:38:58 | INFO |   csv_caption_key: title
+2022-12-21,11:38:58 | INFO |   csv_img_key: filepath
+2022-12-21,11:38:58 | INFO |   csv_separator: 	
+2022-12-21,11:38:58 | INFO |   dataset_resampled: False
+2022-12-21,11:38:58 | INFO |   dataset_type: auto
+2022-12-21,11:38:58 | INFO |   ddp_static_graph: False
+2022-12-21,11:38:58 | INFO |   debug: False
+2022-12-21,11:38:58 | INFO |   device: cuda:0
+2022-12-21,11:38:58 | INFO |   dist_backend: nccl
+2022-12-21,11:38:58 | INFO |   dist_url: env://
+2022-12-21,11:38:58 | INFO |   distributed: False
+2022-12-21,11:38:58 | INFO |   epochs: 32
+2022-12-21,11:38:58 | INFO |   eps: 1e-06
+2022-12-21,11:38:58 | INFO |   force_quick_gelu: False
+2022-12-21,11:38:58 | INFO |   gather_with_grad: False
+2022-12-21,11:38:58 | INFO |   grad_checkpointing: False
+2022-12-21,11:38:58 | INFO |   horovod: False
+2022-12-21,11:38:58 | INFO |   image_mean: None
+2022-12-21,11:38:58 | INFO |   image_std: None
+2022-12-21,11:38:58 | INFO |   imagenet_v2: None
+2022-12-21,11:38:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:38:58 | INFO |   local_loss: False
+2022-12-21,11:38:58 | INFO |   local_rank: 0
+2022-12-21,11:38:58 | INFO |   lock_image: False
+2022-12-21,11:38:58 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:38:58 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:38:58 | INFO |   log_level: 20
+2022-12-21,11:38:58 | INFO |   log_local: False
+2022-12-21,11:38:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_8.pt/out.log
+2022-12-21,11:38:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:38:58 | INFO |   lr: 0.0005
+2022-12-21,11:38:58 | INFO |   model: ViT-B-16
+2022-12-21,11:38:58 | INFO |   name: epoch_8.pt
+2022-12-21,11:38:58 | INFO |   no_set_device_rank: False
+2022-12-21,11:38:58 | INFO |   norm_gradient_clip: None
+2022-12-21,11:38:58 | INFO |   precision: amp_bfloat16
+2022-12-21,11:38:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_8.pt
+2022-12-21,11:38:58 | INFO |   pretrained_image: False
+2022-12-21,11:38:58 | INFO |   rank: 0
+2022-12-21,11:38:58 | INFO |   report_to: 
+2022-12-21,11:38:58 | INFO |   resume: None
+2022-12-21,11:38:58 | INFO |   save_frequency: 1
+2022-12-21,11:38:58 | INFO |   save_most_recent: False
+2022-12-21,11:38:58 | INFO |   seed: 0
+2022-12-21,11:38:58 | INFO |   skip_scheduler: False
+2022-12-21,11:38:58 | INFO |   tensorboard: False
+2022-12-21,11:38:58 | INFO |   tensorboard_path: 
+2022-12-21,11:38:58 | INFO |   torchscript: False
+2022-12-21,11:38:58 | INFO |   trace: False
+2022-12-21,11:38:58 | INFO |   train_data: None
+2022-12-21,11:38:58 | INFO |   train_num_samples: None
+2022-12-21,11:38:58 | INFO |   use_bn_sync: False
+2022-12-21,11:38:58 | INFO |   val_data: None
+2022-12-21,11:38:58 | INFO |   val_frequency: 1
+2022-12-21,11:38:58 | INFO |   val_num_samples: None
+2022-12-21,11:38:58 | INFO |   wandb: False
+2022-12-21,11:38:58 | INFO |   wandb_notes: 
+2022-12-21,11:38:58 | INFO |   warmup: 10000
+2022-12-21,11:38:58 | INFO |   wd: 0.2
+2022-12-21,11:38:58 | INFO |   workers: 1
+2022-12-21,11:38:58 | INFO |   world_size: 1
+2022-12-21,11:38:58 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:38:58 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:38:58 | INFO | Building zero-shot classifier
+2022-12-21,11:39:16 | INFO | Using classifier
+2022-12-21,11:47:28 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:47:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5092	imagenet-zeroshot-val-top5: 0.7909
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..c258d2a582f6b0f5e164314a39d52e689159376d
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_8.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_8.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_8.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_8.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_8.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..32e014ddc1aa9c9706d04af64e9f90e4485f6b6c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.523, "imagenet-zeroshot-val-top5": 0.80066}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..a01762f2d53289d2754dd16aa38d0962a355d616
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:47:43 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:47:43 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:47:44 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_9.pt).
+2022-12-21,11:47:51 | INFO | Model:
+2022-12-21,11:47:51 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:47:51 | INFO | Params:
+2022-12-21,11:47:51 | INFO |   batch_size: 256
+2022-12-21,11:47:51 | INFO |   beta1: 0.9
+2022-12-21,11:47:51 | INFO |   beta2: 0.98
+2022-12-21,11:47:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_9.pt/checkpoints
+2022-12-21,11:47:51 | INFO |   copy_codebase: False
+2022-12-21,11:47:51 | INFO |   csv_caption_key: title
+2022-12-21,11:47:51 | INFO |   csv_img_key: filepath
+2022-12-21,11:47:51 | INFO |   csv_separator: 	
+2022-12-21,11:47:51 | INFO |   dataset_resampled: False
+2022-12-21,11:47:51 | INFO |   dataset_type: auto
+2022-12-21,11:47:51 | INFO |   ddp_static_graph: False
+2022-12-21,11:47:51 | INFO |   debug: False
+2022-12-21,11:47:51 | INFO |   device: cuda:0
+2022-12-21,11:47:51 | INFO |   dist_backend: nccl
+2022-12-21,11:47:51 | INFO |   dist_url: env://
+2022-12-21,11:47:51 | INFO |   distributed: False
+2022-12-21,11:47:51 | INFO |   epochs: 32
+2022-12-21,11:47:51 | INFO |   eps: 1e-06
+2022-12-21,11:47:51 | INFO |   force_quick_gelu: False
+2022-12-21,11:47:51 | INFO |   gather_with_grad: False
+2022-12-21,11:47:51 | INFO |   grad_checkpointing: False
+2022-12-21,11:47:51 | INFO |   horovod: False
+2022-12-21,11:47:51 | INFO |   image_mean: None
+2022-12-21,11:47:51 | INFO |   image_std: None
+2022-12-21,11:47:51 | INFO |   imagenet_v2: None
+2022-12-21,11:47:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:47:51 | INFO |   local_loss: False
+2022-12-21,11:47:51 | INFO |   local_rank: 0
+2022-12-21,11:47:51 | INFO |   lock_image: False
+2022-12-21,11:47:51 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:47:51 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:47:51 | INFO |   log_level: 20
+2022-12-21,11:47:51 | INFO |   log_local: False
+2022-12-21,11:47:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_9.pt/out.log
+2022-12-21,11:47:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:47:51 | INFO |   lr: 0.0005
+2022-12-21,11:47:51 | INFO |   model: ViT-B-16
+2022-12-21,11:47:51 | INFO |   name: epoch_9.pt
+2022-12-21,11:47:51 | INFO |   no_set_device_rank: False
+2022-12-21,11:47:51 | INFO |   norm_gradient_clip: None
+2022-12-21,11:47:51 | INFO |   precision: amp_bfloat16
+2022-12-21,11:47:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_9.pt
+2022-12-21,11:47:51 | INFO |   pretrained_image: False
+2022-12-21,11:47:51 | INFO |   rank: 0
+2022-12-21,11:47:51 | INFO |   report_to: 
+2022-12-21,11:47:51 | INFO |   resume: None
+2022-12-21,11:47:51 | INFO |   save_frequency: 1
+2022-12-21,11:47:51 | INFO |   save_most_recent: False
+2022-12-21,11:47:51 | INFO |   seed: 0
+2022-12-21,11:47:51 | INFO |   skip_scheduler: False
+2022-12-21,11:47:51 | INFO |   tensorboard: False
+2022-12-21,11:47:51 | INFO |   tensorboard_path: 
+2022-12-21,11:47:51 | INFO |   torchscript: False
+2022-12-21,11:47:51 | INFO |   trace: False
+2022-12-21,11:47:51 | INFO |   train_data: None
+2022-12-21,11:47:51 | INFO |   train_num_samples: None
+2022-12-21,11:47:51 | INFO |   use_bn_sync: False
+2022-12-21,11:47:51 | INFO |   val_data: None
+2022-12-21,11:47:51 | INFO |   val_frequency: 1
+2022-12-21,11:47:51 | INFO |   val_num_samples: None
+2022-12-21,11:47:51 | INFO |   wandb: False
+2022-12-21,11:47:51 | INFO |   wandb_notes: 
+2022-12-21,11:47:51 | INFO |   warmup: 10000
+2022-12-21,11:47:51 | INFO |   wd: 0.2
+2022-12-21,11:47:51 | INFO |   workers: 1
+2022-12-21,11:47:51 | INFO |   world_size: 1
+2022-12-21,11:47:51 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:47:52 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:47:52 | INFO | Building zero-shot classifier
+2022-12-21,11:48:09 | INFO | Using classifier
+2022-12-21,11:56:18 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:56:18 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5230	imagenet-zeroshot-val-top5: 0.8007
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..76dd9990de3ab61d580a78f5c675681215a06f9e
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_9.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_9.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_9.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_9.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_9.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/checkpoints/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/checkpoints/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..f9696442911bc60b6d4911d817fa671e4126229a
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/checkpoints/results.jsonl
@@ -0,0 +1 @@
+{"imagenet-zeroshot-val-top1": 0.68124, "imagenet-zeroshot-val-top5": 0.90862}
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/out.log b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/out.log
new file mode 100644
index 0000000000000000000000000000000000000000..aa878b82ea1ce25f7d96798e7597c6a3df8a5316
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/out.log
@@ -0,0 +1,430 @@
+2022-12-21,11:47:43 | INFO | Running with a single process. Device cuda:0.
+2022-12-21,11:47:43 | INFO | Loading ViT-B-16 model config.
+2022-12-21,11:47:44 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_latest.pt).
+2022-12-21,11:47:52 | INFO | Model:
+2022-12-21,11:47:52 | INFO | CLIP(
+  (visual): VisualTransformer(
+    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ln_attn): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (ln): Identity()
+            (gelu): GELU(approximate=none)
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ln_attn): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (ln): Identity()
+          (gelu): GELU(approximate=none)
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2022-12-21,11:47:52 | INFO | Params:
+2022-12-21,11:47:52 | INFO |   batch_size: 256
+2022-12-21,11:47:52 | INFO |   beta1: 0.9
+2022-12-21,11:47:52 | INFO |   beta2: 0.98
+2022-12-21,11:47:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_latest.pt/checkpoints
+2022-12-21,11:47:52 | INFO |   copy_codebase: False
+2022-12-21,11:47:52 | INFO |   csv_caption_key: title
+2022-12-21,11:47:52 | INFO |   csv_img_key: filepath
+2022-12-21,11:47:52 | INFO |   csv_separator: 	
+2022-12-21,11:47:52 | INFO |   dataset_resampled: False
+2022-12-21,11:47:52 | INFO |   dataset_type: auto
+2022-12-21,11:47:52 | INFO |   ddp_static_graph: False
+2022-12-21,11:47:52 | INFO |   debug: False
+2022-12-21,11:47:52 | INFO |   device: cuda:0
+2022-12-21,11:47:52 | INFO |   dist_backend: nccl
+2022-12-21,11:47:52 | INFO |   dist_url: env://
+2022-12-21,11:47:52 | INFO |   distributed: False
+2022-12-21,11:47:52 | INFO |   epochs: 32
+2022-12-21,11:47:52 | INFO |   eps: 1e-06
+2022-12-21,11:47:52 | INFO |   force_quick_gelu: False
+2022-12-21,11:47:52 | INFO |   gather_with_grad: False
+2022-12-21,11:47:52 | INFO |   grad_checkpointing: False
+2022-12-21,11:47:52 | INFO |   horovod: False
+2022-12-21,11:47:52 | INFO |   image_mean: None
+2022-12-21,11:47:52 | INFO |   image_std: None
+2022-12-21,11:47:52 | INFO |   imagenet_v2: None
+2022-12-21,11:47:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
+2022-12-21,11:47:52 | INFO |   local_loss: False
+2022-12-21,11:47:52 | INFO |   local_rank: 0
+2022-12-21,11:47:52 | INFO |   lock_image: False
+2022-12-21,11:47:52 | INFO |   lock_image_freeze_bn_stats: False
+2022-12-21,11:47:52 | INFO |   lock_image_unlocked_groups: 0
+2022-12-21,11:47:52 | INFO |   log_level: 20
+2022-12-21,11:47:52 | INFO |   log_local: False
+2022-12-21,11:47:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_latest.pt/out.log
+2022-12-21,11:47:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+2022-12-21,11:47:52 | INFO |   lr: 0.0005
+2022-12-21,11:47:52 | INFO |   model: ViT-B-16
+2022-12-21,11:47:52 | INFO |   name: epoch_latest.pt
+2022-12-21,11:47:52 | INFO |   no_set_device_rank: False
+2022-12-21,11:47:52 | INFO |   norm_gradient_clip: None
+2022-12-21,11:47:52 | INFO |   precision: amp_bfloat16
+2022-12-21,11:47:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_latest.pt
+2022-12-21,11:47:52 | INFO |   pretrained_image: False
+2022-12-21,11:47:52 | INFO |   rank: 0
+2022-12-21,11:47:52 | INFO |   report_to: 
+2022-12-21,11:47:52 | INFO |   resume: None
+2022-12-21,11:47:52 | INFO |   save_frequency: 1
+2022-12-21,11:47:52 | INFO |   save_most_recent: False
+2022-12-21,11:47:52 | INFO |   seed: 0
+2022-12-21,11:47:52 | INFO |   skip_scheduler: False
+2022-12-21,11:47:52 | INFO |   tensorboard: False
+2022-12-21,11:47:52 | INFO |   tensorboard_path: 
+2022-12-21,11:47:52 | INFO |   torchscript: False
+2022-12-21,11:47:52 | INFO |   trace: False
+2022-12-21,11:47:52 | INFO |   train_data: None
+2022-12-21,11:47:52 | INFO |   train_num_samples: None
+2022-12-21,11:47:52 | INFO |   use_bn_sync: False
+2022-12-21,11:47:52 | INFO |   val_data: None
+2022-12-21,11:47:52 | INFO |   val_frequency: 1
+2022-12-21,11:47:52 | INFO |   val_num_samples: None
+2022-12-21,11:47:52 | INFO |   wandb: False
+2022-12-21,11:47:52 | INFO |   wandb_notes: 
+2022-12-21,11:47:52 | INFO |   warmup: 10000
+2022-12-21,11:47:52 | INFO |   wd: 0.2
+2022-12-21,11:47:52 | INFO |   workers: 1
+2022-12-21,11:47:52 | INFO |   world_size: 1
+2022-12-21,11:47:52 | INFO |   zeroshot_frequency: 2
+2022-12-21,11:47:52 | INFO | Starting zero-shot imagenet.
+2022-12-21,11:47:52 | INFO | Building zero-shot classifier
+2022-12-21,11:48:09 | INFO | Using classifier
+2022-12-21,11:56:18 | INFO | Finished zero-shot imagenet.
+2022-12-21,11:56:18 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6812	imagenet-zeroshot-val-top5: 0.9086
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/params.txt b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/params.txt
new file mode 100644
index 0000000000000000000000000000000000000000..b37d9d80eb08cdfaa5c7aaf1daf26680f340c761
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/eval/epoch_latest.pt/params.txt
@@ -0,0 +1,67 @@
+batch_size: 256
+beta1: 0.9
+beta2: 0.98
+checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_latest.pt/checkpoints
+copy_codebase: False
+csv_caption_key: title
+csv_img_key: filepath
+csv_separator: 	
+dataset_resampled: False
+dataset_type: auto
+ddp_static_graph: False
+debug: False
+device: cuda:0
+dist_backend: nccl
+dist_url: env://
+distributed: False
+epochs: 32
+eps: 1e-06
+force_quick_gelu: False
+gather_with_grad: False
+grad_checkpointing: False
+horovod: False
+image_mean: None
+image_std: None
+imagenet_v2: None
+imagenet_val: /p/fastdata/mmlaion/imagenet_val
+local_loss: False
+local_rank: 0
+lock_image: False
+lock_image_freeze_bn_stats: False
+lock_image_unlocked_groups: 0
+log_level: 20
+log_local: False
+log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_latest.pt/out.log
+logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
+lr: 0.0005
+model: ViT-B-16
+name: epoch_latest.pt
+no_set_device_rank: False
+norm_gradient_clip: None
+precision: amp_bfloat16
+pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_latest.pt
+pretrained_image: False
+rank: 0
+report_to: 
+resume: None
+save_frequency: 1
+save_most_recent: False
+seed: 0
+skip_scheduler: False
+tensorboard: False
+tensorboard_path: 
+torchscript: False
+trace: False
+train_data: None
+train_num_samples: None
+use_bn_sync: False
+val_data: None
+val_frequency: 1
+val_num_samples: None
+wandb: False
+wandb_notes: 
+warmup: 10000
+wd: 0.2
+workers: 1
+world_size: 1
+zeroshot_frequency: 2
diff --git a/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/results.jsonl b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/results.jsonl
new file mode 100644
index 0000000000000000000000000000000000000000..6946e69bd789b79fb31fd0067117b9637732673c
--- /dev/null
+++ b/full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/results.jsonl
@@ -0,0 +1,44 @@
+{"imagenet-zeroshot-val-top1": 0.08232, "imagenet-zeroshot-val-top5": 0.21436}
+{"imagenet-zeroshot-val-top1": 0.26652, "imagenet-zeroshot-val-top5": 0.51302}
+{"imagenet-zeroshot-val-top1": 0.35996, "imagenet-zeroshot-val-top5": 0.63174}
+{"imagenet-zeroshot-val-top1": 0.42486, "imagenet-zeroshot-val-top5": 0.7045}
+{"imagenet-zeroshot-val-top1": 0.46674, "imagenet-zeroshot-val-top5": 0.7494}
+{"imagenet-zeroshot-val-top1": 0.48758, "imagenet-zeroshot-val-top5": 0.76886}
+{"imagenet-zeroshot-val-top1": 0.50508, "imagenet-zeroshot-val-top5": 0.78232}
+{"imagenet-zeroshot-val-top1": 0.50946, "imagenet-zeroshot-val-top5": 0.79098}
+{"imagenet-zeroshot-val-top1": 0.52318, "imagenet-zeroshot-val-top5": 0.80078}
+{"imagenet-zeroshot-val-top1": 0.5272, "imagenet-zeroshot-val-top5": 0.80772}
+{"imagenet-zeroshot-val-top1": 0.53198, "imagenet-zeroshot-val-top5": 0.81268}
+{"imagenet-zeroshot-val-top1": 0.53694, "imagenet-zeroshot-val-top5": 0.81524}
+{"imagenet-zeroshot-val-top1": 0.54496, "imagenet-zeroshot-val-top5": 0.81924}
+{"imagenet-zeroshot-val-top1": 0.54248, "imagenet-zeroshot-val-top5": 0.81862}
+{"imagenet-zeroshot-val-top1": 0.55102, "imagenet-zeroshot-val-top5": 0.8245}
+{"imagenet-zeroshot-val-top1": 0.55342, "imagenet-zeroshot-val-top5": 0.82582}
+{"imagenet-zeroshot-val-top1": 0.55396, "imagenet-zeroshot-val-top5": 0.83012}
+{"imagenet-zeroshot-val-top1": 0.55272, "imagenet-zeroshot-val-top5": 0.8279}
+{"imagenet-zeroshot-val-top1": 0.561, "imagenet-zeroshot-val-top5": 0.83132}
+{"imagenet-zeroshot-val-top1": 0.56468, "imagenet-zeroshot-val-top5": 0.83558}
+{"imagenet-zeroshot-val-top1": 0.57184, "imagenet-zeroshot-val-top5": 0.8408}
+{"imagenet-zeroshot-val-top1": 0.56496, "imagenet-zeroshot-val-top5": 0.83762}
+{"imagenet-zeroshot-val-top1": 0.57334, "imagenet-zeroshot-val-top5": 0.8429}
+{"imagenet-zeroshot-val-top1": 0.573, "imagenet-zeroshot-val-top5": 0.842}
+{"imagenet-zeroshot-val-top1": 0.57316, "imagenet-zeroshot-val-top5": 0.84144}
+{"imagenet-zeroshot-val-top1": 0.57484, "imagenet-zeroshot-val-top5": 0.84542}
+{"imagenet-zeroshot-val-top1": 0.58182, "imagenet-zeroshot-val-top5": 0.8477}
+{"imagenet-zeroshot-val-top1": 0.58422, "imagenet-zeroshot-val-top5": 0.8518}
+{"imagenet-zeroshot-val-top1": 0.57994, "imagenet-zeroshot-val-top5": 0.84982}
+{"imagenet-zeroshot-val-top1": 0.54614, "imagenet-zeroshot-val-top5": 0.82272}
+{"imagenet-zeroshot-val-top1": 0.58818, "imagenet-zeroshot-val-top5": 0.85536}
+{"imagenet-zeroshot-val-top1": 0.59034, "imagenet-zeroshot-val-top5": 0.85848}
+{"imagenet-zeroshot-val-top1": 0.5896, "imagenet-zeroshot-val-top5": 0.85828}
+{"imagenet-zeroshot-val-top1": 0.59306, "imagenet-zeroshot-val-top5": 0.8605}
+{"imagenet-zeroshot-val-top1": 0.59982, "imagenet-zeroshot-val-top5": 0.86348}
+{"imagenet-zeroshot-val-top1": 0.60508, "imagenet-zeroshot-val-top5": 0.86444}
+{"imagenet-zeroshot-val-top1": 0.60316, "imagenet-zeroshot-val-top5": 0.86616}
+{"imagenet-zeroshot-val-top1": 0.6095, "imagenet-zeroshot-val-top5": 0.86964}
+{"imagenet-zeroshot-val-top1": 0.6118, "imagenet-zeroshot-val-top5": 0.86914}
+{"imagenet-zeroshot-val-top1": 0.61056, "imagenet-zeroshot-val-top5": 0.86936}
+{"imagenet-zeroshot-val-top1": 0.61742, "imagenet-zeroshot-val-top5": 0.8743}
+{"imagenet-zeroshot-val-top1": 0.61684, "imagenet-zeroshot-val-top5": 0.87362}
+{"imagenet-zeroshot-val-top1": 0.62204, "imagenet-zeroshot-val-top5": 0.87614}
+{"imagenet-zeroshot-val-top1": 0.62578, "imagenet-zeroshot-val-top5": 0.8788}