remove txt files

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_1.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt).
-2022-12-21,09:36:52 | INFO | Model:
-2022-12-21,09:36:52 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:36:52 | INFO | Params:
-2022-12-21,09:36:52 | INFO |   batch_size: 256
-2022-12-21,09:36:52 | INFO |   beta1: 0.9
-2022-12-21,09:36:52 | INFO |   beta2: 0.98
-2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/checkpoints
-2022-12-21,09:36:52 | INFO |   copy_codebase: False
-2022-12-21,09:36:52 | INFO |   csv_caption_key: title
-2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
-2022-12-21,09:36:52 | INFO |   csv_separator: 	
-2022-12-21,09:36:52 | INFO |   dataset_resampled: False
-2022-12-21,09:36:52 | INFO |   dataset_type: auto
-2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
-2022-12-21,09:36:52 | INFO |   debug: False
-2022-12-21,09:36:52 | INFO |   device: cuda:0
-2022-12-21,09:36:52 | INFO |   dist_backend: nccl
-2022-12-21,09:36:52 | INFO |   dist_url: env://
-2022-12-21,09:36:52 | INFO |   distributed: False
-2022-12-21,09:36:52 | INFO |   epochs: 32
-2022-12-21,09:36:52 | INFO |   eps: 1e-06
-2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
-2022-12-21,09:36:52 | INFO |   gather_with_grad: False
-2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
-2022-12-21,09:36:52 | INFO |   horovod: False
-2022-12-21,09:36:52 | INFO |   image_mean: None
-2022-12-21,09:36:52 | INFO |   image_std: None
-2022-12-21,09:36:52 | INFO |   imagenet_v2: None
-2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:36:52 | INFO |   local_loss: False
-2022-12-21,09:36:52 | INFO |   local_rank: 0
-2022-12-21,09:36:52 | INFO |   lock_image: False
-2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:36:52 | INFO |   log_level: 20
-2022-12-21,09:36:52 | INFO |   log_local: False
-2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_1.pt/out.log
-2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:36:52 | INFO |   lr: 0.0005
-2022-12-21,09:36:52 | INFO |   model: ViT-B-16
-2022-12-21,09:36:52 | INFO |   name: epoch_1.pt
-2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
-2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
-2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
-2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_1.pt
-2022-12-21,09:36:52 | INFO |   pretrained_image: False
-2022-12-21,09:36:52 | INFO |   rank: 0
-2022-12-21,09:36:52 | INFO |   report_to: 
-2022-12-21,09:36:52 | INFO |   resume: None
-2022-12-21,09:36:52 | INFO |   save_frequency: 1
-2022-12-21,09:36:52 | INFO |   save_most_recent: False
-2022-12-21,09:36:52 | INFO |   seed: 0
-2022-12-21,09:36:52 | INFO |   skip_scheduler: False
-2022-12-21,09:36:52 | INFO |   tensorboard: False
-2022-12-21,09:36:52 | INFO |   tensorboard_path: 
-2022-12-21,09:36:52 | INFO |   torchscript: False
-2022-12-21,09:36:52 | INFO |   trace: False
-2022-12-21,09:36:52 | INFO |   train_data: None
-2022-12-21,09:36:52 | INFO |   train_num_samples: None
-2022-12-21,09:36:52 | INFO |   use_bn_sync: False
-2022-12-21,09:36:52 | INFO |   val_data: None
-2022-12-21,09:36:52 | INFO |   val_frequency: 1
-2022-12-21,09:36:52 | INFO |   val_num_samples: None
-2022-12-21,09:36:52 | INFO |   wandb: False
-2022-12-21,09:36:52 | INFO |   wandb_notes: 
-2022-12-21,09:36:52 | INFO |   warmup: 10000
-2022-12-21,09:36:52 | INFO |   wd: 0.2
-2022-12-21,09:36:52 | INFO |   workers: 1
-2022-12-21,09:36:52 | INFO |   world_size: 1
-2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:36:53 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:15,  1.26it/s]
  1%|          | 6/1000 [00:00<01:54,  8.65it/s]
  1%|          | 12/1000 [00:01<00:57, 17.24it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.49it/s]
  2%|▏         | 24/1000 [00:01<00:29, 32.73it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.77it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.60it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.29it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.12it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.20it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.68it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.80it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.51it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.06it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.49it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.85it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.28it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.48it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.79it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.91it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.94it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.92it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.94it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.98it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.69it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.69it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.66it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.59it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.57it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.52it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.36it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.26it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.37it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.32it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.55it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.52it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.44it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.50it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.60it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.64it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.70it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.49it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.56it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.76it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.86it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.06it/s]
 30%|███       | 300/1000 [00:06<00:12, 58.03it/s]
 31%|███       | 306/1000 [00:06<00:11, 57.99it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.06it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.89it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.97it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.07it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.15it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.96it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.02it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.05it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.97it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.63it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.63it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.75it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.91it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.97it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.02it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.03it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.17it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.19it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.99it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.06it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.08it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.17it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.09it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.06it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.07it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.98it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.03it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.11it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.14it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.17it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.06it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.01it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.08it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.98it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.93it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.05it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.04it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.96it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.99it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.99it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.08it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.17it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.00it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.95it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.12it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.06it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.15it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.18it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.10it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.12it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.13it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.16it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.04it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.15it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.10it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.07it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.12it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.10it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.26it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.09it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.19it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.14it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.01it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.07it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.91it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.93it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.99it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.97it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.96it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 58.00it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.10it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.22it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.28it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.27it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.21it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.22it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.16it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.22it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.07it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.14it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.12it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.14it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.23it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.27it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.28it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.26it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.34it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.41it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.33it/s]
-2022-12-21,09:37:11 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:46, 107.03it/s]
  1%|          | 512/50176 [00:04<07:16, 113.68it/s]
  2%|▏         | 768/50176 [00:06<07:28, 110.04it/s]
  2%|▏         | 1024/50176 [00:09<08:01, 102.02it/s]
  3%|▎         | 1280/50176 [00:12<08:09, 99.84it/s] 
  3%|▎         | 1536/50176 [00:15<08:59, 90.20it/s]
  4%|▎         | 1792/50176 [00:18<08:41, 92.86it/s]
  4%|▍         | 2048/50176 [00:21<08:32, 93.89it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.58it/s]
  5%|▌         | 2560/50176 [00:26<08:08, 97.50it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.22it/s]
  6%|▌         | 3072/50176 [00:31<08:17, 94.75it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.91it/s]
  7%|▋         | 3584/50176 [00:36<07:42, 100.65it/s]
  8%|▊         | 3840/50176 [00:39<07:59, 96.73it/s] 
  8%|▊         | 4096/50176 [00:42<08:11, 93.71it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.12it/s]
  9%|▉         | 4608/50176 [00:47<07:36, 99.73it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.93it/s]
 10%|█         | 5120/50176 [00:52<07:20, 102.37it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.30it/s]
 11%|█         | 5632/50176 [00:57<07:16, 101.96it/s]
 12%|█▏        | 5888/50176 [00:59<07:14, 101.87it/s]
 12%|█▏        | 6144/50176 [01:02<07:43, 94.98it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.97it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.36it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.60it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.54it/s]
 15%|█▍        | 7424/50176 [01:15<07:08, 99.67it/s]
 15%|█▌        | 7680/50176 [01:18<07:05, 99.84it/s]
 16%|█▌        | 7936/50176 [01:20<06:49, 103.28it/s]
 16%|█▋        | 8192/50176 [01:22<06:48, 102.90it/s]
 17%|█▋        | 8448/50176 [01:25<06:49, 101.90it/s]
 17%|█▋        | 8704/50176 [01:27<06:48, 101.40it/s]
 18%|█▊        | 8960/50176 [01:30<06:47, 101.09it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.73it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.96it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.24it/s]
 20%|█▉        | 9984/50176 [01:40<06:46, 98.76it/s] 
 20%|██        | 10240/50176 [01:43<06:31, 102.07it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.12it/s]
 21%|██▏       | 10752/50176 [01:48<06:28, 101.39it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.16it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.68it/s]
 23%|██▎       | 11520/50176 [01:55<06:08, 104.82it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.61it/s]
 24%|██▍       | 12032/50176 [02:00<06:02, 105.36it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.68it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.83it/s]
 26%|██▌       | 12800/50176 [02:07<05:51, 106.48it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.22it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.63it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.13it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 99.98it/s] 
 28%|██▊       | 14080/50176 [02:19<05:58, 100.62it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.46it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.42it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.97it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.36it/s]
 31%|███       | 15360/50176 [02:32<05:49, 99.67it/s]
 31%|███       | 15616/50176 [02:35<05:37, 102.52it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.79it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.86it/s]
 33%|███▎      | 16384/50176 [02:42<05:27, 103.28it/s]
 33%|███▎      | 16640/50176 [02:45<05:40, 98.35it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.60it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 103.95it/s]
 35%|███▍      | 17408/50176 [02:52<05:18, 103.03it/s]
 35%|███▌      | 17664/50176 [02:55<05:08, 105.39it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.16it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.38it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.62it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.39it/s]
 38%|███▊      | 18944/50176 [03:07<05:02, 103.10it/s]
 38%|███▊      | 19200/50176 [03:10<05:10, 99.68it/s] 
 39%|███▉      | 19456/50176 [03:12<05:13, 97.90it/s]
 39%|███▉      | 19712/50176 [03:15<05:10, 98.01it/s]
 40%|███▉      | 19968/50176 [03:18<05:08, 97.92it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.82it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.60it/s]
 41%|████▏     | 20736/50176 [03:24<04:38, 105.84it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.23it/s]
 42%|████▏     | 21248/50176 [03:30<04:43, 102.11it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.43it/s]
 43%|████▎     | 21760/50176 [03:35<04:36, 102.72it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.58it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 107.77it/s]
 45%|████▍     | 22528/50176 [03:41<04:16, 107.99it/s]
 45%|████▌     | 22784/50176 [03:44<04:24, 103.40it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.65it/s]
 46%|████▋     | 23296/50176 [03:49<04:27, 100.46it/s]
 47%|████▋     | 23552/50176 [03:52<04:32, 97.85it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.78it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.90it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.56it/s]
 49%|████▉     | 24576/50176 [04:02<04:15, 100.33it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.13it/s]
 50%|█████     | 25088/50176 [04:07<03:59, 104.57it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.77it/s]
 51%|█████     | 25600/50176 [04:12<03:54, 104.79it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.61it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.67it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.03it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.37it/s]
 54%|█████▎    | 26880/50176 [04:24<03:40, 105.68it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 105.03it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.43it/s]
 55%|█████▌    | 27648/50176 [04:30<03:22, 111.09it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.44it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.26it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.24it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 101.98it/s]
 58%|█████▊    | 28928/50176 [04:43<03:24, 103.77it/s]
 58%|█████▊    | 29184/50176 [04:46<03:21, 104.10it/s]
 59%|█████▊    | 29440/50176 [04:49<03:30, 98.50it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:29, 97.58it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.10it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.87it/s]
 61%|██████    | 30464/50176 [04:59<03:09, 104.03it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.27it/s]
 62%|██████▏   | 30976/50176 [05:04<03:06, 102.77it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.54it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.51it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.22it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 106.94it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 106.92it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.50it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.36it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.16it/s]
 66%|██████▋   | 33280/50176 [05:26<02:42, 103.72it/s]
 67%|██████▋   | 33536/50176 [05:28<02:40, 103.43it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.58it/s]
 68%|██████▊   | 34048/50176 [05:33<02:28, 108.59it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.07it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.08it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.39it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.88it/s]
 70%|███████   | 35328/50176 [05:50<03:01, 81.62it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.67it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.84it/s]
 72%|███████▏  | 36096/50176 [06:02<03:30, 66.90it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.70it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.88it/s]
 73%|███████▎  | 36864/50176 [06:14<03:16, 67.61it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.68it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.07it/s]
 75%|███████▌  | 37632/50176 [06:20<02:17, 91.55it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.23it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.90it/s]
 77%|███████▋  | 38400/50176 [06:35<03:19, 59.08it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 67.99it/s]
 78%|███████▊  | 38912/50176 [06:42<03:00, 62.24it/s]
 78%|███████▊  | 39168/50176 [06:45<02:38, 69.40it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.43it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.91it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.15it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.53it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.08it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.95it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.74it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.98it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.63it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.10it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.64it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.64it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.54it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.84it/s]
 86%|████████▌ | 43264/50176 [07:22<01:04, 107.85it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.15it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.41it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.48it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.56it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.37it/s]
 89%|████████▉ | 44800/50176 [07:38<00:52, 102.10it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.23it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 100.96it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 103.99it/s]
 91%|█████████▏| 45824/50176 [07:47<00:41, 104.14it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.59it/s]
 92%|█████████▏| 46336/50176 [07:53<00:37, 102.42it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.61it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.79it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.44it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.03it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.62it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.91it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.62it/s]
 96%|█████████▋| 48384/50176 [08:13<00:17, 103.25it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.85it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.56it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.93it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.30it/s]
 99%|█████████▉| 49664/50176 [08:25<00:05, 100.05it/s]
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.69it/s] 
-2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.0824	imagenet-zeroshot-val-top5: 0.2147

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_10.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt).
-2022-12-21,09:36:52 | INFO | Model:
-2022-12-21,09:36:52 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:36:52 | INFO | Params:
-2022-12-21,09:36:52 | INFO |   batch_size: 256
-2022-12-21,09:36:52 | INFO |   beta1: 0.9
-2022-12-21,09:36:52 | INFO |   beta2: 0.98
-2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/checkpoints
-2022-12-21,09:36:52 | INFO |   copy_codebase: False
-2022-12-21,09:36:52 | INFO |   csv_caption_key: title
-2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
-2022-12-21,09:36:52 | INFO |   csv_separator: 	
-2022-12-21,09:36:52 | INFO |   dataset_resampled: False
-2022-12-21,09:36:52 | INFO |   dataset_type: auto
-2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
-2022-12-21,09:36:52 | INFO |   debug: False
-2022-12-21,09:36:52 | INFO |   device: cuda:0
-2022-12-21,09:36:52 | INFO |   dist_backend: nccl
-2022-12-21,09:36:52 | INFO |   dist_url: env://
-2022-12-21,09:36:52 | INFO |   distributed: False
-2022-12-21,09:36:52 | INFO |   epochs: 32
-2022-12-21,09:36:52 | INFO |   eps: 1e-06
-2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
-2022-12-21,09:36:52 | INFO |   gather_with_grad: False
-2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
-2022-12-21,09:36:52 | INFO |   horovod: False
-2022-12-21,09:36:52 | INFO |   image_mean: None
-2022-12-21,09:36:52 | INFO |   image_std: None
-2022-12-21,09:36:52 | INFO |   imagenet_v2: None
-2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:36:52 | INFO |   local_loss: False
-2022-12-21,09:36:52 | INFO |   local_rank: 0
-2022-12-21,09:36:52 | INFO |   lock_image: False
-2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:36:52 | INFO |   log_level: 20
-2022-12-21,09:36:52 | INFO |   log_local: False
-2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_10.pt/out.log
-2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:36:52 | INFO |   lr: 0.0005
-2022-12-21,09:36:52 | INFO |   model: ViT-B-16
-2022-12-21,09:36:52 | INFO |   name: epoch_10.pt
-2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
-2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
-2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
-2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_10.pt
-2022-12-21,09:36:52 | INFO |   pretrained_image: False
-2022-12-21,09:36:52 | INFO |   rank: 0
-2022-12-21,09:36:52 | INFO |   report_to: 
-2022-12-21,09:36:52 | INFO |   resume: None
-2022-12-21,09:36:52 | INFO |   save_frequency: 1
-2022-12-21,09:36:52 | INFO |   save_most_recent: False
-2022-12-21,09:36:52 | INFO |   seed: 0
-2022-12-21,09:36:52 | INFO |   skip_scheduler: False
-2022-12-21,09:36:52 | INFO |   tensorboard: False
-2022-12-21,09:36:52 | INFO |   tensorboard_path: 
-2022-12-21,09:36:52 | INFO |   torchscript: False
-2022-12-21,09:36:52 | INFO |   trace: False
-2022-12-21,09:36:52 | INFO |   train_data: None
-2022-12-21,09:36:52 | INFO |   train_num_samples: None
-2022-12-21,09:36:52 | INFO |   use_bn_sync: False
-2022-12-21,09:36:52 | INFO |   val_data: None
-2022-12-21,09:36:52 | INFO |   val_frequency: 1
-2022-12-21,09:36:52 | INFO |   val_num_samples: None
-2022-12-21,09:36:52 | INFO |   wandb: False
-2022-12-21,09:36:52 | INFO |   wandb_notes: 
-2022-12-21,09:36:52 | INFO |   warmup: 10000
-2022-12-21,09:36:52 | INFO |   wd: 0.2
-2022-12-21,09:36:52 | INFO |   workers: 1
-2022-12-21,09:36:52 | INFO |   world_size: 1
-2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:36:53 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:09,  1.27it/s]
  1%|          | 6/1000 [00:00<01:54,  8.68it/s]
  1%|          | 12/1000 [00:01<00:57, 17.30it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.22it/s]
  2%|▏         | 24/1000 [00:01<00:30, 32.51it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.66it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.48it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.26it/s]
  5%|▍         | 48/1000 [00:01<00:19, 50.08it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.18it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.72it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.87it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.66it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.20it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.79it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.14it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.56it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.75it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.97it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.12it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.04it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.97it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.05it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.03it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.04it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.15it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.04it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.89it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.89it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.79it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.74it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.68it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.62it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.39it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.28it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.39it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.61it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.61it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.52it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.57it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.68it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.79it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.82it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.65it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.87it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.96it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.18it/s]
 30%|███       | 300/1000 [00:06<00:12, 58.10it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.08it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.21it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.15it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.02it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.14it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.26it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.04it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 58.00it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.09it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.20it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.09it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.77it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.84it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.94it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.07it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.16it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.14it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.17it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.21it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.11it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.16it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.98it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.13it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.24it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.15it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.13it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.04it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.11it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.01it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.10it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.24it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.16it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.26it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.26it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.16it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.17it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.22it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.10it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.04it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.19it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.96it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.91it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.00it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.18it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.20it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.30it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.13it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.09it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.25it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.20it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.27it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.30it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.28it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.15it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.26it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.24it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.28it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.25it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.27it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.26it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.33it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.40it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.23it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.30it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.23it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.28it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.28it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.01it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.08it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.06it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.11it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.23it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.39it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.42it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.34it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.41it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.37it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.39it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.28it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.33it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.18it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.28it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.28it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.23it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.26it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.21it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.32it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.32it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.33it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.36it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.37it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.46it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.42it/s]
-2022-12-21,09:37:11 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:54, 105.19it/s]
  1%|          | 512/50176 [00:04<07:18, 113.27it/s]
  2%|▏         | 768/50176 [00:07<07:31, 109.44it/s]
  2%|▏         | 1024/50176 [00:09<08:01, 102.01it/s]
  3%|▎         | 1280/50176 [00:12<08:10, 99.66it/s] 
  3%|▎         | 1536/50176 [00:15<09:00, 90.03it/s]
  4%|▎         | 1792/50176 [00:18<08:40, 93.03it/s]
  4%|▍         | 2048/50176 [00:21<08:33, 93.69it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.68it/s]
  5%|▌         | 2560/50176 [00:26<08:08, 97.42it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.18it/s]
  6%|▌         | 3072/50176 [00:31<08:17, 94.70it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.84it/s]
  7%|▋         | 3584/50176 [00:36<07:42, 100.64it/s]
  8%|▊         | 3840/50176 [00:39<07:58, 96.79it/s] 
  8%|▊         | 4096/50176 [00:42<08:12, 93.59it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.21it/s]
  9%|▉         | 4608/50176 [00:47<07:37, 99.64it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.84it/s]
 10%|█         | 5120/50176 [00:52<07:20, 102.34it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.30it/s]
 11%|█         | 5632/50176 [00:57<07:16, 102.02it/s]
 12%|█▏        | 5888/50176 [00:59<07:14, 101.89it/s]
 12%|█▏        | 6144/50176 [01:02<07:43, 94.91it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.96it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.35it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.65it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.52it/s]
 15%|█▍        | 7424/50176 [01:15<07:08, 99.68it/s]
 15%|█▌        | 7680/50176 [01:18<07:05, 99.81it/s]
 16%|█▌        | 7936/50176 [01:20<06:49, 103.18it/s]
 16%|█▋        | 8192/50176 [01:22<06:48, 102.89it/s]
 17%|█▋        | 8448/50176 [01:25<06:48, 102.12it/s]
 17%|█▋        | 8704/50176 [01:28<06:48, 101.50it/s]
 18%|█▊        | 8960/50176 [01:30<06:47, 101.03it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.72it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.97it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.26it/s]
 20%|█▉        | 9984/50176 [01:40<06:46, 98.87it/s] 
 20%|██        | 10240/50176 [01:43<06:31, 102.06it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.09it/s]
 21%|██▏       | 10752/50176 [01:48<06:28, 101.36it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.08it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.76it/s]
 23%|██▎       | 11520/50176 [01:55<06:08, 104.94it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.56it/s]
 24%|██▍       | 12032/50176 [02:00<06:01, 105.44it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.51it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.80it/s]
 26%|██▌       | 12800/50176 [02:07<05:51, 106.45it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.19it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.70it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.04it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 100.07it/s]
 28%|██▊       | 14080/50176 [02:19<05:58, 100.57it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.49it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.50it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.84it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.50it/s]
 31%|███       | 15360/50176 [02:32<05:49, 99.66it/s]
 31%|███       | 15616/50176 [02:35<05:36, 102.60it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.62it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.79it/s]
 33%|███▎      | 16384/50176 [02:42<05:27, 103.34it/s]
 33%|███▎      | 16640/50176 [02:45<05:41, 98.34it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.60it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 103.93it/s]
 35%|███▍      | 17408/50176 [02:52<05:17, 103.12it/s]
 35%|███▌      | 17664/50176 [02:55<05:08, 105.34it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.11it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.42it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.63it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.34it/s]
 38%|███▊      | 18944/50176 [03:07<05:03, 103.07it/s]
 38%|███▊      | 19200/50176 [03:10<05:10, 99.66it/s] 
 39%|███▉      | 19456/50176 [03:12<05:13, 97.84it/s]
 39%|███▉      | 19712/50176 [03:15<05:10, 97.97it/s]
 40%|███▉      | 19968/50176 [03:18<05:08, 97.86it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.81it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.61it/s]
 41%|████▏     | 20736/50176 [03:25<04:38, 105.79it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.25it/s]
 42%|████▏     | 21248/50176 [03:30<04:43, 102.17it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.49it/s]
 43%|████▎     | 21760/50176 [03:35<04:36, 102.70it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.69it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 108.04it/s]
 45%|████▍     | 22528/50176 [03:41<04:16, 107.88it/s]
 45%|████▌     | 22784/50176 [03:44<04:25, 103.36it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.54it/s]
 46%|██���█▋     | 23296/50176 [03:49<04:27, 100.48it/s]
 47%|████▋     | 23552/50176 [03:52<04:31, 97.96it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.88it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.93it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.40it/s]
 49%|████▉     | 24576/50176 [04:02<04:14, 100.41it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.06it/s]
 50%|█████     | 25088/50176 [04:07<03:59, 104.54it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.72it/s]
 51%|█████     | 25600/50176 [04:12<03:54, 104.84it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.54it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.67it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.11it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.25it/s]
 54%|█████▎    | 26880/50176 [04:24<03:40, 105.62it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 105.13it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.31it/s]
 55%|█████▌    | 27648/50176 [04:30<03:22, 111.02it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.43it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.27it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.35it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 102.04it/s]
 58%|█████▊    | 28928/50176 [04:43<03:24, 103.71it/s]
 58%|█████▊    | 29184/50176 [04:46<03:21, 104.20it/s]
 59%|█████▊    | 29440/50176 [04:49<03:30, 98.39it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:30, 97.50it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.18it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.86it/s]
 61%|██████    | 30464/50176 [04:59<03:09, 104.10it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.27it/s]
 62%|██████▏   | 30976/50176 [05:04<03:06, 102.80it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.57it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.36it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.27it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 107.03it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 106.97it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.44it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.30it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.10it/s]
 66%|██████▋   | 33280/50176 [05:26<02:42, 103.80it/s]
 67%|██████▋   | 33536/50176 [05:28<02:41, 103.35it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.53it/s]
 68%|██████▊   | 34048/50176 [05:33<02:28, 108.41it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.11it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.00it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.41it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.89it/s]
 70%|███████   | 35328/50176 [05:50<03:01, 81.65it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.65it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.84it/s]
 72%|███████▏  | 36096/50176 [06:02<03:30, 66.87it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.68it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.88it/s]
 73%|███████▎  | 36864/50176 [06:14<03:16, 67.63it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.62it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.12it/s]
 75%|███████▌  | 37632/50176 [06:20<02:16, 91.64it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.19it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.85it/s]
 77%|███████▋  | 38400/50176 [06:35<03:19, 59.15it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 68.01it/s]
 78%|███████▊  | 38912/50176 [06:42<03:01, 62.22it/s]
 78%|███████▊  | 39168/50176 [06:45<02:38, 69.46it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.40it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.81it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.11it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.47it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.12it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.97it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.89it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.95it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.54it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.02it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.43it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.78it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.61it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.42it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.77it/s]
 86%|████████▌ | 43264/50176 [07:23<01:04, 107.81it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.24it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.43it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.50it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.67it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.36it/s]
 89%|████████▉ | 44800/50176 [07:38<00:52, 101.99it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.16it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 101.07it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 104.01it/s]
 91%|█████████▏| 45824/50176 [07:48<00:41, 104.14it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.53it/s]
 92%|█████████▏| 46336/50176 [07:53<00:37, 102.37it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.66it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.70it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.39it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.07it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.66it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.86it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.69it/s]
 96%|█████████▋| 48384/50176 [08:13<00:17, 103.34it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.83it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.71it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.90it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.20it/s]
 99%|█████████▉| 49664/50176 [08:26<00:05, 99.97it/s] 
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.71it/s]
-2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5273	imagenet-zeroshot-val-top5: 0.8075

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_11.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt).
-2022-12-21,09:36:52 | INFO | Model:
-2022-12-21,09:36:52 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:36:52 | INFO | Params:
-2022-12-21,09:36:52 | INFO |   batch_size: 256
-2022-12-21,09:36:52 | INFO |   beta1: 0.9
-2022-12-21,09:36:52 | INFO |   beta2: 0.98
-2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/checkpoints
-2022-12-21,09:36:52 | INFO |   copy_codebase: False
-2022-12-21,09:36:52 | INFO |   csv_caption_key: title
-2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
-2022-12-21,09:36:52 | INFO |   csv_separator: 	
-2022-12-21,09:36:52 | INFO |   dataset_resampled: False
-2022-12-21,09:36:52 | INFO |   dataset_type: auto
-2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
-2022-12-21,09:36:52 | INFO |   debug: False
-2022-12-21,09:36:52 | INFO |   device: cuda:0
-2022-12-21,09:36:52 | INFO |   dist_backend: nccl
-2022-12-21,09:36:52 | INFO |   dist_url: env://
-2022-12-21,09:36:52 | INFO |   distributed: False
-2022-12-21,09:36:52 | INFO |   epochs: 32
-2022-12-21,09:36:52 | INFO |   eps: 1e-06
-2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
-2022-12-21,09:36:52 | INFO |   gather_with_grad: False
-2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
-2022-12-21,09:36:52 | INFO |   horovod: False
-2022-12-21,09:36:52 | INFO |   image_mean: None
-2022-12-21,09:36:52 | INFO |   image_std: None
-2022-12-21,09:36:52 | INFO |   imagenet_v2: None
-2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:36:52 | INFO |   local_loss: False
-2022-12-21,09:36:52 | INFO |   local_rank: 0
-2022-12-21,09:36:52 | INFO |   lock_image: False
-2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:36:52 | INFO |   log_level: 20
-2022-12-21,09:36:52 | INFO |   log_local: False
-2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_11.pt/out.log
-2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:36:52 | INFO |   lr: 0.0005
-2022-12-21,09:36:52 | INFO |   model: ViT-B-16
-2022-12-21,09:36:52 | INFO |   name: epoch_11.pt
-2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
-2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
-2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
-2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_11.pt
-2022-12-21,09:36:52 | INFO |   pretrained_image: False
-2022-12-21,09:36:52 | INFO |   rank: 0
-2022-12-21,09:36:52 | INFO |   report_to: 
-2022-12-21,09:36:52 | INFO |   resume: None
-2022-12-21,09:36:52 | INFO |   save_frequency: 1
-2022-12-21,09:36:52 | INFO |   save_most_recent: False
-2022-12-21,09:36:52 | INFO |   seed: 0
-2022-12-21,09:36:52 | INFO |   skip_scheduler: False
-2022-12-21,09:36:52 | INFO |   tensorboard: False
-2022-12-21,09:36:52 | INFO |   tensorboard_path: 
-2022-12-21,09:36:52 | INFO |   torchscript: False
-2022-12-21,09:36:52 | INFO |   trace: False
-2022-12-21,09:36:52 | INFO |   train_data: None
-2022-12-21,09:36:52 | INFO |   train_num_samples: None
-2022-12-21,09:36:52 | INFO |   use_bn_sync: False
-2022-12-21,09:36:52 | INFO |   val_data: None
-2022-12-21,09:36:52 | INFO |   val_frequency: 1
-2022-12-21,09:36:52 | INFO |   val_num_samples: None
-2022-12-21,09:36:52 | INFO |   wandb: False
-2022-12-21,09:36:52 | INFO |   wandb_notes: 
-2022-12-21,09:36:52 | INFO |   warmup: 10000
-2022-12-21,09:36:52 | INFO |   wd: 0.2
-2022-12-21,09:36:52 | INFO |   workers: 1
-2022-12-21,09:36:52 | INFO |   world_size: 1
-2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:36:53 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<14:16,  1.17it/s]
  1%|          | 6/1000 [00:00<02:02,  8.13it/s]
  1%|          | 12/1000 [00:01<01:00, 16.40it/s]
  2%|▏         | 18/1000 [00:01<00:40, 24.35it/s]
  2%|▏         | 24/1000 [00:01<00:30, 31.55it/s]
  3%|▎         | 30/1000 [00:01<00:25, 37.71it/s]
  4%|▎         | 36/1000 [00:01<00:22, 42.67it/s]
  4%|▍         | 42/1000 [00:01<00:20, 46.55it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.50it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.70it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.29it/s]
  7%|▋         | 66/1000 [00:02<00:17, 54.45it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.27it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.85it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.25it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.63it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.02it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.24it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.54it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.67it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.67it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.62it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.67it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.63it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.64it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.71it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.62it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.32it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.25it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.28it/s]
 18%|█▊        | 180/1000 [00:04<00:14, 57.26it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.24it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.22it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.20it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.01it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.92it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.03it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.03it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.23it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.21it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.13it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.17it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.28it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.19it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.29it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.13it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.26it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.42it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.67it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.65it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.64it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.79it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.70it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.53it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.61it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.73it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.82it/s]
 35%|███▌      | 354/1000 [00:07<00:11, 57.61it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.62it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.67it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.75it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.45it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.20it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.30it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.43it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.61it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.69it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.70it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.70it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.75it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.84it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.85it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.66it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.73it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.78it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.73it/s]
 47%|████▋     | 468/1000 [00:09<00:09, 57.66it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.81it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.67it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.66it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.66it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.56it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.62it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.44it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.61it/s]
 53%|█████▎    | 528/1000 [00:10<00:08, 57.70it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.57it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.71it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.75it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.65it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.68it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.75it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.65it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.56it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.68it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.69it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.60it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.69it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.62it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.63it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.75it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.76it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.85it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.68it/s]
 64%|██████▍   | 642/1000 [00:12<00:06, 57.62it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.76it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.74it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.79it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.83it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.74it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.77it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.77it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.82it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.72it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.79it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.78it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.76it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.71it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.67it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.72it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.71it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.80it/s]
 76%|███████▌  | 756/1000 [00:14<00:04, 57.89it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.73it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.82it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.77it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.80it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.83it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.64it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.63it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.69it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.66it/s]
 82%|████████▏ | 816/1000 [00:15<00:03, 57.66it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.68it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.79it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.92it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.95it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.87it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.93it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.88it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.85it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.79it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.85it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.83it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.71it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.79it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.80it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.79it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.83it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.82it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.62it/s]
 93%|█████████▎| 930/1000 [00:17<00:01, 57.75it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.84it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.84it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.77it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.82it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.81it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.88it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.96it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.94it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.94it/s]
 99%|█████████▉| 990/1000 [00:18<00:00, 57.87it/s]
-2022-12-21,09:37:11 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:17, 114.04it/s]
  1%|          | 512/50176 [00:04<07:03, 117.32it/s]
  2%|▏         | 768/50176 [00:06<07:22, 111.54it/s]
  2%|▏         | 1024/50176 [00:09<07:56, 103.23it/s]
  3%|▎         | 1280/50176 [00:12<08:07, 100.28it/s]
  3%|▎         | 1536/50176 [00:15<08:57, 90.57it/s] 
  4%|▎         | 1792/50176 [00:18<08:39, 93.22it/s]
  4%|▍         | 2048/50176 [00:20<08:31, 94.10it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.70it/s]
  5%|▌         | 2560/50176 [00:25<08:09, 97.37it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.24it/s]
  6%|▌         | 3072/50176 [00:31<08:16, 94.79it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.90it/s]
  7%|▋         | 3584/50176 [00:36<07:43, 100.63it/s]
  8%|▊         | 3840/50176 [00:39<07:58, 96.81it/s] 
  8%|▊         | 4096/50176 [00:42<08:12, 93.64it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.20it/s]
  9%|▉         | 4608/50176 [00:46<07:37, 99.70it/s]
 10%|▉         | 4864/50176 [00:49<07:25, 101.82it/s]
 10%|█         | 5120/50176 [00:51<07:21, 102.16it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.50it/s]
 11%|█         | 5632/50176 [00:56<07:16, 102.03it/s]
 12%|█▏        | 5888/50176 [00:59<07:14, 101.96it/s]
 12%|█▏        | 6144/50176 [01:02<07:44, 94.87it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.91it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.31it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.57it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.49it/s]
 15%|█▍        | 7424/50176 [01:15<07:08, 99.86it/s]
 15%|█▌        | 7680/50176 [01:17<07:05, 99.87it/s]
 16%|█▌        | 7936/50176 [01:20<06:48, 103.41it/s]
 16%|█▋        | 8192/50176 [01:22<06:48, 102.83it/s]
 17%|█▋        | 8448/50176 [01:25<06:49, 101.99it/s]
 17%|█▋        | 8704/50176 [01:27<06:48, 101.40it/s]
 18%|█▊        | 8960/50176 [01:30<06:48, 100.94it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.71it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.95it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.20it/s]
 20%|█▉        | 9984/50176 [01:40<06:46, 98.88it/s] 
 20%|██        | 10240/50176 [01:42<06:31, 101.93it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.13it/s]
 21%|██▏       | 10752/50176 [01:47<06:28, 101.42it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.12it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.79it/s]
 23%|██▎       | 11520/50176 [01:55<06:08, 104.93it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.57it/s]
 24%|██▍       | 12032/50176 [01:59<06:01, 105.45it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.54it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.87it/s]
 26%|██▌       | 12800/50176 [02:07<05:50, 106.49it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.23it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.66it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.07it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 100.10it/s]
 28%|██▊       | 14080/50176 [02:19<05:59, 100.51it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.43it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.50it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.84it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.38it/s]
 31%|███       | 15360/50176 [02:32<05:49, 99.67it/s]
 31%|███       | 15616/50176 [02:35<05:36, 102.63it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.67it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.82it/s]
 33%|███▎      | 16384/50176 [02:42<05:26, 103.38it/s]
 33%|███▎      | 16640/50176 [02:45<05:41, 98.33it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.59it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 103.91it/s]
 35%|███▍      | 17408/50176 [02:52<05:17, 103.20it/s]
 35%|███▌      | 17664/50176 [02:54<05:08, 105.37it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.14it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.37it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.61it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.40it/s]
 38%|███▊      | 18944/50176 [03:07<05:03, 102.98it/s]
 38%|███▊      | 19200/50176 [03:09<05:10, 99.67it/s] 
 39%|███▉      | 19456/50176 [03:12<05:13, 97.87it/s]
 39%|███▉      | 19712/50176 [03:15<05:11, 97.93it/s]
 40%|███▉      | 19968/50176 [03:17<05:08, 97.94it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.85it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.64it/s]
 41%|████▏     | 20736/50176 [03:24<04:38, 105.80it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.24it/s]
 42%|████▏     | 21248/50176 [03:29<04:43, 102.17it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.42it/s]
 43%|████▎     | 21760/50176 [03:34<04:36, 102.72it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.70it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 107.83it/s]
 45%|████▍     | 22528/50176 [03:41<04:15, 108.04it/s]
 45%|████▌     | 22784/50176 [03:44<04:24, 103.37it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.55it/s]
 46%|████▋     | 23296/50176 [03:49<04:27, 100.41it/s]
 47%|████▋     | 23552/50176 [03:52<04:32, 97.85it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.78it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.99it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.49it/s]
 49%|████▉     | 24576/50176 [04:02<04:15, 100.34it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.10it/s]
 50%|█████     | 25088/50176 [04:07<04:00, 104.52it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.71it/s]
 51%|█████     | 25600/50176 [04:11<03:54, 104.81it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.60it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.70it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.12it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.31it/s]
 54%|█████▎    | 26880/50176 [04:23<03:40, 105.77it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 104.99it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.40it/s]
 55%|█████▌    | 27648/50176 [04:30<03:22, 111.15it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.31it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.26it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.26it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 101.92it/s]
 58%|█████▊    | 28928/50176 [04:43<03:24, 103.81it/s]
 58%|█████▊    | 29184/50176 [04:45<03:21, 104.17it/s]
 59%|█████▊    | 29440/50176 [04:48<03:30, 98.48it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:29, 97.57it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.08it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.87it/s]
 61%|██████    | 30464/50176 [04:58<03:09, 104.06it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.35it/s]
 62%|██████▏   | 30976/50176 [05:03<03:06, 102.70it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.53it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.47it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.28it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 106.93it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 107.01it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.49it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.31it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.13it/s]
 66%|██████▋   | 33280/50176 [05:25<02:42, 103.71it/s]
 67%|██████▋   | 33536/50176 [05:28<02:40, 103.43it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.53it/s]
 68%|██████▊   | 34048/50176 [05:32<02:28, 108.51it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.09it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.09it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.40it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.89it/s]
 70%|███████   | 35328/50176 [05:49<03:01, 81.64it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.66it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.83it/s]
 72%|███████▏  | 36096/50176 [06:01<03:30, 66.87it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.71it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.90it/s]
 73%|███████▎  | 36864/50176 [06:13<03:16, 67.63it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.62it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.12it/s]
 75%|███████▌  | 37632/50176 [06:20<02:16, 91.60it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.25it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.88it/s]
 77%|███████▋  | 38400/50176 [06:34<03:19, 59.10it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 67.97it/s]
 78%|███████▊  | 38912/50176 [06:42<03:00, 62.24it/s]
 78%|███████▊  | 39168/50176 [06:44<02:38, 69.46it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.41it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.87it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.13it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.52it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.07it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.93it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.74it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.99it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.62it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.15it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.66it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.66it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.57it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.84it/s]
 86%|████████▌ | 43264/50176 [07:22<01:04, 107.81it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.14it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.38it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.47it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.56it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.36it/s]
 89%|████████▉ | 44800/50176 [07:37<00:52, 102.08it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.24it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 100.89it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 104.01it/s]
 91%|█████████▏| 45824/50176 [07:47<00:41, 104.14it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.63it/s]
 92%|█████████▏| 46336/50176 [07:52<00:37, 102.25it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.74it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.86it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.48it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.08it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.58it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.89it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.59it/s]
 96%|█████████▋| 48384/50176 [08:12<00:17, 103.22it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.91it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.51it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.92it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.32it/s]
 99%|█████████▉| 49664/50176 [08:25<00:05, 100.06it/s]
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.68it/s] 
-2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5318	imagenet-zeroshot-val-top5: 0.8127

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_12.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:36:42 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:36:42 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:36:43 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt).
-2022-12-21,09:36:52 | INFO | Model:
-2022-12-21,09:36:52 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:36:52 | INFO | Params:
-2022-12-21,09:36:52 | INFO |   batch_size: 256
-2022-12-21,09:36:52 | INFO |   beta1: 0.9
-2022-12-21,09:36:52 | INFO |   beta2: 0.98
-2022-12-21,09:36:52 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/checkpoints
-2022-12-21,09:36:52 | INFO |   copy_codebase: False
-2022-12-21,09:36:52 | INFO |   csv_caption_key: title
-2022-12-21,09:36:52 | INFO |   csv_img_key: filepath
-2022-12-21,09:36:52 | INFO |   csv_separator: 	
-2022-12-21,09:36:52 | INFO |   dataset_resampled: False
-2022-12-21,09:36:52 | INFO |   dataset_type: auto
-2022-12-21,09:36:52 | INFO |   ddp_static_graph: False
-2022-12-21,09:36:52 | INFO |   debug: False
-2022-12-21,09:36:52 | INFO |   device: cuda:0
-2022-12-21,09:36:52 | INFO |   dist_backend: nccl
-2022-12-21,09:36:52 | INFO |   dist_url: env://
-2022-12-21,09:36:52 | INFO |   distributed: False
-2022-12-21,09:36:52 | INFO |   epochs: 32
-2022-12-21,09:36:52 | INFO |   eps: 1e-06
-2022-12-21,09:36:52 | INFO |   force_quick_gelu: False
-2022-12-21,09:36:52 | INFO |   gather_with_grad: False
-2022-12-21,09:36:52 | INFO |   grad_checkpointing: False
-2022-12-21,09:36:52 | INFO |   horovod: False
-2022-12-21,09:36:52 | INFO |   image_mean: None
-2022-12-21,09:36:52 | INFO |   image_std: None
-2022-12-21,09:36:52 | INFO |   imagenet_v2: None
-2022-12-21,09:36:52 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:36:52 | INFO |   local_loss: False
-2022-12-21,09:36:52 | INFO |   local_rank: 0
-2022-12-21,09:36:52 | INFO |   lock_image: False
-2022-12-21,09:36:52 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:36:52 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:36:52 | INFO |   log_level: 20
-2022-12-21,09:36:52 | INFO |   log_local: False
-2022-12-21,09:36:52 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_12.pt/out.log
-2022-12-21,09:36:52 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:36:52 | INFO |   lr: 0.0005
-2022-12-21,09:36:52 | INFO |   model: ViT-B-16
-2022-12-21,09:36:52 | INFO |   name: epoch_12.pt
-2022-12-21,09:36:52 | INFO |   no_set_device_rank: False
-2022-12-21,09:36:52 | INFO |   norm_gradient_clip: None
-2022-12-21,09:36:52 | INFO |   precision: amp_bfloat16
-2022-12-21,09:36:52 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_12.pt
-2022-12-21,09:36:52 | INFO |   pretrained_image: False
-2022-12-21,09:36:52 | INFO |   rank: 0
-2022-12-21,09:36:52 | INFO |   report_to: 
-2022-12-21,09:36:52 | INFO |   resume: None
-2022-12-21,09:36:52 | INFO |   save_frequency: 1
-2022-12-21,09:36:52 | INFO |   save_most_recent: False
-2022-12-21,09:36:52 | INFO |   seed: 0
-2022-12-21,09:36:52 | INFO |   skip_scheduler: False
-2022-12-21,09:36:52 | INFO |   tensorboard: False
-2022-12-21,09:36:52 | INFO |   tensorboard_path: 
-2022-12-21,09:36:52 | INFO |   torchscript: False
-2022-12-21,09:36:52 | INFO |   trace: False
-2022-12-21,09:36:52 | INFO |   train_data: None
-2022-12-21,09:36:52 | INFO |   train_num_samples: None
-2022-12-21,09:36:52 | INFO |   use_bn_sync: False
-2022-12-21,09:36:52 | INFO |   val_data: None
-2022-12-21,09:36:52 | INFO |   val_frequency: 1
-2022-12-21,09:36:52 | INFO |   val_num_samples: None
-2022-12-21,09:36:52 | INFO |   wandb: False
-2022-12-21,09:36:52 | INFO |   wandb_notes: 
-2022-12-21,09:36:52 | INFO |   warmup: 10000
-2022-12-21,09:36:52 | INFO |   wd: 0.2
-2022-12-21,09:36:52 | INFO |   workers: 1
-2022-12-21,09:36:52 | INFO |   world_size: 1
-2022-12-21,09:36:52 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:36:53 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:36:53 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:26,  1.24it/s]
  1%|          | 6/1000 [00:00<01:56,  8.56it/s]
  1%|          | 12/1000 [00:01<00:57, 17.16it/s]
  2%|▏         | 18/1000 [00:01<00:38, 25.44it/s]
  2%|▏         | 24/1000 [00:01<00:29, 32.66it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.77it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.60it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.36it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.20it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.27it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.81it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.92it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.65it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.19it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.77it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.04it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.46it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.59it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.88it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.01it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.02it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.91it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.97it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.03it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.95it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.81it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.81it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.76it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.67it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.64it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.43it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.35it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.46it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.42it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.58it/s]
 23%|██▎       | 234/1000 [00:04<00:14, 51.94it/s]
 24%|██▍       | 240/1000 [00:05<00:14, 53.44it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 54.63it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 55.55it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 56.26it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 56.75it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 56.89it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.13it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.46it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.67it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.94it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.97it/s]
 31%|███       | 306/1000 [00:06<00:11, 57.93it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.11it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.12it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.06it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.91it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.98it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.08it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.18it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.94it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.85it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.93it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.04it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.96it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.66it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.70it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.98it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.05it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 58.05it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.05it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.07it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.17it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.96it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.02it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.98it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.13it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.06it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.04it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.04it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.94it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.00it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.92it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.13it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 58.01it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.13it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.06it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.07it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.11it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.94it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 58.07it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.99it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.09it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.02it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.13it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.14it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.22it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.05it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.99it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.14it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.10it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.12it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.15it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.17it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.21it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.09it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.18it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.16it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.12it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.13it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.10it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.18it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.27it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 58.11it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.18it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.13it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.17it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.20it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.02it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.07it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.01it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.02it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.83it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.97it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.16it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.24it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.19it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.20it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.21it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.19it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.19it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.07it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.17it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.15it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.12it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.14it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.22it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.25it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.26it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.29it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.36it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.30it/s]
-2022-12-21,09:37:11 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:42, 108.02it/s]
  1%|          | 512/50176 [00:04<07:12, 114.70it/s]
  2%|▏         | 768/50176 [00:06<07:28, 110.23it/s]
  2%|▏         | 1024/50176 [00:09<08:00, 102.38it/s]
  3%|▎         | 1280/50176 [00:12<08:10, 99.76it/s] 
  3%|▎         | 1536/50176 [00:15<08:59, 90.09it/s]
  4%|▎         | 1792/50176 [00:18<08:39, 93.12it/s]
  4%|▍         | 2048/50176 [00:21<08:32, 93.87it/s]
  5%|▍         | 2304/50176 [00:23<08:20, 95.67it/s]
  5%|▌         | 2560/50176 [00:26<08:08, 97.49it/s]
  6%|▌         | 2816/50176 [00:28<08:17, 95.24it/s]
  6%|▌         | 3072/50176 [00:31<08:16, 94.80it/s]
  7%|▋         | 3328/50176 [00:34<08:03, 96.83it/s]
  7%|▋         | 3584/50176 [00:36<07:43, 100.56it/s]
  8%|▊         | 3840/50176 [00:39<07:58, 96.75it/s] 
  8%|▊         | 4096/50176 [00:42<08:11, 93.73it/s]
  9%|▊         | 4352/50176 [00:44<07:56, 96.11it/s]
  9%|▉         | 4608/50176 [00:47<07:36, 99.82it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.96it/s]
 10%|█         | 5120/50176 [00:51<07:20, 102.25it/s]
 11%|█         | 5376/50176 [00:54<07:17, 102.29it/s]
 11%|█         | 5632/50176 [00:57<07:16, 102.12it/s]
 12%|█▏        | 5888/50176 [00:59<07:15, 101.79it/s]
 12%|█▏        | 6144/50176 [01:02<07:43, 94.98it/s] 
 13%|█▎        | 6400/50176 [01:05<07:36, 95.96it/s]
 13%|█▎        | 6656/50176 [01:07<07:31, 96.41it/s]
 14%|█▍        | 6912/50176 [01:10<07:27, 96.69it/s]
 14%|█▍        | 7168/50176 [01:12<07:12, 99.54it/s]
 15%|█▍        | 7424/50176 [01:15<07:09, 99.58it/s]
 15%|█▌        | 7680/50176 [01:18<07:05, 99.81it/s]
 16%|█▌        | 7936/50176 [01:20<06:49, 103.11it/s]
 16%|█▋        | 8192/50176 [01:22<06:47, 102.93it/s]
 17%|█▋        | 8448/50176 [01:25<06:48, 102.08it/s]
 17%|█▋        | 8704/50176 [01:27<06:49, 101.38it/s]
 18%|█▊        | 8960/50176 [01:30<06:47, 101.13it/s]
 18%|█▊        | 9216/50176 [01:32<06:38, 102.75it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.95it/s]
 19%|█▉        | 9728/50176 [01:37<06:35, 102.18it/s]
 20%|█▉        | 9984/50176 [01:40<06:47, 98.75it/s] 
 20%|██        | 10240/50176 [01:42<06:30, 102.16it/s]
 21%|██        | 10496/50176 [01:45<06:17, 105.06it/s]
 21%|██▏       | 10752/50176 [01:47<06:28, 101.39it/s]
 22%|██▏       | 11008/50176 [01:50<06:23, 102.19it/s]
 22%|██▏       | 11264/50176 [01:52<06:11, 104.67it/s]
 23%|██▎       | 11520/50176 [01:55<06:09, 104.71it/s]
 23%|██▎       | 11776/50176 [01:57<06:07, 104.63it/s]
 24%|██▍       | 12032/50176 [02:00<06:02, 105.37it/s]
 24%|██▍       | 12288/50176 [02:02<05:55, 106.72it/s]
 25%|██▌       | 12544/50176 [02:04<05:55, 105.79it/s]
 26%|██▌       | 12800/50176 [02:07<05:50, 106.56it/s]
 26%|██▌       | 13056/50176 [02:09<05:46, 107.16it/s]
 27%|██▋       | 13312/50176 [02:11<05:42, 107.65it/s]
 27%|██▋       | 13568/50176 [02:14<05:58, 102.07it/s]
 28%|██▊       | 13824/50176 [02:17<06:03, 99.92it/s] 
 28%|██▊       | 14080/50176 [02:19<05:58, 100.69it/s]
 29%|██▊       | 14336/50176 [02:22<05:56, 100.47it/s]
 29%|██▉       | 14592/50176 [02:24<05:47, 102.43it/s]
 30%|██▉       | 14848/50176 [02:27<06:04, 96.88it/s] 
 30%|███       | 15104/50176 [02:30<05:56, 98.42it/s]
 31%|███       | 15360/50176 [02:32<05:48, 99.78it/s]
 31%|███       | 15616/50176 [02:35<05:37, 102.49it/s]
 32%|███▏      | 15872/50176 [02:37<05:37, 101.66it/s]
 32%|███▏      | 16128/50176 [02:40<05:31, 102.80it/s]
 33%|███▎      | 16384/50176 [02:42<05:27, 103.32it/s]
 33%|███▎      | 16640/50176 [02:45<05:40, 98.38it/s] 
 34%|███▎      | 16896/50176 [02:47<05:30, 100.62it/s]
 34%|███▍      | 17152/50176 [02:50<05:17, 104.02it/s]
 35%|███▍      | 17408/50176 [02:52<05:18, 102.95it/s]
 35%|███▌      | 17664/50176 [02:55<05:08, 105.42it/s]
 36%|███▌      | 17920/50176 [02:57<05:01, 107.07it/s]
 36%|███▌      | 18176/50176 [02:59<05:03, 105.41it/s]
 37%|███▋      | 18432/50176 [03:02<05:00, 105.68it/s]
 37%|███▋      | 18688/50176 [03:04<05:04, 103.31it/s]
 38%|███▊      | 18944/50176 [03:07<05:02, 103.17it/s]
 38%|███▊      | 19200/50176 [03:10<05:10, 99.70it/s] 
 39%|███▉      | 19456/50176 [03:12<05:14, 97.81it/s]
 39%|███▉      | 19712/50176 [03:15<05:10, 98.02it/s]
 40%|███▉      | 19968/50176 [03:18<05:08, 97.90it/s]
 40%|████      | 20224/50176 [03:20<04:51, 102.86it/s]
 41%|████      | 20480/50176 [03:22<04:41, 105.51it/s]
 41%|████▏     | 20736/50176 [03:24<04:38, 105.71it/s]
 42%|████▏     | 20992/50176 [03:27<04:34, 106.35it/s]
 42%|████▏     | 21248/50176 [03:30<04:43, 102.11it/s]
 43%|████▎     | 21504/50176 [03:32<04:39, 102.50it/s]
 43%|████▎     | 21760/50176 [03:35<04:36, 102.79it/s]
 44%|████▍     | 22016/50176 [03:37<04:26, 105.59it/s]
 44%|████▍     | 22272/50176 [03:39<04:18, 107.90it/s]
 45%|████▍     | 22528/50176 [03:41<04:16, 107.84it/s]
 45%|████▌     | 22784/50176 [03:44<04:24, 103.38it/s]
 46%|████▌     | 23040/50176 [03:47<04:29, 100.58it/s]
 46%|██���█▋     | 23296/50176 [03:49<04:27, 100.55it/s]
 47%|████▋     | 23552/50176 [03:52<04:32, 97.87it/s] 
 47%|████▋     | 23808/50176 [03:54<04:16, 102.78it/s]
 48%|████▊     | 24064/50176 [03:57<04:11, 103.91it/s]
 48%|████▊     | 24320/50176 [03:59<04:12, 102.55it/s]
 49%|████▉     | 24576/50176 [04:02<04:15, 100.33it/s]
 49%|████▉     | 24832/50176 [04:04<04:05, 103.17it/s]
 50%|█████     | 25088/50176 [04:07<03:59, 104.58it/s]
 51%|█████     | 25344/50176 [04:09<03:52, 106.86it/s]
 51%|█████     | 25600/50176 [04:12<03:54, 104.75it/s]
 52%|█████▏    | 25856/50176 [04:14<03:48, 106.64it/s]
 52%|█████▏    | 26112/50176 [04:16<03:45, 106.67it/s]
 53%|█████▎    | 26368/50176 [04:19<03:46, 105.03it/s]
 53%|█████▎    | 26624/50176 [04:21<03:41, 106.36it/s]
 54%|█████▎    | 26880/50176 [04:24<03:40, 105.65it/s]
 54%|█████▍    | 27136/50176 [04:26<03:39, 104.97it/s]
 55%|█████▍    | 27392/50176 [04:28<03:34, 106.44it/s]
 55%|█████▌    | 27648/50176 [04:30<03:23, 110.88it/s]
 56%|█████▌    | 27904/50176 [04:33<03:31, 105.48it/s]
 56%|█████▌    | 28160/50176 [04:36<03:33, 103.27it/s]
 57%|█████▋    | 28416/50176 [04:38<03:28, 104.27it/s]
 57%|█████▋    | 28672/50176 [04:41<03:30, 101.94it/s]
 58%|█████▊    | 28928/50176 [04:43<03:25, 103.62it/s]
 58%|█████▊    | 29184/50176 [04:46<03:21, 104.01it/s]
 59%|█████▊    | 29440/50176 [04:48<03:30, 98.57it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:30, 97.50it/s]
 60%|█████▉    | 29952/50176 [04:54<03:28, 97.14it/s]
 60%|██████    | 30208/50176 [04:56<03:21, 98.92it/s]
 61%|██████    | 30464/50176 [04:58<03:09, 104.10it/s]
 61%|██████    | 30720/50176 [05:01<03:10, 102.29it/s]
 62%|██████▏   | 30976/50176 [05:04<03:06, 102.74it/s]
 62%|██████▏   | 31232/50176 [05:06<03:02, 103.56it/s]
 63%|██████▎   | 31488/50176 [05:08<02:57, 105.56it/s]
 63%|██████▎   | 31744/50176 [05:11<02:53, 106.31it/s]
 64%|██████▍   | 32000/50176 [05:13<02:49, 106.95it/s]
 64%|██████▍   | 32256/50176 [05:15<02:47, 106.93it/s]
 65%|██████▍   | 32512/50176 [05:18<02:55, 100.50it/s]
 65%|██████▌   | 32768/50176 [05:21<02:50, 102.31it/s]
 66%|██████▌   | 33024/50176 [05:23<02:43, 105.19it/s]
 66%|██████▋   | 33280/50176 [05:26<02:42, 103.71it/s]
 67%|██████▋   | 33536/50176 [05:28<02:40, 103.45it/s]
 67%|██████▋   | 33792/50176 [05:30<02:33, 106.63it/s]
 68%|██████▊   | 34048/50176 [05:33<02:28, 108.49it/s]
 68%|██████▊   | 34304/50176 [05:35<02:22, 111.13it/s]
 69%|██████▉   | 34560/50176 [05:37<02:27, 106.01it/s]
 69%|██████▉   | 34816/50176 [05:45<03:54, 65.40it/s] 
 70%|██████▉   | 35072/50176 [05:47<03:27, 72.87it/s]
 70%|███████   | 35328/50176 [05:50<03:01, 81.63it/s]
 71%|███████   | 35584/50176 [05:52<02:46, 87.68it/s]
 71%|███████▏  | 35840/50176 [05:57<03:19, 71.84it/s]
 72%|███████▏  | 36096/50176 [06:02<03:30, 66.88it/s]
 72%|███████▏  | 36352/50176 [06:06<03:40, 62.68it/s]
 73%|███████▎  | 36608/50176 [06:11<03:46, 59.86it/s]
 73%|███████▎  | 36864/50176 [06:14<03:16, 67.60it/s]
 74%|███████▍  | 37120/50176 [06:16<02:50, 76.67it/s]
 74%|███████▍  | 37376/50176 [06:18<02:28, 86.07it/s]
 75%|███████▌  | 37632/50176 [06:20<02:17, 91.54it/s]
 76%|███████▌  | 37888/50176 [06:23<02:07, 96.25it/s]
 76%|███████▌  | 38144/50176 [06:28<02:47, 71.88it/s]
 77%|███████▋  | 38400/50176 [06:35<03:19, 59.09it/s]
 77%|███████▋  | 38656/50176 [06:37<02:49, 67.99it/s]
 78%|███████▊  | 38912/50176 [06:42<03:00, 62.26it/s]
 78%|███████▊  | 39168/50176 [06:45<02:38, 69.41it/s]
 79%|███████▊  | 39424/50176 [06:47<02:17, 78.44it/s]
 79%|███████▉  | 39680/50176 [06:49<02:02, 85.91it/s]
 80%|███████▉  | 39936/50176 [06:51<01:49, 93.16it/s]
 80%|████████  | 40192/50176 [06:54<01:41, 98.52it/s]
 81%|████████  | 40448/50176 [06:56<01:40, 97.08it/s]
 81%|████████  | 40704/50176 [06:59<01:37, 96.94it/s]
 82%|████████▏ | 40960/50176 [07:01<01:29, 102.75it/s]
 82%|████████▏ | 41216/50176 [07:04<01:27, 102.98it/s]
 83%|████████▎ | 41472/50176 [07:06<01:24, 102.62it/s]
 83%|████████▎ | 41728/50176 [07:08<01:19, 106.00it/s]
 84%|████████▎ | 41984/50176 [07:11<01:16, 106.48it/s]
 84%|████████▍ | 42240/50176 [07:13<01:13, 107.61it/s]
 85%|████████▍ | 42496/50176 [07:15<01:10, 109.63it/s]
 85%|████████▌ | 42752/50176 [07:18<01:07, 109.48it/s]
 86%|████████▌ | 43008/50176 [07:20<01:05, 108.87it/s]
 86%|████████▌ | 43264/50176 [07:22<01:04, 107.89it/s]
 87%|████████▋ | 43520/50176 [07:25<01:04, 103.20it/s]
 87%|████████▋ | 43776/50176 [07:28<01:01, 103.41it/s]
 88%|████████▊ | 44032/50176 [07:30<00:58, 104.53it/s]
 88%|████████▊ | 44288/50176 [07:32<00:55, 105.57it/s]
 89%|████████▉ | 44544/50176 [07:35<00:55, 102.38it/s]
 89%|████████▉ | 44800/50176 [07:38<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:40<00:49, 103.25it/s]
 90%|█████████ | 45312/50176 [07:43<00:48, 100.96it/s]
 91%|█████████ | 45568/50176 [07:45<00:44, 104.00it/s]
 91%|█████████▏| 45824/50176 [07:47<00:41, 104.12it/s]
 92%|█████████▏| 46080/50176 [07:50<00:40, 101.59it/s]
 92%|█████████▏| 46336/50176 [07:53<00:37, 102.43it/s]
 93%|█████████▎| 46592/50176 [07:55<00:34, 104.58it/s]
 93%|█████████▎| 46848/50176 [07:57<00:32, 103.75it/s]
 94%|█████████▍| 47104/50176 [08:00<00:30, 99.42it/s] 
 94%|█████████▍| 47360/50176 [08:03<00:28, 99.09it/s]
 95%|█████████▍| 47616/50176 [08:05<00:25, 101.58it/s]
 95%|█████████▌| 47872/50176 [08:08<00:22, 101.86it/s]
 96%|█████████▌| 48128/50176 [08:10<00:20, 101.51it/s]
 96%|█████████▋| 48384/50176 [08:13<00:17, 103.31it/s]
 97%|█████████▋| 48640/50176 [08:15<00:14, 102.93it/s]
 97%|█████████▋| 48896/50176 [08:18<00:12, 102.61it/s]
 98%|█████████▊| 49152/50176 [08:20<00:10, 99.98it/s] 
 98%|█████████▊| 49408/50176 [08:23<00:07, 101.26it/s]
 99%|█████████▉| 49664/50176 [08:25<00:05, 100.04it/s]
 99%|█████████▉| 49920/50176 [08:28<00:02, 95.65it/s] 
-2022-12-21,09:45:40 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:45:40 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5369	imagenet-zeroshot-val-top5: 0.8153

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_13.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt).
-2022-12-21,09:46:05 | INFO | Model:
-2022-12-21,09:46:05 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:46:05 | INFO | Params:
-2022-12-21,09:46:05 | INFO |   batch_size: 256
-2022-12-21,09:46:05 | INFO |   beta1: 0.9
-2022-12-21,09:46:05 | INFO |   beta2: 0.98
-2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/checkpoints
-2022-12-21,09:46:05 | INFO |   copy_codebase: False
-2022-12-21,09:46:05 | INFO |   csv_caption_key: title
-2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
-2022-12-21,09:46:05 | INFO |   csv_separator: 	
-2022-12-21,09:46:05 | INFO |   dataset_resampled: False
-2022-12-21,09:46:05 | INFO |   dataset_type: auto
-2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
-2022-12-21,09:46:05 | INFO |   debug: False
-2022-12-21,09:46:05 | INFO |   device: cuda:0
-2022-12-21,09:46:05 | INFO |   dist_backend: nccl
-2022-12-21,09:46:05 | INFO |   dist_url: env://
-2022-12-21,09:46:05 | INFO |   distributed: False
-2022-12-21,09:46:05 | INFO |   epochs: 32
-2022-12-21,09:46:05 | INFO |   eps: 1e-06
-2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
-2022-12-21,09:46:05 | INFO |   gather_with_grad: False
-2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
-2022-12-21,09:46:05 | INFO |   horovod: False
-2022-12-21,09:46:05 | INFO |   image_mean: None
-2022-12-21,09:46:05 | INFO |   image_std: None
-2022-12-21,09:46:05 | INFO |   imagenet_v2: None
-2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:46:05 | INFO |   local_loss: False
-2022-12-21,09:46:05 | INFO |   local_rank: 0
-2022-12-21,09:46:05 | INFO |   lock_image: False
-2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:46:05 | INFO |   log_level: 20
-2022-12-21,09:46:05 | INFO |   log_local: False
-2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_13.pt/out.log
-2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:46:05 | INFO |   lr: 0.0005
-2022-12-21,09:46:05 | INFO |   model: ViT-B-16
-2022-12-21,09:46:05 | INFO |   name: epoch_13.pt
-2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
-2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
-2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
-2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_13.pt
-2022-12-21,09:46:05 | INFO |   pretrained_image: False
-2022-12-21,09:46:05 | INFO |   rank: 0
-2022-12-21,09:46:05 | INFO |   report_to: 
-2022-12-21,09:46:05 | INFO |   resume: None
-2022-12-21,09:46:05 | INFO |   save_frequency: 1
-2022-12-21,09:46:05 | INFO |   save_most_recent: False
-2022-12-21,09:46:05 | INFO |   seed: 0
-2022-12-21,09:46:05 | INFO |   skip_scheduler: False
-2022-12-21,09:46:05 | INFO |   tensorboard: False
-2022-12-21,09:46:05 | INFO |   tensorboard_path: 
-2022-12-21,09:46:05 | INFO |   torchscript: False
-2022-12-21,09:46:05 | INFO |   trace: False
-2022-12-21,09:46:05 | INFO |   train_data: None
-2022-12-21,09:46:05 | INFO |   train_num_samples: None
-2022-12-21,09:46:05 | INFO |   use_bn_sync: False
-2022-12-21,09:46:05 | INFO |   val_data: None
-2022-12-21,09:46:05 | INFO |   val_frequency: 1
-2022-12-21,09:46:05 | INFO |   val_num_samples: None
-2022-12-21,09:46:05 | INFO |   wandb: False
-2022-12-21,09:46:05 | INFO |   wandb_notes: 
-2022-12-21,09:46:05 | INFO |   warmup: 10000
-2022-12-21,09:46:05 | INFO |   wd: 0.2
-2022-12-21,09:46:05 | INFO |   workers: 1
-2022-12-21,09:46:05 | INFO |   world_size: 1
-2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:46:05 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:33,  1.58it/s]
  1%|          | 6/1000 [00:00<01:35, 10.46it/s]
  1%|          | 12/1000 [00:00<00:48, 20.31it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.02it/s]
  2%|▏         | 24/1000 [00:01<00:27, 36.07it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.73it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.84it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.07it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.47it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.17it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.39it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.28it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.92it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.40it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.81it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.10it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.47it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.64it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.80it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.78it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.77it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.78it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.85it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.74it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.70it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.64it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.59it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.51it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.34it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.34it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.52it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.50it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.42it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.47it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.52it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.61it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.68it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.48it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.51it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.70it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.82it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.04it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.02it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.00it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.09it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.03it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.85it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.92it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.93it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.02it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.82it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.84it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.94it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.02it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.93it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.61it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.69it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.78it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.91it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.99it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.01it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.03it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.14it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.13it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.90it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.98it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.97it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.10it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.98it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.95it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.96it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.93it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.86it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.98it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.07it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.02it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.15it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.11it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.00it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.99it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.04it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.90it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.85it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.00it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.00it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.91it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.03it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.96it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.96it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.04it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.14it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.96it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.88it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.03it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.00it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.09it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.12it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.07it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.09it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.10it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.14it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.04it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.15it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.13it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.11it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.13it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.12it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.19it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.27it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.09it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.18it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.12it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.15it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.18it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.97it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.95it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.97it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.71it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.76it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.84it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 57.97it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.16it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.22it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.15it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.21it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.16it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.10it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.13it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.00it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.12it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.07it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.09it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.18it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.18it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.21it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.21it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.25it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.27it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.28it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.98it/s]
-2022-12-21,09:46:23 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<06:35, 126.10it/s]
  1%|          | 512/50176 [00:04<06:37, 124.94it/s]
  2%|▏         | 768/50176 [00:06<06:46, 121.63it/s]
  2%|▏         | 1024/50176 [00:08<06:47, 120.50it/s]
  3%|▎         | 1280/50176 [00:10<07:06, 114.59it/s]
  3%|▎         | 1536/50176 [00:13<07:36, 106.55it/s]
  4%|▎         | 1792/50176 [00:15<07:25, 108.67it/s]
  4%|▍         | 2048/50176 [00:18<07:16, 110.37it/s]
  5%|▍         | 2304/50176 [00:20<07:04, 112.78it/s]
  5%|▌         | 2560/50176 [00:22<07:01, 112.94it/s]
  6%|▌         | 2816/50176 [00:24<07:10, 110.04it/s]
  6%|▌         | 3072/50176 [00:27<07:12, 108.79it/s]
  7%|▋         | 3328/50176 [00:29<07:08, 109.37it/s]
  7%|▋         | 3584/50176 [00:31<06:54, 112.27it/s]
  8%|▊         | 3840/50176 [00:34<07:18, 105.68it/s]
  8%|▊         | 4096/50176 [00:37<07:36, 100.84it/s]
  9%|▊         | 4352/50176 [00:39<07:28, 102.07it/s]
  9%|▉         | 4608/50176 [00:42<07:14, 104.82it/s]
 10%|▉         | 4864/50176 [00:44<07:06, 106.16it/s]
 10%|█         | 5120/50176 [00:46<07:04, 106.18it/s]
 11%|█         | 5376/50176 [00:49<07:09, 104.33it/s]
 11%|█         | 5632/50176 [00:51<07:10, 103.37it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.62it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.88it/s] 
 13%|█▎        | 6400/50176 [01:00<07:31, 97.00it/s]
 13%|█▎        | 6656/50176 [01:02<07:21, 98.56it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.67it/s]
 14%|█▍        | 7168/50176 [01:07<07:14, 98.98it/s]
 15%|█▍        | 7424/50176 [01:10<07:01, 101.32it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.04it/s]
 16%|█▌        | 7936/50176 [01:14<06:47, 103.63it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.05it/s]
 17%|█▋        | 8448/50176 [01:20<06:51, 101.46it/s]
 17%|█▋        | 8704/50176 [01:22<06:47, 101.68it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.82it/s]
 18%|█▊        | 9216/50176 [01:27<06:43, 101.51it/s]
 19%|█▉        | 9472/50176 [01:30<06:41, 101.39it/s]
 19%|█▉        | 9728/50176 [01:32<06:43, 100.12it/s]
 20%|█▉        | 9984/50176 [01:35<06:43, 99.72it/s] 
 20%|██        | 10240/50176 [01:37<06:28, 102.75it/s]
 21%|██        | 10496/50176 [01:40<06:21, 104.07it/s]
 21%|██▏       | 10752/50176 [01:42<06:28, 101.51it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.79it/s]
 22%|██▏       | 11264/50176 [01:47<06:16, 103.22it/s]
 23%|██▎       | 11520/50176 [01:50<06:09, 104.51it/s]
 23%|██▎       | 11776/50176 [01:52<06:06, 104.85it/s]
 24%|██▍       | 12032/50176 [01:54<06:05, 104.27it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.93it/s]
 25%|██▌       | 12544/50176 [01:59<05:49, 107.54it/s]
 26%|██▌       | 12800/50176 [02:02<05:51, 106.22it/s]
 26%|██▌       | 13056/50176 [02:04<05:47, 106.69it/s]
 27%|██▋       | 13312/50176 [02:06<05:48, 105.85it/s]
 27%|██▋       | 13568/50176 [02:09<06:05, 100.23it/s]
 28%|██▊       | 13824/50176 [02:12<06:08, 98.77it/s] 
 28%|██▊       | 14080/50176 [02:14<06:00, 100.11it/s]
 29%|██▊       | 14336/50176 [02:17<05:57, 100.20it/s]
 29%|██▉       | 14592/50176 [02:19<05:50, 101.59it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.66it/s] 
 30%|███       | 15104/50176 [02:25<05:55, 98.70it/s]
 31%|███       | 15360/50176 [02:27<05:47, 100.13it/s]
 31%|███       | 15616/50176 [02:30<05:37, 102.28it/s]
 32%|███▏      | 15872/50176 [02:32<05:39, 101.18it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.89it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.62it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.06it/s] 
 34%|███▎      | 16896/50176 [02:42<05:30, 100.59it/s]
 34%|███▍      | 17152/50176 [02:45<05:23, 102.23it/s]
 35%|███▍      | 17408/50176 [02:47<05:17, 103.13it/s]
 35%|███▌      | 17664/50176 [02:49<05:03, 107.05it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 108.98it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.76it/s]
 37%|███▋      | 18432/50176 [02:56<04:54, 107.65it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.57it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.67it/s]
 38%|███▊      | 19200/50176 [03:04<05:09, 99.93it/s] 
 39%|███▉      | 19456/50176 [03:07<05:12, 98.24it/s]
 39%|███▉      | 19712/50176 [03:10<05:13, 97.23it/s]
 40%|███▉      | 19968/50176 [03:12<05:11, 96.95it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.37it/s]
 41%|████      | 20480/50176 [03:17<04:48, 103.11it/s]
 41%|████▏     | 20736/50176 [03:19<04:42, 104.37it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.82it/s]
 42%|████▏     | 21248/50176 [03:25<04:41, 102.60it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 103.09it/s]
 43%|████▎     | 21760/50176 [03:29<04:30, 105.19it/s]
 44%|████▍     | 22016/50176 [03:32<04:20, 108.30it/s]
 44%|████▍     | 22272/50176 [03:34<04:17, 108.19it/s]
 45%|████▍     | 22528/50176 [03:36<04:19, 106.40it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.29it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.27it/s] 
 46%|████▋     | 23296/50176 [03:45<04:32, 98.47it/s]
 47%|████▋     | 23552/50176 [03:50<05:45, 77.16it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.02it/s]
 48%|████▊     | 24064/50176 [03:55<04:56, 88.13it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.88it/s]
 49%|████▉     | 24576/50176 [04:00<04:32, 93.92it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.92it/s]
 50%|█████     | 25088/50176 [04:04<04:06, 101.81it/s]
 51%|█████     | 25344/50176 [04:07<03:56, 104.96it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.34it/s]
 52%|█████▏    | 25856/50176 [04:11<03:46, 107.17it/s]
 52%|█████▏    | 26112/50176 [04:14<03:43, 107.51it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.18it/s]
 53%|█████▎    | 26624/50176 [04:18<03:38, 107.85it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.27it/s]
 54%|█████▍    | 27136/50176 [04:23<03:41, 104.05it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.75it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.57it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.64it/s]
 56%|█████▌    | 28160/50176 [04:35<04:00, 91.56it/s]
 57%|█████▋    | 28416/50176 [04:38<03:52, 93.63it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 93.94it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.45it/s]
 58%|█████▊    | 29184/50176 [04:45<03:30, 99.87it/s]
 59%|█████▊    | 29440/50176 [04:48<03:36, 95.62it/s]
 59%|█████▉    | 29696/50176 [04:51<03:34, 95.69it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.61it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.79it/s]
 61%|██████    | 30464/50176 [04:58<03:10, 103.24it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.61it/s]
 62%|██████▏   | 30976/50176 [05:03<03:02, 105.01it/s]
 62%|██████▏   | 31232/50176 [05:06<02:59, 105.45it/s]
 63%|██████▎   | 31488/50176 [05:08<02:52, 108.61it/s]
 63%|██████▎   | 31744/50176 [05:10<02:50, 107.83it/s]
 64%|██████▍   | 32000/50176 [05:13<02:48, 107.94it/s]
 64%|██████▍   | 32256/50176 [05:15<02:43, 109.84it/s]
 65%|██████▍   | 32512/50176 [05:17<02:47, 105.42it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.20it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.51it/s]
 66%|██████▋   | 33280/50176 [05:24<02:36, 107.95it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.61it/s]
 67%|██████▋   | 33792/50176 [05:29<02:32, 107.72it/s]
 68%|██████▊   | 34048/50176 [05:31<02:24, 111.47it/s]
 68%|██████▊   | 34304/50176 [05:33<02:19, 113.92it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.33it/s]
 69%|██████▉   | 34816/50176 [05:39<02:23, 107.21it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.06it/s]
 70%|███████   | 35328/50176 [05:43<02:17, 108.28it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 107.92it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.76it/s]
 72%|███████▏  | 36096/50176 [05:51<02:11, 107.40it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.45it/s]
 73%|███████▎  | 36608/50176 [05:56<02:09, 104.57it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.98it/s]
 74%|███████▍  | 37120/50176 [06:01<02:05, 104.05it/s]
 74%|███████▍  | 37376/50176 [06:03<01:56, 109.52it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.65it/s]
 76%|███████▌  | 37888/50176 [06:07<01:53, 108.00it/s]
 76%|███████▌  | 38144/50176 [06:10<01:51, 108.33it/s]
 77%|███████▋  | 38400/50176 [06:12<01:49, 107.33it/s]
 77%|███████▋  | 38656/50176 [06:15<01:47, 106.68it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.49it/s]
 78%|███████▊  | 39168/50176 [06:19<01:44, 105.63it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.14it/s]
 79%|███████▉  | 39680/50176 [06:24<01:38, 106.64it/s]
 80%|███████▉  | 39936/50176 [06:27<01:34, 107.86it/s]
 80%|████████  | 40192/50176 [06:29<01:34, 106.19it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.85it/s]
 81%|████████  | 40704/50176 [06:34<01:33, 101.16it/s]
 82%|████████▏ | 40960/50176 [06:37<01:27, 105.51it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.40it/s]
 83%|████████▎ | 41472/50176 [06:41<01:22, 105.55it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.36it/s]
 84%|████████▎ | 41984/50176 [06:46<01:18, 104.80it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.98it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.39it/s]
 85%|████████▌ | 42752/50176 [06:53<01:08, 108.28it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.81it/s]
 86%|████████▌ | 43264/50176 [06:58<01:05, 105.53it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.29it/s]
 87%|████████▋ | 43776/50176 [07:03<01:02, 102.79it/s]
 88%|████████▊ | 44032/50176 [07:06<00:59, 104.04it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.92it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 101.06it/s]
 89%|████████▉ | 44800/50176 [07:13<00:52, 102.46it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.09it/s]
 90%|█████████ | 45312/50176 [07:18<00:47, 102.51it/s]
 91%|█████████ | 45568/50176 [07:21<00:43, 106.03it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.93it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.07it/s]
 92%|█████████▏| 46336/50176 [07:28<00:37, 101.22it/s]
 93%|█████████▎| 46592/50176 [07:31<00:34, 105.22it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.18it/s]
 94%|█████████▍| 47104/50176 [07:36<00:30, 101.93it/s]
 94%|█████████▍| 47360/50176 [07:38<00:28, 100.20it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.33it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.63it/s]
 96%|█████████▌| 48128/50176 [07:45<00:19, 104.59it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.18it/s]
 97%|█████████▋| 48640/50176 [07:50<00:14, 104.61it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.50it/s]
 98%|█████████▊| 49152/50176 [07:55<00:09, 103.33it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.29it/s]
 99%|█████████▉| 49664/50176 [08:00<00:05, 102.33it/s]
 99%|█████████▉| 49920/50176 [08:04<00:02, 95.33it/s] 
-2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5455	imagenet-zeroshot-val-top5: 0.8192

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_14.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt).
-2022-12-21,09:46:05 | INFO | Model:
-2022-12-21,09:46:05 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:46:05 | INFO | Params:
-2022-12-21,09:46:05 | INFO |   batch_size: 256
-2022-12-21,09:46:05 | INFO |   beta1: 0.9
-2022-12-21,09:46:05 | INFO |   beta2: 0.98
-2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/checkpoints
-2022-12-21,09:46:05 | INFO |   copy_codebase: False
-2022-12-21,09:46:05 | INFO |   csv_caption_key: title
-2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
-2022-12-21,09:46:05 | INFO |   csv_separator: 	
-2022-12-21,09:46:05 | INFO |   dataset_resampled: False
-2022-12-21,09:46:05 | INFO |   dataset_type: auto
-2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
-2022-12-21,09:46:05 | INFO |   debug: False
-2022-12-21,09:46:05 | INFO |   device: cuda:0
-2022-12-21,09:46:05 | INFO |   dist_backend: nccl
-2022-12-21,09:46:05 | INFO |   dist_url: env://
-2022-12-21,09:46:05 | INFO |   distributed: False
-2022-12-21,09:46:05 | INFO |   epochs: 32
-2022-12-21,09:46:05 | INFO |   eps: 1e-06
-2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
-2022-12-21,09:46:05 | INFO |   gather_with_grad: False
-2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
-2022-12-21,09:46:05 | INFO |   horovod: False
-2022-12-21,09:46:05 | INFO |   image_mean: None
-2022-12-21,09:46:05 | INFO |   image_std: None
-2022-12-21,09:46:05 | INFO |   imagenet_v2: None
-2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:46:05 | INFO |   local_loss: False
-2022-12-21,09:46:05 | INFO |   local_rank: 0
-2022-12-21,09:46:05 | INFO |   lock_image: False
-2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:46:05 | INFO |   log_level: 20
-2022-12-21,09:46:05 | INFO |   log_local: False
-2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_14.pt/out.log
-2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:46:05 | INFO |   lr: 0.0005
-2022-12-21,09:46:05 | INFO |   model: ViT-B-16
-2022-12-21,09:46:05 | INFO |   name: epoch_14.pt
-2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
-2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
-2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
-2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_14.pt
-2022-12-21,09:46:05 | INFO |   pretrained_image: False
-2022-12-21,09:46:05 | INFO |   rank: 0
-2022-12-21,09:46:05 | INFO |   report_to: 
-2022-12-21,09:46:05 | INFO |   resume: None
-2022-12-21,09:46:05 | INFO |   save_frequency: 1
-2022-12-21,09:46:05 | INFO |   save_most_recent: False
-2022-12-21,09:46:05 | INFO |   seed: 0
-2022-12-21,09:46:05 | INFO |   skip_scheduler: False
-2022-12-21,09:46:05 | INFO |   tensorboard: False
-2022-12-21,09:46:05 | INFO |   tensorboard_path: 
-2022-12-21,09:46:05 | INFO |   torchscript: False
-2022-12-21,09:46:05 | INFO |   trace: False
-2022-12-21,09:46:05 | INFO |   train_data: None
-2022-12-21,09:46:05 | INFO |   train_num_samples: None
-2022-12-21,09:46:05 | INFO |   use_bn_sync: False
-2022-12-21,09:46:05 | INFO |   val_data: None
-2022-12-21,09:46:05 | INFO |   val_frequency: 1
-2022-12-21,09:46:05 | INFO |   val_num_samples: None
-2022-12-21,09:46:05 | INFO |   wandb: False
-2022-12-21,09:46:05 | INFO |   wandb_notes: 
-2022-12-21,09:46:05 | INFO |   warmup: 10000
-2022-12-21,09:46:05 | INFO |   wd: 0.2
-2022-12-21,09:46:05 | INFO |   workers: 1
-2022-12-21,09:46:05 | INFO |   world_size: 1
-2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:46:05 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:59,  1.51it/s]
  1%|          | 6/1000 [00:00<01:38, 10.13it/s]
  1%|          | 12/1000 [00:00<00:50, 19.61it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.22it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.35it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.08it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.43it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.70it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.13it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.87it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.12it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.07it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.69it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.12it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.60it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.88it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.23it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.38it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.36it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.40it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.42it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.39it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.30it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.17it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.34it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.42it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.41it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.14it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.16it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.11it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.15it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.16it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.11it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.11it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.91it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.85it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.00it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.95it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.16it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.13it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.03it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.13it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.22it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.34it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.40it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.22it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.25it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.42it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.73it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.65it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.63it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.78it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.74it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.58it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.65it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.74it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.84it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.64it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.61it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.67it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.77it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.64it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.33it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.30it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.42it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.55it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.67it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.67it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.70it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.73it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.87it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.87it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.67it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.70it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.75it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.72it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.69it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.80it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.70it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.67it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.66it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.49it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.57it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.47it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.56it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.63it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.74it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.67it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.71it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.75it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.66it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.67it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.73it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.59it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.54it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.69it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.68it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.61it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.73it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.64it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.65it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.76it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.76it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.82it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.48it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.47it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.66it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.64it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.74it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.78it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.71it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.75it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.77it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.80it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.69it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.77it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.76it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.72it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.76it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.72it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.76it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.75it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.85it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.92it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.74it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.84it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.79it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.82it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.84it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.62it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.57it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.65it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.63it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.42it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.51it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.66it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.83it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.90it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.83it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.89it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.84it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.88it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.82it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.87it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 57.85it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.68it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.77it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.76it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.72it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.76it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.74it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.74it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.83it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.86it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.86it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.86it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.91it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.83it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.91it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.70it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.74it/s]
-2022-12-21,09:46:23 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:09, 134.94it/s]
  1%|          | 512/50176 [00:03<06:26, 128.53it/s]
  2%|▏         | 768/50176 [00:06<06:39, 123.59it/s]
  2%|▏         | 1024/50176 [00:08<06:45, 121.34it/s]
  3%|▎         | 1280/50176 [00:10<07:04, 115.16it/s]
  3%|▎         | 1536/50176 [00:13<07:35, 106.67it/s]
  4%|▎         | 1792/50176 [00:15<07:24, 108.76it/s]
  4%|▍         | 2048/50176 [00:17<07:15, 110.46it/s]
  5%|▍         | 2304/50176 [00:20<07:04, 112.67it/s]
  5%|▌         | 2560/50176 [00:22<07:01, 113.05it/s]
  6%|▌         | 2816/50176 [00:24<07:08, 110.48it/s]
  6%|▌         | 3072/50176 [00:27<07:12, 108.87it/s]
  7%|▋         | 3328/50176 [00:29<07:08, 109.30it/s]
  7%|▋         | 3584/50176 [00:31<06:54, 112.39it/s]
  8%|▊         | 3840/50176 [00:34<07:19, 105.50it/s]
  8%|▊         | 4096/50176 [00:37<07:37, 100.78it/s]
  9%|▊         | 4352/50176 [00:39<07:29, 101.93it/s]
  9%|▉         | 4608/50176 [00:41<07:13, 105.04it/s]
 10%|▉         | 4864/50176 [00:44<07:07, 106.05it/s]
 10%|█         | 5120/50176 [00:46<07:04, 106.17it/s]
 11%|█         | 5376/50176 [00:49<07:08, 104.62it/s]
 11%|█         | 5632/50176 [00:51<07:11, 103.30it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.67it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.93it/s] 
 13%|█▎        | 6400/50176 [00:59<07:31, 97.05it/s]
 13%|█▎        | 6656/50176 [01:02<07:22, 98.46it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.67it/s]
 14%|█▍        | 7168/50176 [01:07<07:14, 99.02it/s]
 15%|█▍        | 7424/50176 [01:09<07:02, 101.29it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.10it/s]
 16%|█▌        | 7936/50176 [01:14<06:47, 103.60it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.09it/s]
 17%|█▋        | 8448/50176 [01:19<06:51, 101.50it/s]
 17%|█▋        | 8704/50176 [01:22<06:48, 101.48it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.79it/s]
 18%|█▊        | 9216/50176 [01:27<06:44, 101.37it/s]
 19%|█▉        | 9472/50176 [01:30<06:41, 101.44it/s]
 19%|█▉        | 9728/50176 [01:32<06:43, 100.15it/s]
 20%|█▉        | 9984/50176 [01:35<06:42, 99.87it/s] 
 20%|██        | 10240/50176 [01:37<06:29, 102.62it/s]
 21%|██        | 10496/50176 [01:39<06:21, 104.06it/s]
 21%|██▏       | 10752/50176 [01:42<06:29, 101.31it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.85it/s]
 22%|██▏       | 11264/50176 [01:47<06:16, 103.36it/s]
 23%|██▎       | 11520/50176 [01:49<06:10, 104.43it/s]
 23%|██▎       | 11776/50176 [01:52<06:06, 104.89it/s]
 24%|██▍       | 12032/50176 [01:54<06:05, 104.41it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.88it/s]
 25%|██▌       | 12544/50176 [01:59<05:49, 107.56it/s]
 26%|██▌       | 12800/50176 [02:01<05:51, 106.31it/s]
 26%|██▌       | 13056/50176 [02:04<05:47, 106.71it/s]
 27%|██▋       | 13312/50176 [02:06<05:47, 105.95it/s]
 27%|██▋       | 13568/50176 [02:09<06:05, 100.11it/s]
 28%|██▊       | 13824/50176 [02:12<06:08, 98.63it/s] 
 28%|██▊       | 14080/50176 [02:14<06:00, 100.18it/s]
 29%|██▊       | 14336/50176 [02:17<05:57, 100.14it/s]
 29%|██▉       | 14592/50176 [02:19<05:50, 101.48it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.77it/s] 
 30%|███       | 15104/50176 [02:25<05:54, 98.80it/s]
 31%|███       | 15360/50176 [02:27<05:47, 100.11it/s]
 31%|███       | 15616/50176 [02:29<05:37, 102.36it/s]
 32%|███▏      | 15872/50176 [02:32<05:38, 101.20it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.87it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.52it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.05it/s] 
 34%|███▎      | 16896/50176 [02:42<05:31, 100.45it/s]
 34%|███▍      | 17152/50176 [02:45<05:22, 102.29it/s]
 35%|███▍      | 17408/50176 [02:47<05:17, 103.10it/s]
 35%|███▌      | 17664/50176 [02:49<05:03, 107.07it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 108.98it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.83it/s]
 37%|███▋      | 18432/50176 [02:56<04:54, 107.67it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.42it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.70it/s]
 38%|███▊      | 19200/50176 [03:04<05:09, 100.02it/s]
 39%|███▉      | 19456/50176 [03:07<05:12, 98.35it/s] 
 39%|███▉      | 19712/50176 [03:10<05:13, 97.13it/s]
 40%|███▉      | 19968/50176 [03:12<05:11, 96.85it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.39it/s]
 41%|████      | 20480/50176 [03:17<04:48, 103.02it/s]
 41%|████▏     | 20736/50176 [03:19<04:42, 104.36it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.83it/s]
 42%|████▏     | 21248/50176 [03:24<04:41, 102.60it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 103.03it/s]
 43%|████▎     | 21760/50176 [03:29<04:29, 105.42it/s]
 44%|████▍     | 22016/50176 [03:31<04:19, 108.40it/s]
 44%|████▍     | 22272/50176 [03:34<04:18, 108.15it/s]
 45%|████▍     | 22528/50176 [03:36<04:19, 106.52it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.33it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.16it/s] 
 46%|████▋     | 23296/50176 [03:44<04:33, 98.35it/s]
 47%|████▋     | 23552/50176 [03:49<05:44, 77.17it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.03it/s]
 48%|████▊     | 24064/50176 [03:54<04:55, 88.23it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.81it/s]
 49%|████▉     | 24576/50176 [03:59<04:32, 93.94it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.96it/s]
 50%|█████     | 25088/50176 [04:04<04:06, 101.79it/s]
 51%|█████     | 25344/50176 [04:06<03:56, 104.94it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.33it/s]
 52%|█████▏    | 25856/50176 [04:11<03:46, 107.21it/s]
 52%|█████▏    | 26112/50176 [04:13<03:44, 107.37it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.23it/s]
 53%|█████▎    | 26624/50176 [04:18<03:38, 107.89it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.14it/s]
 54%|█████▍    | 27136/50176 [04:23<03:41, 104.07it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.72it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.57it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.50it/s]
 56%|█████▌    | 28160/50176 [04:35<04:00, 91.52it/s]
 57%|█████▋    | 28416/50176 [04:38<03:52, 93.77it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 94.01it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.48it/s]
 58%|█████▊    | 29184/50176 [04:45<03:30, 99.85it/s]
 59%|█████▊    | 29440/50176 [04:48<03:36, 95.73it/s]
 59%|█████▉    | 29696/50176 [04:51<03:34, 95.69it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.72it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.76it/s]
 61%|██████    | 30464/50176 [04:58<03:10, 103.21it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.62it/s]
 62%|██████▏   | 30976/50176 [05:03<03:02, 104.98it/s]
 62%|██████▏   | 31232/50176 [05:05<02:59, 105.43it/s]
 63%|██████▎   | 31488/50176 [05:08<02:51, 108.71it/s]
 63%|██████▎   | 31744/50176 [05:10<02:50, 107.96it/s]
 64%|██████▍   | 32000/50176 [05:12<02:48, 107.91it/s]
 64%|██████▍   | 32256/50176 [05:15<02:43, 109.83it/s]
 65%|██████▍   | 32512/50176 [05:17<02:47, 105.47it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.23it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.54it/s]
 66%|██████▋   | 33280/50176 [05:24<02:36, 107.64it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.59it/s]
 67%|██████▋   | 33792/50176 [05:29<02:32, 107.61it/s]
 68%|██████▊   | 34048/50176 [05:31<02:24, 111.59it/s]
 68%|██████▊   | 34304/50176 [05:33<02:19, 113.81it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.24it/s]
 69%|██████▉   | 34816/50176 [05:38<02:23, 107.19it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.10it/s]
 70%|███████   | 35328/50176 [05:43<02:17, 108.33it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 108.03it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.74it/s]
 72%|███████▏  | 36096/50176 [05:50<02:11, 107.48it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.47it/s]
 73%|███████▎  | 36608/50176 [05:55<02:09, 104.72it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.81it/s]
 74%|███████▍  | 37120/50176 [06:00<02:05, 103.87it/s]
 74%|███████▍  | 37376/50176 [06:02<01:56, 109.54it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.82it/s]
 76%|███████▌  | 37888/50176 [06:07<01:53, 107.89it/s]
 76%|███████▌  | 38144/50176 [06:10<01:51, 108.29it/s]
 77%|███████▋  | 38400/50176 [06:12<01:49, 107.26it/s]
 77%|███████▋  | 38656/50176 [06:15<01:48, 106.64it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.51it/s]
 78%|███████▊  | 39168/50176 [06:19<01:44, 105.57it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.32it/s]
 79%|███████▉  | 39680/50176 [06:24<01:38, 106.78it/s]
 80%|███████▉  | 39936/50176 [06:26<01:34, 107.85it/s]
 80%|████████  | 40192/50176 [06:29<01:33, 106.26it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.81it/s]
 81%|████████  | 40704/50176 [06:34<01:33, 101.17it/s]
 82%|████████▏ | 40960/50176 [06:36<01:27, 105.60it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.24it/s]
 83%|████████▎ | 41472/50176 [06:41<01:22, 105.66it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.30it/s]
 84%|████████▎ | 41984/50176 [06:46<01:18, 104.67it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.87it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.39it/s]
 85%|████████▌ | 42752/50176 [06:53<01:08, 108.34it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.90it/s]
 86%|████████▌ | 43264/50176 [06:58<01:05, 105.68it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.21it/s]
 87%|████████▋ | 43776/50176 [07:03<01:02, 102.71it/s]
 88%|████████▊ | 44032/50176 [07:06<00:59, 104.05it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.89it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 101.01it/s]
 89%|████████▉ | 44800/50176 [07:13<00:52, 102.41it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.25it/s]
 90%|█████████ | 45312/50176 [07:18<00:47, 102.62it/s]
 91%|█████████ | 45568/50176 [07:20<00:43, 106.11it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.85it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.18it/s]
 92%|█████████▏| 46336/50176 [07:28<00:37, 101.07it/s]
 93%|█████████▎| 46592/50176 [07:30<00:34, 105.08it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.00it/s]
 94%|█████████▍| 47104/50176 [07:35<00:30, 102.11it/s]
 94%|█████████▍| 47360/50176 [07:38<00:28, 100.22it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.34it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.69it/s]
 96%|█████████▌| 48128/50176 [07:45<00:19, 104.57it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.11it/s]
 97%|█████████▋| 48640/50176 [07:50<00:14, 104.75it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.40it/s]
 98%|█████████▊| 49152/50176 [07:55<00:09, 103.41it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.11it/s]
 99%|█████████▉| 49664/50176 [08:00<00:04, 102.46it/s]
 99%|█████████▉| 49920/50176 [08:03<00:02, 95.31it/s] 
-2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5424	imagenet-zeroshot-val-top5: 0.8184

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_15.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt).
-2022-12-21,09:46:05 | INFO | Model:
-2022-12-21,09:46:05 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:46:05 | INFO | Params:
-2022-12-21,09:46:05 | INFO |   batch_size: 256
-2022-12-21,09:46:05 | INFO |   beta1: 0.9
-2022-12-21,09:46:05 | INFO |   beta2: 0.98
-2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/checkpoints
-2022-12-21,09:46:05 | INFO |   copy_codebase: False
-2022-12-21,09:46:05 | INFO |   csv_caption_key: title
-2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
-2022-12-21,09:46:05 | INFO |   csv_separator: 	
-2022-12-21,09:46:05 | INFO |   dataset_resampled: False
-2022-12-21,09:46:05 | INFO |   dataset_type: auto
-2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
-2022-12-21,09:46:05 | INFO |   debug: False
-2022-12-21,09:46:05 | INFO |   device: cuda:0
-2022-12-21,09:46:05 | INFO |   dist_backend: nccl
-2022-12-21,09:46:05 | INFO |   dist_url: env://
-2022-12-21,09:46:05 | INFO |   distributed: False
-2022-12-21,09:46:05 | INFO |   epochs: 32
-2022-12-21,09:46:05 | INFO |   eps: 1e-06
-2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
-2022-12-21,09:46:05 | INFO |   gather_with_grad: False
-2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
-2022-12-21,09:46:05 | INFO |   horovod: False
-2022-12-21,09:46:05 | INFO |   image_mean: None
-2022-12-21,09:46:05 | INFO |   image_std: None
-2022-12-21,09:46:05 | INFO |   imagenet_v2: None
-2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:46:05 | INFO |   local_loss: False
-2022-12-21,09:46:05 | INFO |   local_rank: 0
-2022-12-21,09:46:05 | INFO |   lock_image: False
-2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:46:05 | INFO |   log_level: 20
-2022-12-21,09:46:05 | INFO |   log_local: False
-2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_15.pt/out.log
-2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:46:05 | INFO |   lr: 0.0005
-2022-12-21,09:46:05 | INFO |   model: ViT-B-16
-2022-12-21,09:46:05 | INFO |   name: epoch_15.pt
-2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
-2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
-2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
-2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_15.pt
-2022-12-21,09:46:05 | INFO |   pretrained_image: False
-2022-12-21,09:46:05 | INFO |   rank: 0
-2022-12-21,09:46:05 | INFO |   report_to: 
-2022-12-21,09:46:05 | INFO |   resume: None
-2022-12-21,09:46:05 | INFO |   save_frequency: 1
-2022-12-21,09:46:05 | INFO |   save_most_recent: False
-2022-12-21,09:46:05 | INFO |   seed: 0
-2022-12-21,09:46:05 | INFO |   skip_scheduler: False
-2022-12-21,09:46:05 | INFO |   tensorboard: False
-2022-12-21,09:46:05 | INFO |   tensorboard_path: 
-2022-12-21,09:46:05 | INFO |   torchscript: False
-2022-12-21,09:46:05 | INFO |   trace: False
-2022-12-21,09:46:05 | INFO |   train_data: None
-2022-12-21,09:46:05 | INFO |   train_num_samples: None
-2022-12-21,09:46:05 | INFO |   use_bn_sync: False
-2022-12-21,09:46:05 | INFO |   val_data: None
-2022-12-21,09:46:05 | INFO |   val_frequency: 1
-2022-12-21,09:46:05 | INFO |   val_num_samples: None
-2022-12-21,09:46:05 | INFO |   wandb: False
-2022-12-21,09:46:05 | INFO |   wandb_notes: 
-2022-12-21,09:46:05 | INFO |   warmup: 10000
-2022-12-21,09:46:05 | INFO |   wd: 0.2
-2022-12-21,09:46:05 | INFO |   workers: 1
-2022-12-21,09:46:05 | INFO |   world_size: 1
-2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:46:05 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:27,  1.45it/s]
  1%|          | 7/1000 [00:00<01:29, 11.16it/s]
  1%|▏         | 13/1000 [00:00<00:48, 20.28it/s]
  2%|▏         | 19/1000 [00:01<00:34, 28.59it/s]
  2%|▎         | 25/1000 [00:01<00:27, 35.55it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.26it/s]
  4%|▎         | 37/1000 [00:01<00:21, 45.66it/s]
  4%|▍         | 43/1000 [00:01<00:19, 48.78it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.04it/s]
  6%|▌         | 55/1000 [00:01<00:17, 52.84it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.31it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.29it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.00it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.60it/s]
  8%|▊         | 85/1000 [00:02<00:16, 57.09it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.28it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.68it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.87it/s]
 11%|█         | 109/1000 [00:02<00:15, 57.96it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 57.90it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.92it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.00it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.06it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.79it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 57.95it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.06it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.08it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.11it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.92it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.89it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.84it/s]
 19%|█▉        | 193/1000 [00:04<00:13, 57.80it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.74it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.60it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.40it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.61it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.83it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.82it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.63it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.76it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.83it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.93it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.90it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.74it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.89it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.81it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 57.94it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.14it/s]
 30%|███       | 301/1000 [00:05<00:12, 58.16it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.15it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.35it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.39it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.23it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.02it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.20it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.29it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.25it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 58.18it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.24it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.29it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.34it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.12it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.95it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 58.01it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.07it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.85it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.03it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.16it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.22it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.25it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.15it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.24it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.18it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.24it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.30it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.25it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.40it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.20it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.31it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.27it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.18it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.23it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.13it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.26it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.24it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.30it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.34it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.37it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.39it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.29it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.34it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.14it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.20it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.29it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.30it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.22it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.32it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.19it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.22it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.37it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.34it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.33it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.26it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.23it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.30it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.29it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.37it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.35it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.39it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.37it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.30it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.31it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.32it/s]
 72%|███████▏  | 715/1000 [00:12<00:04, 58.30it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.31it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.28it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.29it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.34it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.36it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.43it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.46it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.27it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.35it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.35it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.38it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.37it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.23it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.23it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.17it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.22it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.19it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.31it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.47it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.51it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.37it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.40it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.40it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.36it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.39it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.34it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.24it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.34it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.31it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.28it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.36it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.33it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.43it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.48it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.43it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.40it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.55it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.42it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.54it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.65it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.64it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.58it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.46it/s]
-2022-12-21,09:46:23 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:36, 109.39it/s]
  1%|          | 512/50176 [00:04<07:01, 117.87it/s]
  2%|▏         | 768/50176 [00:06<06:58, 117.94it/s]
  2%|▏         | 1024/50176 [00:08<06:57, 117.62it/s]
  3%|▎         | 1280/50176 [00:11<07:11, 113.30it/s]
  3%|▎         | 1536/50176 [00:13<07:41, 105.31it/s]
  4%|▎         | 1792/50176 [00:16<07:28, 107.87it/s]
  4%|▍         | 2048/50176 [00:18<07:17, 110.04it/s]
  5%|▍         | 2304/50176 [00:20<07:06, 112.21it/s]
  5%|▌         | 2560/50176 [00:22<07:02, 112.70it/s]
  6%|▌         | 2816/50176 [00:25<07:09, 110.17it/s]
  6%|▌         | 3072/50176 [00:27<07:13, 108.72it/s]
  7%|▋         | 3328/50176 [00:30<07:08, 109.35it/s]
  7%|▋         | 3584/50176 [00:32<06:54, 112.38it/s]
  8%|▊         | 3840/50176 [00:34<07:19, 105.43it/s]
  8%|▊         | 4096/50176 [00:37<07:37, 100.75it/s]
  9%|▊         | 4352/50176 [00:40<07:29, 101.95it/s]
  9%|▉         | 4608/50176 [00:42<07:14, 104.96it/s]
 10%|▉         | 4864/50176 [00:44<07:07, 106.07it/s]
 10%|█         | 5120/50176 [00:47<07:03, 106.32it/s]
 11%|█         | 5376/50176 [00:49<07:09, 104.42it/s]
 11%|█         | 5632/50176 [00:52<07:10, 103.35it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.62it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.94it/s] 
 13%|█▎        | 6400/50176 [01:00<07:30, 97.11it/s]
 13%|█▎        | 6656/50176 [01:02<07:21, 98.58it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.71it/s]
 14%|█▍        | 7168/50176 [01:08<07:14, 98.91it/s]
 15%|█▍        | 7424/50176 [01:10<07:02, 101.29it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.02it/s]
 16%|█▌        | 7936/50176 [01:15<06:47, 103.61it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.06it/s]
 17%|█▋        | 8448/50176 [01:20<06:51, 101.52it/s]
 17%|█▋        | 8704/50176 [01:22<06:48, 101.49it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.90it/s]
 18%|█▊        | 9216/50176 [01:27<06:43, 101.39it/s]
 19%|█▉        | 9472/50176 [01:30<06:40, 101.53it/s]
 19%|█▉        | 9728/50176 [01:33<06:43, 100.17it/s]
 20%|█▉        | 9984/50176 [01:35<06:42, 99.84it/s] 
 20%|██        | 10240/50176 [01:38<06:29, 102.54it/s]
 21%|██        | 10496/50176 [01:40<06:20, 104.28it/s]
 21%|██▏       | 10752/50176 [01:43<06:29, 101.33it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.75it/s]
 22%|██▏       | 11264/50176 [01:47<06:16, 103.22it/s]
 23%|██▎       | 11520/50176 [01:50<06:09, 104.52it/s]
 23%|██▎       | 11776/50176 [01:52<06:05, 105.06it/s]
 24%|██▍       | 12032/50176 [01:55<06:05, 104.44it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.82it/s]
 25%|██▌       | 12544/50176 [01:59<05:49, 107.55it/s]
 26%|██▌       | 12800/50176 [02:02<05:51, 106.22it/s]
 26%|██▌       | 13056/50176 [02:04<05:48, 106.58it/s]
 27%|██▋       | 13312/50176 [02:07<05:48, 105.79it/s]
 27%|██▋       | 13568/50176 [02:10<06:05, 100.27it/s]
 28%|██▊       | 13824/50176 [02:12<06:07, 98.80it/s] 
 28%|██▊       | 14080/50176 [02:15<06:00, 100.03it/s]
 29%|██▊       | 14336/50176 [02:17<05:58, 100.10it/s]
 29%|██▉       | 14592/50176 [02:20<05:50, 101.48it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.68it/s] 
 30%|███       | 15104/50176 [02:25<05:55, 98.74it/s]
 31%|███       | 15360/50176 [02:28<05:47, 100.18it/s]
 31%|███       | 15616/50176 [02:30<05:37, 102.31it/s]
 32%|███▏      | 15872/50176 [02:32<05:39, 101.16it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.91it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.56it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.15it/s] 
 34%|███▎      | 16896/50176 [02:43<05:30, 100.61it/s]
 34%|███▍      | 17152/50176 [02:45<05:22, 102.32it/s]
 35%|███▍      | 17408/50176 [02:48<05:17, 103.14it/s]
 35%|███▌      | 17664/50176 [02:50<05:03, 107.06it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 109.03it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.73it/s]
 37%|███▋      | 18432/50176 [02:57<04:54, 107.74it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.54it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.65it/s]
 38%|███▊      | 19200/50176 [03:05<05:10, 99.88it/s] 
 39%|███▉      | 19456/50176 [03:07<05:12, 98.16it/s]
 39%|███▉      | 19712/50176 [03:10<05:13, 97.24it/s]
 40%|███▉      | 19968/50176 [03:13<05:11, 96.84it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.33it/s]
 41%|████      | 20480/50176 [03:17<04:47, 103.17it/s]
 41%|████▏     | 20736/50176 [03:20<04:42, 104.36it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.85it/s]
 42%|████▏     | 21248/50176 [03:25<04:41, 102.69it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 102.93it/s]
 43%|████▎     | 21760/50176 [03:30<04:30, 105.24it/s]
 44%|████▍     | 22016/50176 [03:32<04:19, 108.34it/s]
 44%|████▍     | 22272/50176 [03:34<04:17, 108.29it/s]
 45%|████▍     | 22528/50176 [03:37<04:19, 106.38it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.41it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.27it/s] 
 46%|████▋     | 23296/50176 [03:45<04:33, 98.30it/s]
 47%|████▋     | 23552/50176 [03:50<05:44, 77.24it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.05it/s]
 48%|████▊     | 24064/50176 [03:55<04:56, 88.08it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.81it/s]
 49%|████▉     | 24576/50176 [04:00<04:32, 93.99it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.98it/s]
 50%|█████     | 25088/50176 [04:05<04:06, 101.84it/s]
 51%|█████     | 25344/50176 [04:07<03:56, 104.89it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.28it/s]
 52%|█████▏    | 25856/50176 [04:12<03:46, 107.15it/s]
 52%|█████▏    | 26112/50176 [04:14<03:44, 107.33it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.22it/s]
 53%|█████▎    | 26624/50176 [04:19<03:38, 107.97it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.17it/s]
 54%|█████▍    | 27136/50176 [04:24<03:41, 104.16it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.66it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.59it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.54it/s]
 56%|█████▌    | 28160/50176 [04:36<04:00, 91.52it/s]
 57%|█████▋    | 28416/50176 [04:38<03:52, 93.71it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 94.05it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.45it/s]
 58%|█████▊    | 29184/50176 [04:46<03:30, 99.91it/s]
 59%|█████▊    | 29440/50176 [04:49<03:36, 95.63it/s]
 59%|█████▉    | 29696/50176 [04:51<03:33, 95.86it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.58it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.84it/s]
 61%|██████    | 30464/50176 [04:59<03:10, 103.40it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.59it/s]
 62%|██████▏   | 30976/50176 [05:04<03:03, 104.87it/s]
 62%|██████▏   | 31232/50176 [05:06<02:59, 105.56it/s]
 63%|██████▎   | 31488/50176 [05:08<02:52, 108.53it/s]
 63%|██████▎   | 31744/50176 [05:11<02:50, 107.80it/s]
 64%|██████▍   | 32000/50176 [05:13<02:48, 107.86it/s]
 64%|██████▍   | 32256/50176 [05:15<02:42, 110.02it/s]
 65%|██████▍   | 32512/50176 [05:18<02:47, 105.37it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.18it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.64it/s]
 66%|██████▋   | 33280/50176 [05:25<02:36, 107.65it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.56it/s]
 67%|██████▋   | 33792/50176 [05:30<02:32, 107.62it/s]
 68%|██████▊   | 34048/50176 [05:32<02:24, 111.54it/s]
 68%|██████▊   | 34304/50176 [05:34<02:19, 113.79it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.15it/s]
 69%|██████▉   | 34816/50176 [05:39<02:23, 107.18it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.13it/s]
 70%|███████   | 35328/50176 [05:44<02:16, 108.61it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 108.00it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.68it/s]
 72%|███████▏  | 36096/50176 [05:51<02:11, 107.34it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.51it/s]
 73%|███████▎  | 36608/50176 [05:56<02:09, 104.60it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.83it/s]
 74%|███████▍  | 37120/50176 [06:01<02:05, 103.96it/s]
 74%|███████▍  | 37376/50176 [06:03<01:56, 109.46it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.81it/s]
 76%|███████▌  | 37888/50176 [06:08<01:53, 107.85it/s]
 76%|███████▌  | 38144/50176 [06:10<01:51, 108.36it/s]
 77%|███████▋  | 38400/50176 [06:13<01:49, 107.24it/s]
 77%|███████▋  | 38656/50176 [06:15<01:48, 106.63it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.65it/s]
 78%|███████▊  | 39168/50176 [06:20<01:44, 105.55it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.21it/s]
 79%|███████▉  | 39680/50176 [06:25<01:38, 106.76it/s]
 80%|███████▉  | 39936/50176 [06:27<01:34, 108.03it/s]
 80%|████████  | 40192/50176 [06:29<01:34, 106.19it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.91it/s]
 81%|████████  | 40704/50176 [06:35<01:33, 101.16it/s]
 82%|████████▏ | 40960/50176 [06:37<01:27, 105.46it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.41it/s]
 83%|████████▎ | 41472/50176 [06:42<01:22, 105.51it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.42it/s]
 84%|████████▎ | 41984/50176 [06:47<01:18, 104.67it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.82it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.36it/s]
 85%|████████▌ | 42752/50176 [06:54<01:08, 108.37it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.84it/s]
 86%|████████▌ | 43264/50176 [06:59<01:05, 105.59it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.31it/s]
 87%|████████▋ | 43776/50176 [07:04<01:02, 102.92it/s]
 88%|████████▊ | 44032/50176 [07:06<00:58, 104.21it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.78it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 100.91it/s]
 89%|████████▉ | 44800/50176 [07:14<00:52, 102.36it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.15it/s]
 90%|█████████ | 45312/50176 [07:19<00:47, 102.54it/s]
 91%|█████████ | 45568/50176 [07:21<00:43, 106.08it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.86it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.19it/s]
 92%|█████████▏| 46336/50176 [07:29<00:37, 101.12it/s]
 93%|█████████▎| 46592/50176 [07:31<00:34, 105.08it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.18it/s]
 94%|█████████▍| 47104/50176 [07:36<00:30, 102.05it/s]
 94%|█████████▍| 47360/50176 [07:39<00:28, 100.27it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.36it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.79it/s]
 96%|█████████▌| 48128/50176 [07:46<00:19, 104.47it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.21it/s]
 97%|█████████▋| 48640/50176 [07:51<00:14, 104.58it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.36it/s]
 98%|█████████▊| 49152/50176 [07:56<00:09, 103.35it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.19it/s]
 99%|█████████▉| 49664/50176 [08:01<00:05, 102.35it/s]
 99%|█████████▉| 49920/50176 [08:04<00:02, 95.28it/s] 
-2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5514	imagenet-zeroshot-val-top5: 0.8246

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_16.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:45:55 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:45:55 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:45:56 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt).
-2022-12-21,09:46:05 | INFO | Model:
-2022-12-21,09:46:05 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:46:05 | INFO | Params:
-2022-12-21,09:46:05 | INFO |   batch_size: 256
-2022-12-21,09:46:05 | INFO |   beta1: 0.9
-2022-12-21,09:46:05 | INFO |   beta2: 0.98
-2022-12-21,09:46:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/checkpoints
-2022-12-21,09:46:05 | INFO |   copy_codebase: False
-2022-12-21,09:46:05 | INFO |   csv_caption_key: title
-2022-12-21,09:46:05 | INFO |   csv_img_key: filepath
-2022-12-21,09:46:05 | INFO |   csv_separator: 	
-2022-12-21,09:46:05 | INFO |   dataset_resampled: False
-2022-12-21,09:46:05 | INFO |   dataset_type: auto
-2022-12-21,09:46:05 | INFO |   ddp_static_graph: False
-2022-12-21,09:46:05 | INFO |   debug: False
-2022-12-21,09:46:05 | INFO |   device: cuda:0
-2022-12-21,09:46:05 | INFO |   dist_backend: nccl
-2022-12-21,09:46:05 | INFO |   dist_url: env://
-2022-12-21,09:46:05 | INFO |   distributed: False
-2022-12-21,09:46:05 | INFO |   epochs: 32
-2022-12-21,09:46:05 | INFO |   eps: 1e-06
-2022-12-21,09:46:05 | INFO |   force_quick_gelu: False
-2022-12-21,09:46:05 | INFO |   gather_with_grad: False
-2022-12-21,09:46:05 | INFO |   grad_checkpointing: False
-2022-12-21,09:46:05 | INFO |   horovod: False
-2022-12-21,09:46:05 | INFO |   image_mean: None
-2022-12-21,09:46:05 | INFO |   image_std: None
-2022-12-21,09:46:05 | INFO |   imagenet_v2: None
-2022-12-21,09:46:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:46:05 | INFO |   local_loss: False
-2022-12-21,09:46:05 | INFO |   local_rank: 0
-2022-12-21,09:46:05 | INFO |   lock_image: False
-2022-12-21,09:46:05 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:46:05 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:46:05 | INFO |   log_level: 20
-2022-12-21,09:46:05 | INFO |   log_local: False
-2022-12-21,09:46:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_16.pt/out.log
-2022-12-21,09:46:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:46:05 | INFO |   lr: 0.0005
-2022-12-21,09:46:05 | INFO |   model: ViT-B-16
-2022-12-21,09:46:05 | INFO |   name: epoch_16.pt
-2022-12-21,09:46:05 | INFO |   no_set_device_rank: False
-2022-12-21,09:46:05 | INFO |   norm_gradient_clip: None
-2022-12-21,09:46:05 | INFO |   precision: amp_bfloat16
-2022-12-21,09:46:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_16.pt
-2022-12-21,09:46:05 | INFO |   pretrained_image: False
-2022-12-21,09:46:05 | INFO |   rank: 0
-2022-12-21,09:46:05 | INFO |   report_to: 
-2022-12-21,09:46:05 | INFO |   resume: None
-2022-12-21,09:46:05 | INFO |   save_frequency: 1
-2022-12-21,09:46:05 | INFO |   save_most_recent: False
-2022-12-21,09:46:05 | INFO |   seed: 0
-2022-12-21,09:46:05 | INFO |   skip_scheduler: False
-2022-12-21,09:46:05 | INFO |   tensorboard: False
-2022-12-21,09:46:05 | INFO |   tensorboard_path: 
-2022-12-21,09:46:05 | INFO |   torchscript: False
-2022-12-21,09:46:05 | INFO |   trace: False
-2022-12-21,09:46:05 | INFO |   train_data: None
-2022-12-21,09:46:05 | INFO |   train_num_samples: None
-2022-12-21,09:46:05 | INFO |   use_bn_sync: False
-2022-12-21,09:46:05 | INFO |   val_data: None
-2022-12-21,09:46:05 | INFO |   val_frequency: 1
-2022-12-21,09:46:05 | INFO |   val_num_samples: None
-2022-12-21,09:46:05 | INFO |   wandb: False
-2022-12-21,09:46:05 | INFO |   wandb_notes: 
-2022-12-21,09:46:05 | INFO |   warmup: 10000
-2022-12-21,09:46:05 | INFO |   wd: 0.2
-2022-12-21,09:46:05 | INFO |   workers: 1
-2022-12-21,09:46:05 | INFO |   world_size: 1
-2022-12-21,09:46:05 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:46:05 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:46:05 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:00,  1.39it/s]
  1%|          | 6/1000 [00:00<01:45,  9.41it/s]
  1%|          | 12/1000 [00:00<00:53, 18.51it/s]
  2%|▏         | 18/1000 [00:01<00:36, 27.00it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.24it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.18it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.69it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.17it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.79it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.70it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.05it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.12it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.84it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.36it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.90it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.20it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.59it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.64it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.91it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.05it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.03it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.99it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.04it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.00it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.04it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.09it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.96it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.80it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.80it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.72it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.66it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.42it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.35it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.52it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.68it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.66it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.56it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.60it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.69it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.80it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.85it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.70it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.86it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.89it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.09it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.06it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.05it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.20it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.19it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.11it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.11it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.20it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.99it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.99it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.07it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.16it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.74it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.79it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.88it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.02it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.07it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.09it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.13it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.14it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.23it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.11it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.02it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.07it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.05it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.08it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.12it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.22it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.25it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.14it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.12it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.04it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.00it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.12it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.04it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.05it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.15it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.15it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.25it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.08it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.02it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.17it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.14it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.22it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.22it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.23it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.12it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.21it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.06it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.98it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.04it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.02it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.07it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.12it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.12it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.21it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.29it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.14it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.23it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.18it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.21it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.24it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.04it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.04it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.01it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.03it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.14it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.28it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.31it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.28it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.24it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.25it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.22it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.09it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.17it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.13it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.22it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.26it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.27it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.31it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.26it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.40it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.22it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.22it/s]
-2022-12-21,09:46:23 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:14, 133.40it/s]
  1%|          | 512/50176 [00:04<06:31, 126.77it/s]
  2%|▏         | 768/50176 [00:06<06:42, 122.82it/s]
  2%|▏         | 1024/50176 [00:08<06:46, 121.01it/s]
  3%|▎         | 1280/50176 [00:10<07:05, 114.85it/s]
  3%|▎         | 1536/50176 [00:13<07:37, 106.42it/s]
  4%|▎         | 1792/50176 [00:15<07:24, 108.95it/s]
  4%|▍         | 2048/50176 [00:17<07:15, 110.41it/s]
  5%|▍         | 2304/50176 [00:20<07:04, 112.80it/s]
  5%|▌         | 2560/50176 [00:22<07:01, 112.99it/s]
  6%|▌         | 2816/50176 [00:24<07:08, 110.47it/s]
  6%|▌         | 3072/50176 [00:27<07:13, 108.77it/s]
  7%|▋         | 3328/50176 [00:29<07:07, 109.48it/s]
  7%|▋         | 3584/50176 [00:31<06:55, 112.19it/s]
  8%|▊         | 3840/50176 [00:34<07:19, 105.46it/s]
  8%|▊         | 4096/50176 [00:37<07:36, 100.91it/s]
  9%|▊         | 4352/50176 [00:39<07:29, 101.89it/s]
  9%|▉         | 4608/50176 [00:42<07:13, 105.05it/s]
 10%|▉         | 4864/50176 [00:44<07:06, 106.23it/s]
 10%|█         | 5120/50176 [00:46<07:03, 106.27it/s]
 11%|█         | 5376/50176 [00:49<07:08, 104.59it/s]
 11%|█         | 5632/50176 [00:51<07:11, 103.18it/s]
 12%|█▏        | 5888/50176 [00:54<07:07, 103.56it/s]
 12%|█▏        | 6144/50176 [00:57<07:34, 96.77it/s] 
 13%|█▎        | 6400/50176 [00:59<07:31, 97.06it/s]
 13%|█▎        | 6656/50176 [01:02<07:21, 98.60it/s]
 14%|█▍        | 6912/50176 [01:05<07:18, 98.76it/s]
 14%|█▍        | 7168/50176 [01:07<07:14, 98.91it/s]
 15%|█▍        | 7424/50176 [01:10<07:02, 101.31it/s]
 15%|█▌        | 7680/50176 [01:12<06:56, 102.02it/s]
 16%|█▌        | 7936/50176 [01:14<06:47, 103.56it/s]
 16%|█▋        | 8192/50176 [01:17<06:43, 104.02it/s]
 17%|█▋        | 8448/50176 [01:19<06:50, 101.54it/s]
 17%|█▋        | 8704/50176 [01:22<06:47, 101.66it/s]
 18%|█▊        | 8960/50176 [01:25<06:48, 100.87it/s]
 18%|█▊        | 9216/50176 [01:27<06:44, 101.37it/s]
 19%|█▉        | 9472/50176 [01:30<06:40, 101.52it/s]
 19%|█▉        | 9728/50176 [01:32<06:44, 100.08it/s]
 20%|█▉        | 9984/50176 [01:35<06:42, 99.86it/s] 
 20%|██        | 10240/50176 [01:37<06:29, 102.65it/s]
 21%|██        | 10496/50176 [01:40<06:20, 104.16it/s]
 21%|██▏       | 10752/50176 [01:42<06:28, 101.35it/s]
 22%|██▏       | 11008/50176 [01:45<06:24, 101.76it/s]
 22%|██▏       | 11264/50176 [01:47<06:17, 103.18it/s]
 23%|██▎       | 11520/50176 [01:49<06:09, 104.59it/s]
 23%|██▎       | 11776/50176 [01:52<06:05, 104.97it/s]
 24%|██▍       | 12032/50176 [01:54<06:05, 104.40it/s]
 24%|██▍       | 12288/50176 [01:57<05:54, 106.82it/s]
 25%|██▌       | 12544/50176 [01:59<05:50, 107.48it/s]
 26%|██▌       | 12800/50176 [02:01<05:51, 106.20it/s]
 26%|██▌       | 13056/50176 [02:04<05:48, 106.65it/s]
 27%|██▋       | 13312/50176 [02:06<05:48, 105.84it/s]
 27%|██▋       | 13568/50176 [02:09<06:05, 100.19it/s]
 28%|██▊       | 13824/50176 [02:12<06:08, 98.72it/s] 
 28%|██▊       | 14080/50176 [02:14<06:00, 100.10it/s]
 29%|██▊       | 14336/50176 [02:17<05:57, 100.21it/s]
 29%|██▉       | 14592/50176 [02:19<05:50, 101.57it/s]
 30%|██▉       | 14848/50176 [02:22<05:54, 99.68it/s] 
 30%|███       | 15104/50176 [02:25<05:55, 98.69it/s]
 31%|███       | 15360/50176 [02:27<05:47, 100.14it/s]
 31%|███       | 15616/50176 [02:29<05:37, 102.28it/s]
 32%|███▏      | 15872/50176 [02:32<05:38, 101.21it/s]
 32%|███▏      | 16128/50176 [02:35<05:34, 101.86it/s]
 33%|███▎      | 16384/50176 [02:37<05:32, 101.67it/s]
 33%|███▎      | 16640/50176 [02:40<05:38, 99.06it/s] 
 34%|███▎      | 16896/50176 [02:42<05:30, 100.68it/s]
 34%|███▍      | 17152/50176 [02:45<05:22, 102.29it/s]
 35%|███▍      | 17408/50176 [02:47<05:17, 103.11it/s]
 35%|███▌      | 17664/50176 [02:49<05:03, 107.00it/s]
 36%|███▌      | 17920/50176 [02:52<04:55, 109.00it/s]
 36%|███▌      | 18176/50176 [02:54<04:59, 106.73it/s]
 37%|███▋      | 18432/50176 [02:56<04:54, 107.71it/s]
 37%|███▋      | 18688/50176 [02:59<05:01, 104.57it/s]
 38%|███▊      | 18944/50176 [03:02<05:04, 102.62it/s]
 38%|███▊      | 19200/50176 [03:04<05:09, 99.95it/s] 
 39%|███▉      | 19456/50176 [03:07<05:12, 98.19it/s]
 39%|███▉      | 19712/50176 [03:10<05:12, 97.36it/s]
 40%|███▉      | 19968/50176 [03:12<05:12, 96.78it/s]
 40%|████      | 20224/50176 [03:15<04:55, 101.35it/s]
 41%|████      | 20480/50176 [03:17<04:47, 103.12it/s]
 41%|████▏     | 20736/50176 [03:19<04:41, 104.40it/s]
 42%|████▏     | 20992/50176 [03:22<04:38, 104.90it/s]
 42%|████▏     | 21248/50176 [03:24<04:41, 102.59it/s]
 43%|████▎     | 21504/50176 [03:27<04:38, 102.97it/s]
 43%|████▎     | 21760/50176 [03:29<04:30, 105.19it/s]
 44%|████▍     | 22016/50176 [03:31<04:20, 108.30it/s]
 44%|████▍     | 22272/50176 [03:34<04:17, 108.27it/s]
 45%|████▍     | 22528/50176 [03:36<04:19, 106.36it/s]
 45%|████▌     | 22784/50176 [03:39<04:30, 101.43it/s]
 46%|████▌     | 23040/50176 [03:42<04:33, 99.28it/s] 
 46%|████▋     | 23296/50176 [03:44<04:33, 98.30it/s]
 47%|████▋     | 23552/50176 [03:49<05:44, 77.21it/s]
 47%|████▋     | 23808/50176 [03:52<05:13, 84.04it/s]
 48%|████▊     | 24064/50176 [03:54<04:56, 88.14it/s]
 48%|████▊     | 24320/50176 [03:57<04:41, 91.89it/s]
 49%|████▉     | 24576/50176 [04:00<04:32, 93.92it/s]
 49%|████▉     | 24832/50176 [04:02<04:18, 97.96it/s]
 50%|█████     | 25088/50176 [04:04<04:06, 101.89it/s]
 51%|█████     | 25344/50176 [04:06<03:56, 104.97it/s]
 51%|█████     | 25600/50176 [04:09<03:55, 104.23it/s]
 52%|█████▏    | 25856/50176 [04:11<03:47, 107.10it/s]
 52%|█████▏    | 26112/50176 [04:14<03:44, 107.40it/s]
 53%|█████▎    | 26368/50176 [04:16<03:44, 106.16it/s]
 53%|█████▎    | 26624/50176 [04:18<03:38, 107.90it/s]
 54%|█████▎    | 26880/50176 [04:21<03:39, 106.14it/s]
 54%|█████▍    | 27136/50176 [04:23<03:41, 104.15it/s]
 55%|█████▍    | 27392/50176 [04:26<03:39, 103.66it/s]
 55%|█████▌    | 27648/50176 [04:30<04:20, 86.59it/s] 
 56%|█████▌    | 27904/50176 [04:33<04:17, 86.53it/s]
 56%|█████▌    | 28160/50176 [04:35<04:00, 91.51it/s]
 57%|█████▋    | 28416/50176 [04:38<03:51, 93.86it/s]
 57%|█████▋    | 28672/50176 [04:41<03:48, 94.01it/s]
 58%|█████▊    | 28928/50176 [04:43<03:33, 99.41it/s]
 58%|█████▊    | 29184/50176 [04:45<03:29, 100.00it/s]
 59%|█████▊    | 29440/50176 [04:48<03:37, 95.56it/s] 
 59%|█████▉    | 29696/50176 [04:51<03:33, 95.78it/s]
 60%|█████▉    | 29952/50176 [04:54<03:29, 96.60it/s]
 60%|██████    | 30208/50176 [04:56<03:22, 98.78it/s]
 61%|██████    | 30464/50176 [04:58<03:10, 103.34it/s]
 61%|██████    | 30720/50176 [05:01<03:11, 101.55it/s]
 62%|██████▏   | 30976/50176 [05:03<03:02, 105.01it/s]
 62%|██████▏   | 31232/50176 [05:05<02:59, 105.51it/s]
 63%|██████▎   | 31488/50176 [05:08<02:52, 108.54it/s]
 63%|██████▎   | 31744/50176 [05:10<02:50, 107.85it/s]
 64%|██████▍   | 32000/50176 [05:12<02:48, 107.86it/s]
 64%|██████▍   | 32256/50176 [05:15<02:42, 109.99it/s]
 65%|██████▍   | 32512/50176 [05:17<02:47, 105.30it/s]
 65%|██████▌   | 32768/50176 [05:20<02:40, 108.14it/s]
 66%|██████▌   | 33024/50176 [05:22<02:39, 107.65it/s]
 66%|██████▋   | 33280/50176 [05:24<02:36, 107.81it/s]
 67%|██████▋   | 33536/50176 [05:27<02:37, 105.63it/s]
 67%|██████▋   | 33792/50176 [05:29<02:32, 107.59it/s]
 68%|██████▊   | 34048/50176 [05:31<02:24, 111.43it/s]
 68%|██████▊   | 34304/50176 [05:33<02:19, 114.06it/s]
 69%|██████▉   | 34560/50176 [05:36<02:24, 108.16it/s]
 69%|██████▉   | 34816/50176 [05:38<02:23, 107.11it/s]
 70%|██████▉   | 35072/50176 [05:41<02:23, 105.20it/s]
 70%|███████   | 35328/50176 [05:43<02:16, 108.46it/s]
 71%|███████   | 35584/50176 [05:46<02:15, 107.88it/s]
 71%|███████▏  | 35840/50176 [05:48<02:14, 106.77it/s]
 72%|███████▏  | 36096/50176 [05:50<02:11, 107.35it/s]
 72%|███████▏  | 36352/50176 [05:53<02:12, 104.50it/s]
 73%|███████▎  | 36608/50176 [05:55<02:09, 104.60it/s]
 73%|███████▎  | 36864/50176 [05:58<02:09, 102.82it/s]
 74%|███████▍  | 37120/50176 [06:00<02:05, 103.92it/s]
 74%|███████▍  | 37376/50176 [06:03<01:56, 109.60it/s]
 75%|███████▌  | 37632/50176 [06:05<01:56, 107.75it/s]
 76%|███████▌  | 37888/50176 [06:07<01:53, 107.83it/s]
 76%|███████▌  | 38144/50176 [06:10<01:50, 108.46it/s]
 77%|███████▋  | 38400/50176 [06:12<01:49, 107.37it/s]
 77%|███████▋  | 38656/50176 [06:15<01:48, 106.56it/s]
 78%|███████▊  | 38912/50176 [06:17<01:44, 107.47it/s]
 78%|███████▊  | 39168/50176 [06:19<01:44, 105.49it/s]
 79%|███████▊  | 39424/50176 [06:22<01:40, 107.20it/s]
 79%|███████▉  | 39680/50176 [06:24<01:38, 106.74it/s]
 80%|███████▉  | 39936/50176 [06:26<01:34, 108.17it/s]
 80%|████████  | 40192/50176 [06:29<01:34, 106.13it/s]
 81%|████████  | 40448/50176 [06:32<01:34, 102.81it/s]
 81%|████████  | 40704/50176 [06:34<01:33, 101.17it/s]
 82%|████████▏ | 40960/50176 [06:36<01:27, 105.48it/s]
 82%|████████▏ | 41216/50176 [06:39<01:24, 106.51it/s]
 83%|████████▎ | 41472/50176 [06:41<01:22, 105.53it/s]
 83%|████████▎ | 41728/50176 [06:44<01:22, 102.33it/s]
 84%|████████▎ | 41984/50176 [06:46<01:18, 104.65it/s]
 84%|████████▍ | 42240/50176 [06:49<01:15, 104.90it/s]
 85%|████████▍ | 42496/50176 [06:51<01:10, 108.41it/s]
 85%|████████▌ | 42752/50176 [06:53<01:08, 108.34it/s]
 86%|████████▌ | 43008/50176 [06:56<01:07, 105.82it/s]
 86%|████████▌ | 43264/50176 [06:58<01:05, 105.60it/s]
 87%|████████▋ | 43520/50176 [07:01<01:05, 102.31it/s]
 87%|████████▋ | 43776/50176 [07:03<01:02, 103.01it/s]
 88%|████████▊ | 44032/50176 [07:06<00:59, 104.13it/s]
 88%|████████▊ | 44288/50176 [07:08<00:55, 106.88it/s]
 89%|████████▉ | 44544/50176 [07:11<00:55, 100.87it/s]
 89%|████████▉ | 44800/50176 [07:13<00:52, 102.34it/s]
 90%|████████▉ | 45056/50176 [07:16<00:48, 105.10it/s]
 90%|█████████ | 45312/50176 [07:18<00:47, 102.52it/s]
 91%|█████████ | 45568/50176 [07:20<00:43, 106.01it/s]
 91%|█████████▏| 45824/50176 [07:23<00:41, 103.86it/s]
 92%|█████████▏| 46080/50176 [07:26<00:40, 101.16it/s]
 92%|█████████▏| 46336/50176 [07:28<00:37, 101.09it/s]
 93%|█████████▎| 46592/50176 [07:30<00:34, 105.18it/s]
 93%|█████████▎| 46848/50176 [07:33<00:31, 105.24it/s]
 94%|█████████▍| 47104/50176 [07:36<00:30, 101.97it/s]
 94%|█████████▍| 47360/50176 [07:38<00:28, 100.20it/s]
 95%|█████████▍| 47616/50176 [07:41<00:25, 101.48it/s]
 95%|█████████▌| 47872/50176 [07:43<00:22, 103.77it/s]
 96%|█████████▌| 48128/50176 [07:45<00:19, 104.47it/s]
 96%|█████████▋| 48384/50176 [07:48<00:17, 105.08it/s]
 97%|█████████▋| 48640/50176 [07:50<00:14, 104.60it/s]
 97%|█████████▋| 48896/50176 [07:53<00:12, 103.40it/s]
 98%|█████████▊| 49152/50176 [07:55<00:09, 103.34it/s]
 98%|█████████▊| 49408/50176 [07:58<00:07, 102.21it/s]
 99%|█████████▉| 49664/50176 [08:00<00:05, 102.34it/s]
 99%|█████████▉| 49920/50176 [08:03<00:02, 95.40it/s] 
-2022-12-21,09:54:28 | INFO | Finished zero-shot imagenet.
-2022-12-21,09:54:28 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5533	imagenet-zeroshot-val-top5: 0.8258

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_17.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt).
-2022-12-21,09:54:46 | INFO | Model:
-2022-12-21,09:54:46 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:54:46 | INFO | Params:
-2022-12-21,09:54:46 | INFO |   batch_size: 256
-2022-12-21,09:54:46 | INFO |   beta1: 0.9
-2022-12-21,09:54:46 | INFO |   beta2: 0.98
-2022-12-21,09:54:46 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/checkpoints
-2022-12-21,09:54:46 | INFO |   copy_codebase: False
-2022-12-21,09:54:46 | INFO |   csv_caption_key: title
-2022-12-21,09:54:46 | INFO |   csv_img_key: filepath
-2022-12-21,09:54:46 | INFO |   csv_separator: 	
-2022-12-21,09:54:46 | INFO |   dataset_resampled: False
-2022-12-21,09:54:46 | INFO |   dataset_type: auto
-2022-12-21,09:54:46 | INFO |   ddp_static_graph: False
-2022-12-21,09:54:46 | INFO |   debug: False
-2022-12-21,09:54:46 | INFO |   device: cuda:0
-2022-12-21,09:54:46 | INFO |   dist_backend: nccl
-2022-12-21,09:54:46 | INFO |   dist_url: env://
-2022-12-21,09:54:46 | INFO |   distributed: False
-2022-12-21,09:54:46 | INFO |   epochs: 32
-2022-12-21,09:54:46 | INFO |   eps: 1e-06
-2022-12-21,09:54:46 | INFO |   force_quick_gelu: False
-2022-12-21,09:54:46 | INFO |   gather_with_grad: False
-2022-12-21,09:54:46 | INFO |   grad_checkpointing: False
-2022-12-21,09:54:46 | INFO |   horovod: False
-2022-12-21,09:54:46 | INFO |   image_mean: None
-2022-12-21,09:54:46 | INFO |   image_std: None
-2022-12-21,09:54:46 | INFO |   imagenet_v2: None
-2022-12-21,09:54:46 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:54:46 | INFO |   local_loss: False
-2022-12-21,09:54:46 | INFO |   local_rank: 0
-2022-12-21,09:54:46 | INFO |   lock_image: False
-2022-12-21,09:54:46 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:54:46 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:54:46 | INFO |   log_level: 20
-2022-12-21,09:54:46 | INFO |   log_local: False
-2022-12-21,09:54:46 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_17.pt/out.log
-2022-12-21,09:54:46 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:54:46 | INFO |   lr: 0.0005
-2022-12-21,09:54:46 | INFO |   model: ViT-B-16
-2022-12-21,09:54:46 | INFO |   name: epoch_17.pt
-2022-12-21,09:54:46 | INFO |   no_set_device_rank: False
-2022-12-21,09:54:46 | INFO |   norm_gradient_clip: None
-2022-12-21,09:54:46 | INFO |   precision: amp_bfloat16
-2022-12-21,09:54:46 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_17.pt
-2022-12-21,09:54:46 | INFO |   pretrained_image: False
-2022-12-21,09:54:46 | INFO |   rank: 0
-2022-12-21,09:54:46 | INFO |   report_to: 
-2022-12-21,09:54:46 | INFO |   resume: None
-2022-12-21,09:54:46 | INFO |   save_frequency: 1
-2022-12-21,09:54:46 | INFO |   save_most_recent: False
-2022-12-21,09:54:46 | INFO |   seed: 0
-2022-12-21,09:54:46 | INFO |   skip_scheduler: False
-2022-12-21,09:54:46 | INFO |   tensorboard: False
-2022-12-21,09:54:46 | INFO |   tensorboard_path: 
-2022-12-21,09:54:46 | INFO |   torchscript: False
-2022-12-21,09:54:46 | INFO |   trace: False
-2022-12-21,09:54:46 | INFO |   train_data: None
-2022-12-21,09:54:46 | INFO |   train_num_samples: None
-2022-12-21,09:54:46 | INFO |   use_bn_sync: False
-2022-12-21,09:54:46 | INFO |   val_data: None
-2022-12-21,09:54:46 | INFO |   val_frequency: 1
-2022-12-21,09:54:46 | INFO |   val_num_samples: None
-2022-12-21,09:54:46 | INFO |   wandb: False
-2022-12-21,09:54:46 | INFO |   wandb_notes: 
-2022-12-21,09:54:46 | INFO |   warmup: 10000
-2022-12-21,09:54:46 | INFO |   wd: 0.2
-2022-12-21,09:54:46 | INFO |   workers: 1
-2022-12-21,09:54:46 | INFO |   world_size: 1
-2022-12-21,09:54:46 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:54:46 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:54:46 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:17,  1.48it/s]
  1%|          | 6/1000 [00:00<01:40,  9.91it/s]
  1%|          | 11/1000 [00:00<00:55, 17.83it/s]
  2%|▏         | 17/1000 [00:00<00:36, 26.77it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.34it/s]
  3%|▎         | 29/1000 [00:01<00:24, 40.44it/s]
  4%|▎         | 35/1000 [00:01<00:21, 45.07it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.64it/s]
  5%|▍         | 47/1000 [00:01<00:18, 51.05it/s]
  5%|▌         | 53/1000 [00:01<00:17, 52.98it/s]
  6%|▌         | 59/1000 [00:01<00:17, 54.22it/s]
  6%|▋         | 65/1000 [00:01<00:16, 55.23it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.91it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.42it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.93it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.32it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.52it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.69it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.97it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.06it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.01it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.96it/s]
 13%|█▎        | 131/1000 [00:02<00:14, 58.06it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.88it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.92it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.97it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.06it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.95it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.80it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.83it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.64it/s]
 19%|█▉        | 191/1000 [00:03<00:14, 57.62it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.47it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.36it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.34it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.37it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.57it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.62it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.48it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 57.47it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.53it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.73it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.78it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.55it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.79it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.67it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.94it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.96it/s]
 30%|███       | 305/1000 [00:05<00:11, 57.99it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.08it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.90it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.98it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.15it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.94it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.08it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.02it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.16it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.80it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.77it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.85it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.03it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.06it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 419/1000 [00:07<00:09, 58.13it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.04it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.21it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.23it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.04it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.10it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.02it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.15it/s]
 48%|████▊     | 479/1000 [00:08<00:08, 58.06it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.05it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.04it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.96it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.09it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.05it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.14it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.04it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.15it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.98it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.02it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.92it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.87it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.04it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 593/1000 [00:10<00:07, 57.92it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.05it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.95it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.12it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.16it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.88it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.09it/s]
 65%|██████▌   | 653/1000 [00:11<00:05, 58.02it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.11it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.08it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.07it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.11it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.06it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.81it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.80it/s]
 70%|███████   | 701/1000 [00:12<00:05, 57.96it/s]
 71%|███████   | 707/1000 [00:12<00:05, 57.99it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.03it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.04it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.03it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.03it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.06it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.31it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.03it/s]
 77%|███████▋  | 767/1000 [00:13<00:04, 58.06it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.04it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.10it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.14it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.95it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.94it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.97it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.98it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 57.98it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.03it/s]
 83%|████████▎ | 827/1000 [00:14<00:02, 58.05it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.25it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.34it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.25it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.21it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.25it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.21it/s]
 89%|████████▊ | 887/1000 [00:15<00:01, 58.02it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.12it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.07it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.20it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.28it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.27it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.26it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.27it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.20it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.32it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.06it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.02it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 57.77it/s]
-2022-12-21,09:55:04 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:54, 140.83it/s]
  1%|          | 512/50176 [00:03<05:38, 146.89it/s]
  2%|▏         | 768/50176 [00:05<05:35, 147.25it/s]
  2%|▏         | 1024/50176 [00:06<05:35, 146.69it/s]
  3%|▎         | 1280/50176 [00:08<05:38, 144.60it/s]
  3%|▎         | 1536/50176 [00:10<05:54, 137.35it/s]
  4%|▎         | 1792/50176 [00:12<05:53, 136.69it/s]
  4%|▍         | 2048/50176 [00:14<05:44, 139.63it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 138.94it/s]
  5%|▌         | 2560/50176 [00:18<05:35, 141.81it/s]
  6%|▌         | 2816/50176 [00:19<05:37, 140.14it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 142.75it/s]
  7%|▋         | 3328/50176 [00:23<05:26, 143.44it/s]
  7%|▋         | 3584/50176 [00:25<05:19, 145.73it/s]
  8%|▊         | 3840/50176 [00:27<05:31, 139.89it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.33it/s]
  9%|▊         | 4352/50176 [00:30<05:22, 142.16it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.32it/s]
 10%|▉         | 4864/50176 [00:34<05:11, 145.25it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.29it/s]
 11%|█         | 5376/50176 [00:37<05:04, 147.19it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.08it/s]
 12%|█▏        | 5888/50176 [00:41<04:58, 148.20it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 135.96it/s]
 13%|█▎        | 6400/50176 [00:44<05:11, 140.38it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.45it/s]
 14%|█▍        | 6912/50176 [00:48<05:03, 142.57it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 144.09it/s]
 15%|█▍        | 7424/50176 [00:51<04:51, 146.62it/s]
 15%|█▌        | 7680/50176 [00:53<04:51, 145.80it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 148.94it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.55it/s]
 17%|█▋        | 8448/50176 [00:58<04:38, 149.83it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.39it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.57it/s]
 18%|█▊        | 9216/50176 [01:03<04:35, 148.60it/s]
 19%|█▉        | 9472/50176 [01:05<04:32, 149.36it/s]
 19%|█▉        | 9728/50176 [01:07<04:36, 146.29it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.14it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.99it/s]
 21%|██        | 10496/50176 [01:12<04:21, 151.81it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.24it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.50it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.22it/s]
 23%|██▎       | 11520/50176 [01:19<04:18, 149.78it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 148.90it/s]
 24%|██▍       | 12032/50176 [01:22<04:14, 149.66it/s]
 24%|██▍       | 12288/50176 [01:24<04:13, 149.66it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.17it/s]
 26%|██▌       | 12800/50176 [01:27<04:07, 150.78it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.37it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.69it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 145.22it/s]
 28%|██▊       | 13824/50176 [01:35<04:16, 141.81it/s]
 28%|██▊       | 14080/50176 [01:36<04:12, 143.01it/s]
 29%|██▊       | 14336/50176 [01:38<04:15, 140.40it/s]
 29%|██▉       | 14592/50176 [01:40<04:05, 145.14it/s]
 30%|██▉       | 14848/50176 [01:42<04:17, 137.00it/s]
 30%|███       | 15104/50176 [01:44<04:18, 135.79it/s]
 31%|███       | 15360/50176 [01:46<04:10, 139.10it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.56it/s]
 32%|███▏      | 15872/50176 [01:49<04:09, 137.62it/s]
 32%|███▏      | 16128/50176 [01:51<04:02, 140.29it/s]
 33%|███▎      | 16384/50176 [01:53<04:00, 140.54it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 135.22it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.71it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.16it/s]
 35%|███▍      | 17408/50176 [02:00<03:46, 144.77it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.51it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.35it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.54it/s]
 37%|███▋      | 18432/50176 [02:07<03:33, 148.88it/s]
 37%|███▋      | 18688/50176 [02:09<03:33, 147.49it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.20it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.73it/s]
 39%|███▉      | 19456/50176 [02:15<03:54, 131.21it/s]
 39%|███▉      | 19712/50176 [02:17<03:52, 131.30it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.53it/s]
 40%|████      | 20224/50176 [02:20<03:32, 140.88it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.96it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 144.10it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.63it/s]
 42%|████▏     | 21248/50176 [02:27<03:24, 141.49it/s]
 43%|████▎     | 21504/50176 [02:29<03:16, 145.96it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.07it/s]
 44%|████▍     | 22016/50176 [02:32<03:07, 150.37it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.76it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.34it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 144.98it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.40it/s]
 46%|████▋     | 23296/50176 [02:41<03:11, 140.05it/s]
 47%|████▋     | 23552/50176 [02:43<03:15, 136.33it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.94it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.33it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 142.98it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.50it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.74it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 149.03it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.48it/s]
 51%|█████     | 25600/50176 [02:57<02:47, 147.06it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.12it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.27it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 143.56it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 146.10it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 146.05it/s]
 54%|█████▍    | 27136/50176 [03:07<02:36, 147.48it/s]
 55%|█████▍    | 27392/50176 [03:09<02:34, 147.82it/s]
 55%|█████▌    | 27648/50176 [03:11<02:27, 153.08it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.63it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.77it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.44it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.06it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.10it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.38it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.10it/s]
 59%|█████▉    | 29696/50176 [03:26<02:31, 134.87it/s]
 60%|█████▉    | 29952/50176 [03:28<02:31, 133.79it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 132.80it/s]
 61%|██████    | 30464/50176 [03:31<02:20, 140.35it/s]
 61%|██████    | 30720/50176 [03:33<02:18, 140.50it/s]
 62%|██████▏   | 30976/50176 [03:35<02:13, 143.40it/s]
 62%|██████▏   | 31232/50176 [03:36<02:09, 146.61it/s]
 63%|██████▎   | 31488/50176 [03:38<02:06, 148.27it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 148.44it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 146.86it/s]
 64%|██████▍   | 32256/50176 [03:43<02:01, 147.32it/s]
 65%|██████▍   | 32512/50176 [03:45<02:03, 142.46it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 145.75it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.31it/s]
 66%|██████▋   | 33280/50176 [03:50<01:55, 145.70it/s]
 67%|██████▋   | 33536/50176 [03:52<01:56, 142.64it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.39it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.55it/s]
 68%|██████▊   | 34304/50176 [03:57<01:44, 151.60it/s]
 69%|██████▉   | 34560/50176 [03:59<01:47, 145.26it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.35it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 143.96it/s]
 70%|███████   | 35328/50176 [04:04<01:40, 148.04it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 146.22it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 141.53it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 143.71it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.67it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 144.37it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 142.46it/s]
 74%|███████▍  | 37120/50176 [04:17<01:31, 143.37it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.56it/s]
 75%|███████▌  | 37632/50176 [04:20<01:27, 144.11it/s]
 76%|███████▌  | 37888/50176 [04:22<01:25, 143.66it/s]
 76%|███████▌  | 38144/50176 [04:24<01:22, 145.85it/s]
 77%|███████▋  | 38400/50176 [04:26<01:21, 145.37it/s]
 77%|███████▋  | 38656/50176 [04:28<01:19, 144.06it/s]
 78%|███████▊  | 38912/50176 [04:29<01:18, 142.60it/s]
 78%|███████▊  | 39168/50176 [04:31<01:19, 138.50it/s]
 79%|███████▊  | 39424/50176 [04:33<01:16, 141.21it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.33it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.25it/s]
 80%|████████  | 40192/50176 [04:38<01:07, 147.20it/s]
 81%|████████  | 40448/50176 [04:40<01:09, 140.74it/s]
 81%|████████  | 40704/50176 [04:42<01:09, 135.34it/s]
 82%|████████▏ | 40960/50176 [04:44<01:05, 140.40it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.24it/s]
 83%|████████▎ | 41472/50176 [04:47<00:59, 146.68it/s]
 83%|████████▎ | 41728/50176 [04:49<00:56, 149.11it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.17it/s]
 84%|██████���█▍ | 42240/50176 [04:52<00:54, 146.01it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 148.13it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.84it/s]
 86%|████████▌ | 43008/50176 [04:58<00:47, 149.62it/s]
 86%|████████▌ | 43264/50176 [04:59<00:46, 149.12it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.04it/s]
 87%|████████▋ | 43776/50176 [05:05<01:03, 101.41it/s]
 88%|████████▊ | 44032/50176 [05:07<00:53, 113.79it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.46it/s]
 89%|████████▉ | 44544/50176 [05:10<00:43, 129.85it/s]
 89%|████████▉ | 44800/50176 [05:12<00:40, 134.23it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.14it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.27it/s]
 91%|█████████ | 45568/50176 [05:18<00:35, 128.04it/s]
 91%|█████████▏| 45824/50176 [05:20<00:32, 135.52it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.19it/s]
 92%|█████████▏| 46336/50176 [05:23<00:26, 145.11it/s]
 93%|█████████▎| 46592/50176 [05:25<00:24, 148.03it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.66it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 144.70it/s]
 94%|█████████▍| 47360/50176 [05:30<00:19, 146.61it/s]
 95%|█████████▍| 47616/50176 [05:32<00:17, 145.44it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.42it/s]
 96%|█████████▌| 48128/50176 [05:35<00:13, 150.32it/s]
 96%|█████████▋| 48384/50176 [05:37<00:11, 152.05it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.19it/s]
 97%|█████████▋| 48896/50176 [05:40<00:08, 154.93it/s]
 98%|█████████▊| 49152/50176 [05:42<00:06, 150.32it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.14it/s]
 99%|█████████▉| 49664/50176 [05:45<00:03, 150.15it/s]
 99%|█████████▉| 49920/50176 [05:47<00:01, 142.49it/s]
-2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5545	imagenet-zeroshot-val-top5: 0.8302

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_18.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt).
-2022-12-21,09:54:45 | INFO | Model:
-2022-12-21,09:54:45 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:54:45 | INFO | Params:
-2022-12-21,09:54:45 | INFO |   batch_size: 256
-2022-12-21,09:54:45 | INFO |   beta1: 0.9
-2022-12-21,09:54:45 | INFO |   beta2: 0.98
-2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/checkpoints
-2022-12-21,09:54:45 | INFO |   copy_codebase: False
-2022-12-21,09:54:45 | INFO |   csv_caption_key: title
-2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
-2022-12-21,09:54:45 | INFO |   csv_separator: 	
-2022-12-21,09:54:45 | INFO |   dataset_resampled: False
-2022-12-21,09:54:45 | INFO |   dataset_type: auto
-2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
-2022-12-21,09:54:45 | INFO |   debug: False
-2022-12-21,09:54:45 | INFO |   device: cuda:0
-2022-12-21,09:54:45 | INFO |   dist_backend: nccl
-2022-12-21,09:54:45 | INFO |   dist_url: env://
-2022-12-21,09:54:45 | INFO |   distributed: False
-2022-12-21,09:54:45 | INFO |   epochs: 32
-2022-12-21,09:54:45 | INFO |   eps: 1e-06
-2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
-2022-12-21,09:54:45 | INFO |   gather_with_grad: False
-2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
-2022-12-21,09:54:45 | INFO |   horovod: False
-2022-12-21,09:54:45 | INFO |   image_mean: None
-2022-12-21,09:54:45 | INFO |   image_std: None
-2022-12-21,09:54:45 | INFO |   imagenet_v2: None
-2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:54:45 | INFO |   local_loss: False
-2022-12-21,09:54:45 | INFO |   local_rank: 0
-2022-12-21,09:54:45 | INFO |   lock_image: False
-2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:54:45 | INFO |   log_level: 20
-2022-12-21,09:54:45 | INFO |   log_local: False
-2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_18.pt/out.log
-2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:54:45 | INFO |   lr: 0.0005
-2022-12-21,09:54:45 | INFO |   model: ViT-B-16
-2022-12-21,09:54:45 | INFO |   name: epoch_18.pt
-2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
-2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
-2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
-2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_18.pt
-2022-12-21,09:54:45 | INFO |   pretrained_image: False
-2022-12-21,09:54:45 | INFO |   rank: 0
-2022-12-21,09:54:45 | INFO |   report_to: 
-2022-12-21,09:54:45 | INFO |   resume: None
-2022-12-21,09:54:45 | INFO |   save_frequency: 1
-2022-12-21,09:54:45 | INFO |   save_most_recent: False
-2022-12-21,09:54:45 | INFO |   seed: 0
-2022-12-21,09:54:45 | INFO |   skip_scheduler: False
-2022-12-21,09:54:45 | INFO |   tensorboard: False
-2022-12-21,09:54:45 | INFO |   tensorboard_path: 
-2022-12-21,09:54:45 | INFO |   torchscript: False
-2022-12-21,09:54:45 | INFO |   trace: False
-2022-12-21,09:54:45 | INFO |   train_data: None
-2022-12-21,09:54:45 | INFO |   train_num_samples: None
-2022-12-21,09:54:45 | INFO |   use_bn_sync: False
-2022-12-21,09:54:45 | INFO |   val_data: None
-2022-12-21,09:54:45 | INFO |   val_frequency: 1
-2022-12-21,09:54:45 | INFO |   val_num_samples: None
-2022-12-21,09:54:45 | INFO |   wandb: False
-2022-12-21,09:54:45 | INFO |   wandb_notes: 
-2022-12-21,09:54:45 | INFO |   warmup: 10000
-2022-12-21,09:54:45 | INFO |   wd: 0.2
-2022-12-21,09:54:45 | INFO |   workers: 1
-2022-12-21,09:54:45 | INFO |   world_size: 1
-2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:54:45 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:54:45 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:34,  1.32it/s]
  1%|          | 6/1000 [00:00<01:50,  9.00it/s]
  1%|          | 11/1000 [00:00<01:00, 16.46it/s]
  2%|▏         | 16/1000 [00:01<00:42, 23.37it/s]
  2%|▏         | 22/1000 [00:01<00:31, 31.08it/s]
  3%|▎         | 28/1000 [00:01<00:25, 37.45it/s]
  3%|▎         | 34/1000 [00:01<00:22, 42.70it/s]
  4%|▍         | 40/1000 [00:01<00:20, 46.76it/s]
  5%|▍         | 46/1000 [00:01<00:19, 49.66it/s]
  5%|▌         | 52/1000 [00:01<00:18, 51.88it/s]
  6%|▌         | 58/1000 [00:01<00:17, 53.40it/s]
  6%|▋         | 64/1000 [00:01<00:17, 54.58it/s]
  7%|▋         | 70/1000 [00:02<00:16, 55.38it/s]
  8%|▊         | 76/1000 [00:02<00:16, 55.95it/s]
  8%|▊         | 82/1000 [00:02<00:16, 56.58it/s]
  9%|▉         | 88/1000 [00:02<00:16, 56.96it/s]
  9%|▉         | 94/1000 [00:02<00:16, 53.61it/s]
 10%|█         | 100/1000 [00:02<00:16, 54.81it/s]
 11%|█         | 106/1000 [00:02<00:16, 55.82it/s]
 11%|█         | 112/1000 [00:02<00:15, 56.49it/s]
 12%|█▏        | 118/1000 [00:02<00:15, 56.98it/s]
 12%|█▏        | 124/1000 [00:02<00:15, 57.00it/s]
 13%|█▎        | 130/1000 [00:03<00:15, 57.33it/s]
 14%|█▎        | 136/1000 [00:03<00:15, 57.48it/s]
 14%|█▍        | 142/1000 [00:03<00:14, 57.54it/s]
 15%|█▍        | 148/1000 [00:03<00:14, 57.71it/s]
 15%|█▌        | 154/1000 [00:03<00:14, 57.83it/s]
 16%|█▌        | 160/1000 [00:03<00:14, 57.75it/s]
 17%|█▋        | 166/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 172/1000 [00:03<00:14, 57.66it/s]
 18%|█▊        | 178/1000 [00:03<00:14, 57.68it/s]
 18%|█▊        | 184/1000 [00:04<00:14, 57.56it/s]
 19%|█▉        | 190/1000 [00:04<00:14, 57.57it/s]
 20%|█▉        | 196/1000 [00:04<00:13, 57.53it/s]
 20%|██        | 202/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 208/1000 [00:04<00:13, 57.22it/s]
 21%|██▏       | 214/1000 [00:04<00:13, 57.24it/s]
 22%|██▏       | 220/1000 [00:04<00:13, 57.28it/s]
 23%|██▎       | 226/1000 [00:04<00:13, 57.43it/s]
 23%|██▎       | 232/1000 [00:04<00:13, 57.53it/s]
 24%|██▍       | 238/1000 [00:04<00:13, 57.49it/s]
 24%|██▍       | 244/1000 [00:05<00:13, 57.40it/s]
 25%|██▌       | 250/1000 [00:05<00:13, 57.50it/s]
 26%|██▌       | 256/1000 [00:05<00:12, 57.68it/s]
 26%|██▌       | 262/1000 [00:05<00:12, 57.73it/s]
 27%|██▋       | 268/1000 [00:05<00:12, 57.65it/s]
 27%|██▋       | 274/1000 [00:05<00:12, 57.49it/s]
 28%|██▊       | 280/1000 [00:05<00:12, 57.69it/s]
 29%|██▊       | 286/1000 [00:05<00:12, 57.65it/s]
 29%|██▉       | 292/1000 [00:05<00:12, 57.89it/s]
 30%|██▉       | 298/1000 [00:05<00:12, 57.92it/s]
 30%|███       | 304/1000 [00:06<00:12, 57.97it/s]
 31%|███       | 310/1000 [00:06<00:11, 58.05it/s]
 32%|███▏      | 316/1000 [00:06<00:11, 58.06it/s]
 32%|███▏      | 322/1000 [00:06<00:11, 58.08it/s]
 33%|███▎      | 328/1000 [00:06<00:11, 57.84it/s]
 33%|███▎      | 334/1000 [00:06<00:11, 57.87it/s]
 34%|███▍      | 340/1000 [00:06<00:11, 57.89it/s]
 35%|███▍      | 346/1000 [00:06<00:11, 58.04it/s]
 35%|███▌      | 352/1000 [00:06<00:11, 57.90it/s]
 36%|███▌      | 358/1000 [00:07<00:11, 57.99it/s]
 36%|███▋      | 364/1000 [00:07<00:10, 57.98it/s]
 37%|███▋      | 370/1000 [00:07<00:10, 57.98it/s]
 38%|███▊      | 376/1000 [00:07<00:10, 57.85it/s]
 38%|███▊      | 382/1000 [00:07<00:10, 57.65it/s]
 39%|███▉      | 388/1000 [00:07<00:10, 57.58it/s]
 39%|███▉      | 394/1000 [00:07<00:10, 57.66it/s]
 40%|████      | 400/1000 [00:07<00:10, 57.88it/s]
 41%|████      | 406/1000 [00:07<00:10, 57.93it/s]
 41%|████      | 412/1000 [00:07<00:10, 58.03it/s]
 42%|████▏     | 418/1000 [00:08<00:10, 58.05it/s]
 42%|████▏     | 424/1000 [00:08<00:09, 57.95it/s]
 43%|████▎     | 430/1000 [00:08<00:09, 58.11it/s]
 44%|████▎     | 436/1000 [00:08<00:09, 58.13it/s]
 44%|████▍     | 442/1000 [00:08<00:09, 57.92it/s]
 45%|████▍     | 448/1000 [00:08<00:09, 57.93it/s]
 45%|████▌     | 454/1000 [00:08<00:09, 57.98it/s]
 46%|████▌     | 460/1000 [00:08<00:09, 57.96it/s]
 47%|████▋     | 466/1000 [00:08<00:09, 58.04it/s]
 47%|████▋     | 472/1000 [00:08<00:09, 58.02it/s]
 48%|████▊     | 478/1000 [00:09<00:08, 58.09it/s]
 48%|████▊     | 484/1000 [00:09<00:08, 57.93it/s]
 49%|████▉     | 490/1000 [00:09<00:08, 57.91it/s]
 50%|████▉     | 496/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 502/1000 [00:09<00:08, 57.94it/s]
 51%|█████     | 508/1000 [00:09<00:08, 57.87it/s]
 51%|█████▏    | 514/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 520/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 526/1000 [00:09<00:08, 58.01it/s]
 53%|█████▎    | 532/1000 [00:10<00:08, 57.97it/s]
 54%|█████▍    | 538/1000 [00:10<00:07, 58.03it/s]
 54%|█████▍    | 544/1000 [00:10<00:07, 58.10it/s]
 55%|█████▌    | 550/1000 [00:10<00:07, 57.98it/s]
 56%|█████▌    | 556/1000 [00:10<00:07, 57.95it/s]
 56%|█████▌    | 562/1000 [00:10<00:07, 58.00it/s]
 57%|█████▋    | 568/1000 [00:10<00:07, 57.92it/s]
 57%|█████▋    | 574/1000 [00:10<00:07, 57.85it/s]
 58%|█████▊    | 580/1000 [00:10<00:07, 58.01it/s]
 59%|█████▊    | 586/1000 [00:10<00:07, 57.99it/s]
 59%|█████▉    | 592/1000 [00:11<00:07, 57.92it/s]
 60%|█████▉    | 598/1000 [00:11<00:06, 57.98it/s]
 60%|██████    | 604/1000 [00:11<00:06, 57.89it/s]
 61%|██████    | 610/1000 [00:11<00:06, 58.02it/s]
 62%|██████▏   | 616/1000 [00:11<00:06, 58.01it/s]
 62%|██████▏   | 622/1000 [00:11<00:06, 58.07it/s]
 63%|██████▎   | 628/1000 [00:11<00:06, 58.14it/s]
 63%|██████▎   | 634/1000 [00:11<00:06, 57.99it/s]
 64%|██████▍   | 640/1000 [00:11<00:06, 57.93it/s]
 65%|██████▍   | 646/1000 [00:11<00:06, 58.06it/s]
 65%|██████▌   | 652/1000 [00:12<00:05, 58.06it/s]
 66%|██████▌   | 658/1000 [00:12<00:05, 58.07it/s]
 66%|██████▋   | 664/1000 [00:12<00:05, 58.08it/s]
 67%|██████▋   | 670/1000 [00:12<00:05, 58.07it/s]
 68%|██████▊   | 676/1000 [00:12<00:05, 58.11it/s]
 68%|██████▊   | 682/1000 [00:12<00:05, 58.08it/s]
 69%|██████▉   | 688/1000 [00:12<00:05, 58.15it/s]
 69%|██████▉   | 694/1000 [00:12<00:05, 58.01it/s]
 70%|███████   | 700/1000 [00:12<00:05, 58.10it/s]
 71%|███████   | 706/1000 [00:13<00:05, 58.05it/s]
 71%|███████   | 712/1000 [00:13<00:04, 58.10it/s]
 72%|███████▏  | 718/1000 [00:13<00:04, 58.06it/s]
 72%|███████▏  | 724/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 730/1000 [00:13<00:04, 58.04it/s]
 74%|███████▎  | 736/1000 [00:13<00:04, 58.09it/s]
 74%|███████▍  | 742/1000 [00:13<00:04, 58.00it/s]
 75%|███████▍  | 748/1000 [00:13<00:04, 58.07it/s]
 75%|███████▌  | 754/1000 [00:13<00:04, 58.09it/s]
 76%|███████▌  | 760/1000 [00:13<00:04, 57.94it/s]
 77%|███████▋  | 766/1000 [00:14<00:04, 58.01it/s]
 77%|███████▋  | 772/1000 [00:14<00:03, 58.03it/s]
 78%|███████▊  | 778/1000 [00:14<00:03, 58.08it/s]
 78%|███████▊  | 784/1000 [00:14<00:03, 58.08it/s]
 79%|███████▉  | 790/1000 [00:14<00:03, 57.93it/s]
 80%|███████▉  | 796/1000 [00:14<00:03, 57.97it/s]
 80%|████████  | 802/1000 [00:14<00:03, 57.91it/s]
 81%|████████  | 808/1000 [00:14<00:03, 57.90it/s]
 81%|████████▏ | 814/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 820/1000 [00:14<00:03, 58.01it/s]
 83%|████████▎ | 826/1000 [00:15<00:03, 57.95it/s]
 83%|████████▎ | 832/1000 [00:15<00:02, 58.11it/s]
 84%|████████▍ | 838/1000 [00:15<00:02, 58.19it/s]
 84%|████████▍ | 844/1000 [00:15<00:02, 58.07it/s]
 85%|████████▌ | 850/1000 [00:15<00:02, 58.11it/s]
 86%|████████▌ | 856/1000 [00:15<00:02, 58.07it/s]
 86%|████████▌ | 862/1000 [00:15<00:02, 58.04it/s]
 87%|████████▋ | 868/1000 [00:15<00:02, 57.99it/s]
 87%|████████▋ | 874/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 880/1000 [00:16<00:02, 58.05it/s]
 89%|████████▊ | 886/1000 [00:16<00:01, 57.92it/s]
 89%|████████▉ | 892/1000 [00:16<00:01, 57.99it/s]
 90%|████████▉ | 898/1000 [00:16<00:01, 58.04it/s]
 90%|█████████ | 904/1000 [00:16<00:01, 58.04it/s]
 91%|█████████ | 910/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 916/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 922/1000 [00:16<00:01, 57.95it/s]
 93%|█████████▎| 928/1000 [00:16<00:01, 58.11it/s]
 93%|█████████▎| 934/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▍| 940/1000 [00:17<00:01, 58.14it/s]
 95%|█████████▍| 946/1000 [00:17<00:00, 58.06it/s]
 95%|█████████▌| 952/1000 [00:17<00:00, 58.09it/s]
 96%|█████████▌| 958/1000 [00:17<00:00, 58.05it/s]
 96%|█████████▋| 964/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 970/1000 [00:17<00:00, 58.20it/s]
 98%|█████████▊| 976/1000 [00:17<00:00, 58.26it/s]
 98%|█████████▊| 982/1000 [00:17<00:00, 58.26it/s]
 99%|█████████▉| 988/1000 [00:17<00:00, 58.16it/s]
 99%|█████████▉| 994/1000 [00:17<00:00, 58.12it/s]
-2022-12-21,09:55:03 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:16, 114.32it/s]
  1%|          | 512/50176 [00:04<06:27, 128.28it/s]
  2%|▏         | 768/50176 [00:05<06:01, 136.74it/s]
  2%|▏         | 1024/50176 [00:07<05:51, 139.91it/s]
  3%|▎         | 1280/50176 [00:09<05:48, 140.22it/s]
  3%|▎         | 1536/50176 [00:11<06:01, 134.62it/s]
  4%|▎         | 1792/50176 [00:13<05:56, 135.67it/s]
  4%|▍         | 2048/50176 [00:15<05:48, 138.26it/s]
  5%|▍         | 2304/50176 [00:16<05:46, 138.26it/s]
  5%|▌         | 2560/50176 [00:18<05:38, 140.69it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 139.74it/s]
  6%|▌         | 3072/50176 [00:22<05:31, 142.10it/s]
  7%|▋         | 3328/50176 [00:24<05:27, 143.14it/s]
  7%|▋         | 3584/50176 [00:25<05:20, 145.30it/s]
  8%|▊         | 3840/50176 [00:27<05:31, 139.86it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.15it/s]
  9%|▊         | 4352/50176 [00:31<05:22, 142.08it/s]
  9%|▉         | 4608/50176 [00:33<05:18, 142.86it/s]
 10%|▉         | 4864/50176 [00:34<05:12, 144.81it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.33it/s]
 11%|█         | 5376/50176 [00:38<05:04, 147.11it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.43it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 148.08it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 136.15it/s]
 13%|█▎        | 6400/50176 [00:45<05:12, 140.27it/s]
 13%|█▎        | 6656/50176 [00:47<05:05, 142.65it/s]
 14%|█▍        | 6912/50176 [00:49<05:03, 142.45it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 143.90it/s]
 15%|█▍        | 7424/50176 [00:52<04:51, 146.49it/s]
 15%|█▌        | 7680/50176 [00:54<04:51, 145.80it/s]
 16%|█▌        | 7936/50176 [00:55<04:42, 149.28it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.38it/s]
 17%|█▋        | 8448/50176 [00:59<04:38, 149.75it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.58it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.38it/s]
 18%|█▊        | 9216/50176 [01:04<04:35, 148.53it/s]
 19%|█▉        | 9472/50176 [01:06<04:32, 149.29it/s]
 19%|█▉        | 9728/50176 [01:08<04:36, 146.34it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.24it/s]
 20%|██        | 10240/50176 [01:11<04:26, 150.11it/s]
 21%|██        | 10496/50176 [01:12<04:21, 151.70it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.27it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.34it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.14it/s]
 23%|██▎       | 11520/50176 [01:19<04:18, 149.62it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 149.02it/s]
 24%|██▍       | 12032/50176 [01:23<04:15, 149.07it/s]
 24%|██▍       | 12288/50176 [01:25<04:12, 149.80it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.31it/s]
 26%|██▌       | 12800/50176 [01:28<04:07, 150.80it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.55it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.85it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 145.07it/s]
 28%|██▊       | 13824/50176 [01:35<04:15, 142.21it/s]
 28%|██▊       | 14080/50176 [01:37<04:12, 142.90it/s]
 29%|██▊       | 14336/50176 [01:39<04:16, 139.75it/s]
 29%|██▉       | 14592/50176 [01:40<04:04, 145.32it/s]
 30%|██▉       | 14848/50176 [01:43<04:17, 137.12it/s]
 30%|███       | 15104/50176 [01:44<04:18, 135.62it/s]
 31%|███       | 15360/50176 [01:46<04:09, 139.33it/s]
 31%|███       | 15616/50176 [01:48<04:04, 141.38it/s]
 32%|███▏      | 15872/50176 [01:50<04:08, 137.82it/s]
 32%|███▏      | 16128/50176 [01:52<04:02, 140.44it/s]
 33%|███▎      | 16384/50176 [01:53<04:00, 140.58it/s]
 33%|███▎      | 16640/50176 [01:56<04:07, 135.47it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.74it/s]
 34%|███▍      | 17152/50176 [01:59<03:49, 144.10it/s]
 35%|███▍      | 17408/50176 [02:01<03:45, 145.26it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.32it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.18it/s]
 36%|███▌      | 18176/50176 [02:06<03:35, 148.50it/s]
 37%|███▋      | 18432/50176 [02:07<03:33, 148.77it/s]
 37%|███▋      | 18688/50176 [02:09<03:34, 147.09it/s]
 38%|███▊      | 18944/50176 [02:11<03:40, 141.70it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.97it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.40it/s]
 39%|███▉      | 19712/50176 [02:17<03:51, 131.73it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.43it/s]
 40%|████      | 20224/50176 [02:21<03:32, 140.86it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.91it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 143.90it/s]
 42%|████▏     | 20992/50176 [02:26<03:19, 146.64it/s]
 42%|████▏     | 21248/50176 [02:28<03:23, 141.94it/s]
 43%|████▎     | 21504/50176 [02:29<03:17, 145.12it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.15it/s]
 44%|████▍     | 22016/50176 [02:33<03:07, 150.40it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.81it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.48it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.37it/s]
 46%|████▌     | 23040/50176 [02:40<03:09, 143.19it/s]
 46%|████▋     | 23296/50176 [02:42<03:12, 139.47it/s]
 47%|████▋     | 23552/50176 [02:44<03:14, 136.69it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.90it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.43it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.07it/s]
 49%|████▉     | 24576/50176 [02:51<02:55, 145.50it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.78it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 149.04it/s]
 51%|█████     | 25344/50176 [02:56<02:46, 149.54it/s]
 51%|█████     | 25600/50176 [02:58<02:47, 147.06it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.09it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.07it/s]
 53%|█████▎    | 26368/50176 [03:03<02:45, 143.65it/s]
 53%|█████▎    | 26624/50176 [03:05<02:40, 146.33it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 145.94it/s]
 54%|█████▍    | 27136/50176 [03:08<02:36, 147.48it/s]
 55%|█████▍    | 27392/50176 [03:10<02:34, 147.71it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.73it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.37it/s]
 56%|█████▌    | 28160/50176 [03:15<02:33, 143.67it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.37it/s]
 57%|█████▋    | 28672/50176 [03:19<02:32, 141.04it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.12it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.34it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.29it/s]
 59%|█████▉    | 29696/50176 [03:26<02:32, 134.32it/s]
 60%|█████▉    | 29952/50176 [03:28<02:30, 134.03it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 133.02it/s]
 61%|██████    | 30464/50176 [03:32<02:20, 140.28it/s]
 61%|██████    | 30720/50176 [03:34<02:18, 140.35it/s]
 62%|██████▏   | 30976/50176 [03:35<02:14, 142.84it/s]
 62%|██████▏   | 31232/50176 [03:37<02:08, 146.88it/s]
 63%|██████▎   | 31488/50176 [03:39<02:05, 148.37it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 148.54it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 146.93it/s]
 64%|██████▍   | 32256/50176 [03:44<02:01, 147.39it/s]
 65%|██████▍   | 32512/50176 [03:46<02:04, 142.17it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 145.74it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.50it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 145.72it/s]
 67%|██████▋   | 33536/50176 [03:53<01:56, 142.70it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.46it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.51it/s]
 68%|██████▊   | 34304/50176 [03:58<01:44, 151.72it/s]
 69%|██████▉   | 34560/50176 [04:00<01:47, 144.61it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.53it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.14it/s]
 70%|███████   | 35328/50176 [04:05<01:40, 148.06it/s]
 71%|███████   | 35584/50176 [04:07<01:39, 146.36it/s]
 71%|███████▏  | 35840/50176 [04:09<01:41, 141.64it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 143.79it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.60it/s]
 73%|███████▎  | 36608/50176 [04:14<01:34, 144.33it/s]
 73%|███████▎  | 36864/50176 [04:16<01:33, 142.06it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.71it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.31it/s]
 75%|███████▌  | 37632/50176 [04:21<01:27, 144.17it/s]
 76%|███████▌  | 37888/50176 [04:23<01:25, 143.68it/s]
 76%|███████▌  | 38144/50176 [04:25<01:22, 145.80it/s]
 77%|███████▋  | 38400/50176 [04:26<01:21, 145.32it/s]
 77%|███████▋  | 38656/50176 [04:28<01:19, 144.11it/s]
 78%|███████▊  | 38912/50176 [04:30<01:18, 142.66it/s]
 78%|███████▊  | 39168/50176 [04:32<01:19, 138.34it/s]
 79%|███████▊  | 39424/50176 [04:34<01:16, 141.27it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.34it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.29it/s]
 80%|████████  | 40192/50176 [04:39<01:07, 147.24it/s]
 81%|████████  | 40448/50176 [04:41<01:09, 140.84it/s]
 81%|████████  | 40704/50176 [04:43<01:10, 135.25it/s]
 82%|████████▏ | 40960/50176 [04:45<01:05, 140.42it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.21it/s]
 83%|████████▎ | 41472/50176 [04:48<00:59, 146.06it/s]
 83%|████████▎ | 41728/50176 [04:50<00:56, 149.28it/s]
 84%|████████�� | 41984/50176 [04:51<00:54, 149.40it/s]
 84%|████████▍ | 42240/50176 [04:53<00:54, 145.49it/s]
 85%|████████▍ | 42496/50176 [04:55<00:51, 148.42it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.94it/s]
 86%|████████▌ | 43008/50176 [04:58<00:48, 149.07it/s]
 86%|████████▌ | 43264/50176 [05:00<00:46, 149.43it/s]
 87%|████████▋ | 43520/50176 [05:02<00:44, 148.28it/s]
 87%|████████▋ | 43776/50176 [05:06<01:03, 101.49it/s]
 88%|████████▊ | 44032/50176 [05:08<00:54, 113.60it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.29it/s]
 89%|████████▉ | 44544/50176 [05:11<00:43, 129.93it/s]
 89%|████████▉ | 44800/50176 [05:13<00:40, 133.77it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.08it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.48it/s]
 91%|█████████ | 45568/50176 [05:19<00:35, 128.29it/s]
 91%|█████████▏| 45824/50176 [05:21<00:32, 135.68it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.83it/s]
 92%|█████████▏| 46336/50176 [05:24<00:26, 145.11it/s]
 93%|█████████▎| 46592/50176 [05:26<00:24, 147.96it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.55it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 145.06it/s]
 94%|█████████▍| 47360/50176 [05:31<00:19, 146.44it/s]
 95%|█████████▍| 47616/50176 [05:33<00:17, 145.30it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.32it/s]
 96%|█████████▌| 48128/50176 [05:36<00:13, 150.22it/s]
 96%|█████████▋| 48384/50176 [05:38<00:11, 152.09it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.58it/s]
 97%|█████████▋| 48896/50176 [05:41<00:08, 154.83it/s]
 98%|█████████▊| 49152/50176 [05:43<00:06, 150.80it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.19it/s]
 99%|█████████▉| 49664/50176 [05:46<00:03, 149.80it/s]
 99%|█████████▉| 49920/50176 [05:48<00:01, 142.19it/s]
-2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5534	imagenet-zeroshot-val-top5: 0.8278

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_19.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt).
-2022-12-21,09:54:45 | INFO | Model:
-2022-12-21,09:54:45 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:54:45 | INFO | Params:
-2022-12-21,09:54:45 | INFO |   batch_size: 256
-2022-12-21,09:54:45 | INFO |   beta1: 0.9
-2022-12-21,09:54:45 | INFO |   beta2: 0.98
-2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/checkpoints
-2022-12-21,09:54:45 | INFO |   copy_codebase: False
-2022-12-21,09:54:45 | INFO |   csv_caption_key: title
-2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
-2022-12-21,09:54:45 | INFO |   csv_separator: 	
-2022-12-21,09:54:45 | INFO |   dataset_resampled: False
-2022-12-21,09:54:45 | INFO |   dataset_type: auto
-2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
-2022-12-21,09:54:45 | INFO |   debug: False
-2022-12-21,09:54:45 | INFO |   device: cuda:0
-2022-12-21,09:54:45 | INFO |   dist_backend: nccl
-2022-12-21,09:54:45 | INFO |   dist_url: env://
-2022-12-21,09:54:45 | INFO |   distributed: False
-2022-12-21,09:54:45 | INFO |   epochs: 32
-2022-12-21,09:54:45 | INFO |   eps: 1e-06
-2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
-2022-12-21,09:54:45 | INFO |   gather_with_grad: False
-2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
-2022-12-21,09:54:45 | INFO |   horovod: False
-2022-12-21,09:54:45 | INFO |   image_mean: None
-2022-12-21,09:54:45 | INFO |   image_std: None
-2022-12-21,09:54:45 | INFO |   imagenet_v2: None
-2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:54:45 | INFO |   local_loss: False
-2022-12-21,09:54:45 | INFO |   local_rank: 0
-2022-12-21,09:54:45 | INFO |   lock_image: False
-2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:54:45 | INFO |   log_level: 20
-2022-12-21,09:54:45 | INFO |   log_local: False
-2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_19.pt/out.log
-2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:54:45 | INFO |   lr: 0.0005
-2022-12-21,09:54:45 | INFO |   model: ViT-B-16
-2022-12-21,09:54:45 | INFO |   name: epoch_19.pt
-2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
-2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
-2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
-2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_19.pt
-2022-12-21,09:54:45 | INFO |   pretrained_image: False
-2022-12-21,09:54:45 | INFO |   rank: 0
-2022-12-21,09:54:45 | INFO |   report_to: 
-2022-12-21,09:54:45 | INFO |   resume: None
-2022-12-21,09:54:45 | INFO |   save_frequency: 1
-2022-12-21,09:54:45 | INFO |   save_most_recent: False
-2022-12-21,09:54:45 | INFO |   seed: 0
-2022-12-21,09:54:45 | INFO |   skip_scheduler: False
-2022-12-21,09:54:45 | INFO |   tensorboard: False
-2022-12-21,09:54:45 | INFO |   tensorboard_path: 
-2022-12-21,09:54:45 | INFO |   torchscript: False
-2022-12-21,09:54:45 | INFO |   trace: False
-2022-12-21,09:54:45 | INFO |   train_data: None
-2022-12-21,09:54:45 | INFO |   train_num_samples: None
-2022-12-21,09:54:45 | INFO |   use_bn_sync: False
-2022-12-21,09:54:45 | INFO |   val_data: None
-2022-12-21,09:54:45 | INFO |   val_frequency: 1
-2022-12-21,09:54:45 | INFO |   val_num_samples: None
-2022-12-21,09:54:45 | INFO |   wandb: False
-2022-12-21,09:54:45 | INFO |   wandb_notes: 
-2022-12-21,09:54:45 | INFO |   warmup: 10000
-2022-12-21,09:54:45 | INFO |   wd: 0.2
-2022-12-21,09:54:45 | INFO |   workers: 1
-2022-12-21,09:54:45 | INFO |   world_size: 1
-2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:54:46 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:54:46 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:41,  1.56it/s]
  1%|          | 6/1000 [00:00<01:35, 10.36it/s]
  1%|          | 12/1000 [00:00<00:49, 19.86it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.10it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.26it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.07it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.38it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.71it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.20it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.03it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.38it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.11it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.82it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.80it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.13it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.54it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.75it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.01it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.15it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.12it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.04it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.03it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.97it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.02it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.10it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.13it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.00it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.87it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.90it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.81it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.75it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.74it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.69it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.37it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.50it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.44it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.62it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.54it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.62it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.73it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.82it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.86it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.67it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.89it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.99it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.20it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.18it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.17it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.30it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.18it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.01it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.10it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.20it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.29it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.07it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 58.07it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.96it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.03it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.72it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.81it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.92it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.09it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.18it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.19it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.21it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.23it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.33it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.12it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.19it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.25it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.90it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.13it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.09it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.08it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.11it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.03it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.09it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.98it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.06it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.20it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.14it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.26it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.21it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.09it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.06it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.03it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.16it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.13it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.16it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.07it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.19it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.27it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.10it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.05it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.19it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.13it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.26it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.20it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.30it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.19it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.29it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.27it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.25it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.27it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.22it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.24it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.20it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.29it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.37it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.19it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.31it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.26it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.29it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.30it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.10it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.11it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.18it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.15it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.24it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.15it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.20it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.14it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.19it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.16it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.18it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.17it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.03it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.14it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.10it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.11it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.25it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.26it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.31it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.44it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.05it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.10it/s]
-2022-12-21,09:55:04 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:57, 139.64it/s]
  1%|          | 512/50176 [00:03<06:00, 137.68it/s]
  2%|▏         | 768/50176 [00:05<05:46, 142.67it/s]
  2%|▏         | 1024/50176 [00:07<05:41, 143.90it/s]
  3%|▎         | 1280/50176 [00:09<05:43, 142.33it/s]
  3%|▎         | 1536/50176 [00:11<05:57, 136.16it/s]
  4%|▎         | 1792/50176 [00:12<05:54, 136.56it/s]
  4%|▍         | 2048/50176 [00:14<05:45, 139.15it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 139.02it/s]
  5%|▌         | 2560/50176 [00:18<05:37, 141.20it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 139.99it/s]
  6%|▌         | 3072/50176 [00:21<05:30, 142.53it/s]
  7%|▋         | 3328/50176 [00:23<05:26, 143.34it/s]
  7%|▋         | 3584/50176 [00:25<05:20, 145.49it/s]
  8%|▊         | 3840/50176 [00:27<05:30, 140.04it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.26it/s]
  9%|▊         | 4352/50176 [00:30<05:22, 141.96it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.55it/s]
 10%|▉         | 4864/50176 [00:34<05:12, 144.93it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.45it/s]
 11%|█         | 5376/50176 [00:37<05:04, 146.98it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.19it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 147.92it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 136.02it/s]
 13%|█▎        | 6400/50176 [00:45<05:11, 140.53it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.44it/s]
 14%|█▍        | 6912/50176 [00:48<05:03, 142.50it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 144.09it/s]
 15%|█▍        | 7424/50176 [00:52<04:51, 146.63it/s]
 15%|█▌        | 7680/50176 [00:53<04:51, 145.87it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 148.95it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.55it/s]
 17%|█▋        | 8448/50176 [00:58<04:38, 149.69it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.51it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.59it/s]
 18%|█▊        | 9216/50176 [01:04<04:35, 148.72it/s]
 19%|█▉        | 9472/50176 [01:05<04:32, 149.35it/s]
 19%|█▉        | 9728/50176 [01:07<04:37, 145.53it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.29it/s]
 20%|██        | 10240/50176 [01:10<04:26, 150.09it/s]
 21%|██        | 10496/50176 [01:12<04:21, 152.00it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.27it/s]
 22%|██▏       | 11008/50176 [01:16<04:26, 146.86it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.49it/s]
 23%|██▎       | 11520/50176 [01:19<04:19, 149.18it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 149.18it/s]
 24%|██▍       | 12032/50176 [01:23<04:14, 149.85it/s]
 24%|██▍       | 12288/50176 [01:24<04:12, 149.90it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.38it/s]
 26%|██▌       | 12800/50176 [01:28<04:08, 150.68it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.40it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.75it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 145.19it/s]
 28%|██▊       | 13824/50176 [01:35<04:15, 142.25it/s]
 28%|██▊       | 14080/50176 [01:37<04:12, 142.79it/s]
 29%|██▊       | 14336/50176 [01:38<04:16, 139.97it/s]
 29%|██▉       | 14592/50176 [01:40<04:05, 145.18it/s]
 30%|██▉       | 14848/50176 [01:42<04:17, 137.15it/s]
 30%|███       | 15104/50176 [01:44<04:18, 135.51it/s]
 31%|███       | 15360/50176 [01:46<04:10, 139.26it/s]
 31%|███       | 15616/50176 [01:48<04:04, 141.20it/s]
 32%|███▏      | 15872/50176 [01:50<04:08, 137.88it/s]
 32%|███▏      | 16128/50176 [01:51<04:02, 140.42it/s]
 33%|███▎      | 16384/50176 [01:53<04:01, 140.17it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.73it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.78it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.15it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.46it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.26it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.15it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.48it/s]
 37%|███▋      | 18432/50176 [02:07<03:33, 148.77it/s]
 37%|███▋      | 18688/50176 [02:09<03:33, 147.19it/s]
 38%|███▊      | 18944/50176 [02:11<03:40, 141.65it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.92it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.80it/s]
 39%|███▉      | 19712/50176 [02:17<03:51, 131.72it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.30it/s]
 40%|████      | 20224/50176 [02:20<03:32, 140.77it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.83it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 144.10it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.54it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 141.88it/s]
 43%|████▎     | 21504/50176 [02:29<03:17, 145.25it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.23it/s]
 44%|████▍     | 22016/50176 [02:32<03:07, 150.46it/s]
 44%|████▍     | 22272/50176 [02:34<03:04, 150.86it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.31it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.27it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.18it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.38it/s]
 47%|████▋     | 23552/50176 [02:43<03:14, 136.56it/s]
 47%|████▋     | 23808/50176 [02:45<03:06, 141.06it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.38it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.02it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.48it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.78it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 149.04it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.55it/s]
 51%|█████     | 25600/50176 [02:57<02:47, 147.03it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.30it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.20it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 143.73it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 146.18it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 146.02it/s]
 54%|█████▍    | 27136/50176 [03:08<02:36, 147.37it/s]
 55%|█████▍    | 27392/50176 [03:09<02:34, 147.84it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.81it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.26it/s]
 56%|█████▌    | 28160/50176 [03:15<02:33, 143.73it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.27it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.20it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 142.98it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.43it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.18it/s]
 59%|█████▉    | 29696/50176 [03:26<02:32, 134.47it/s]
 60%|█████▉    | 29952/50176 [03:28<02:31, 133.88it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 133.10it/s]
 61%|██████    | 30464/50176 [03:31<02:20, 140.13it/s]
 61%|██████    | 30720/50176 [03:33<02:19, 139.96it/s]
 62%|██████▏   | 30976/50176 [03:35<02:13, 143.54it/s]
 62%|██████▏   | 31232/50176 [03:37<02:09, 146.71it/s]
 63%|██████▎   | 31488/50176 [03:38<02:06, 148.24it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 147.81it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 147.09it/s]
 64%|██████▍   | 32256/50176 [03:43<02:01, 147.50it/s]
 65%|██████▍   | 32512/50176 [03:45<02:04, 142.02it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 146.03it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.42it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 145.99it/s]
 67%|██████▋   | 33536/50176 [03:52<01:56, 142.67it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.44it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.71it/s]
 68%|██████▊   | 34304/50176 [03:57<01:44, 151.64it/s]
 69%|██████▉   | 34560/50176 [03:59<01:48, 144.57it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.51it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.12it/s]
 70%|███████   | 35328/50176 [04:04<01:40, 148.21it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 146.24it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 141.61it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 143.77it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.76it/s]
 73%|███████▎  | 36608/50176 [04:13<01:34, 144.27it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 141.81it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.77it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.40it/s]
 75%|███████▌  | 37632/50176 [04:21<01:27, 144.17it/s]
 76%|███████▌  | 37888/50176 [04:22<01:25, 143.84it/s]
 76%|███████▌  | 38144/50176 [04:24<01:22, 145.94it/s]
 77%|███████▋  | 38400/50176 [04:26<01:20, 145.41it/s]
 77%|███████▋  | 38656/50176 [04:28<01:20, 144.00it/s]
 78%|███████▊  | 38912/50176 [04:30<01:18, 142.84it/s]
 78%|███████▊  | 39168/50176 [04:32<01:19, 138.21it/s]
 79%|███████▊  | 39424/50176 [04:33<01:16, 141.19it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.41it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.61it/s]
 80%|████████  | 40192/50176 [04:38<01:07, 147.06it/s]
 81%|████████  | 40448/50176 [04:40<01:09, 140.82it/s]
 81%|████████  | 40704/50176 [04:42<01:10, 135.10it/s]
 82%|████████▏ | 40960/50176 [04:44<01:05, 140.39it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.16it/s]
 83%|████████▎ | 41472/50176 [04:47<00:59, 146.67it/s]
 83%|████████▎ | 41728/50176 [04:49<00:56, 149.22it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.15it/s]
 84%|██████���█▍ | 42240/50176 [04:53<00:54, 145.96it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 148.11it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.90it/s]
 86%|████████▌ | 43008/50176 [04:58<00:47, 149.52it/s]
 86%|████████▌ | 43264/50176 [04:59<00:46, 149.09it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.06it/s]
 87%|████████▋ | 43776/50176 [05:06<01:03, 101.40it/s]
 88%|████████▊ | 44032/50176 [05:07<00:54, 113.47it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.35it/s]
 89%|████████▉ | 44544/50176 [05:11<00:43, 129.70it/s]
 89%|████████▉ | 44800/50176 [05:12<00:40, 134.08it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.15it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.55it/s]
 91%|█████████ | 45568/50176 [05:19<00:35, 128.35it/s]
 91%|█████████▏| 45824/50176 [05:20<00:32, 135.68it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.64it/s]
 92%|█████████▏| 46336/50176 [05:24<00:26, 144.31it/s]
 93%|█████████▎| 46592/50176 [05:25<00:24, 148.07it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.66it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 145.12it/s]
 94%|█████████▍| 47360/50176 [05:30<00:19, 146.50it/s]
 95%|█████████▍| 47616/50176 [05:32<00:17, 145.48it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.50it/s]
 96%|█████████▌| 48128/50176 [05:36<00:13, 150.31it/s]
 96%|█████████▋| 48384/50176 [05:37<00:11, 152.01it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.66it/s]
 97%|█████████▋| 48896/50176 [05:40<00:08, 154.75it/s]
 98%|█████████▊| 49152/50176 [05:42<00:06, 150.65it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.43it/s]
 99%|█████████▉| 49664/50176 [05:46<00:03, 149.85it/s]
 99%|█████████▉| 49920/50176 [05:48<00:01, 142.22it/s]
-2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5604	imagenet-zeroshot-val-top5: 0.8313

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_2.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,09:54:36 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,09:54:36 | INFO | Loading ViT-B-16 model config.
-2022-12-21,09:54:37 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt).
-2022-12-21,09:54:45 | INFO | Model:
-2022-12-21,09:54:45 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,09:54:45 | INFO | Params:
-2022-12-21,09:54:45 | INFO |   batch_size: 256
-2022-12-21,09:54:45 | INFO |   beta1: 0.9
-2022-12-21,09:54:45 | INFO |   beta2: 0.98
-2022-12-21,09:54:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/checkpoints
-2022-12-21,09:54:45 | INFO |   copy_codebase: False
-2022-12-21,09:54:45 | INFO |   csv_caption_key: title
-2022-12-21,09:54:45 | INFO |   csv_img_key: filepath
-2022-12-21,09:54:45 | INFO |   csv_separator: 	
-2022-12-21,09:54:45 | INFO |   dataset_resampled: False
-2022-12-21,09:54:45 | INFO |   dataset_type: auto
-2022-12-21,09:54:45 | INFO |   ddp_static_graph: False
-2022-12-21,09:54:45 | INFO |   debug: False
-2022-12-21,09:54:45 | INFO |   device: cuda:0
-2022-12-21,09:54:45 | INFO |   dist_backend: nccl
-2022-12-21,09:54:45 | INFO |   dist_url: env://
-2022-12-21,09:54:45 | INFO |   distributed: False
-2022-12-21,09:54:45 | INFO |   epochs: 32
-2022-12-21,09:54:45 | INFO |   eps: 1e-06
-2022-12-21,09:54:45 | INFO |   force_quick_gelu: False
-2022-12-21,09:54:45 | INFO |   gather_with_grad: False
-2022-12-21,09:54:45 | INFO |   grad_checkpointing: False
-2022-12-21,09:54:45 | INFO |   horovod: False
-2022-12-21,09:54:45 | INFO |   image_mean: None
-2022-12-21,09:54:45 | INFO |   image_std: None
-2022-12-21,09:54:45 | INFO |   imagenet_v2: None
-2022-12-21,09:54:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,09:54:45 | INFO |   local_loss: False
-2022-12-21,09:54:45 | INFO |   local_rank: 0
-2022-12-21,09:54:45 | INFO |   lock_image: False
-2022-12-21,09:54:45 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,09:54:45 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,09:54:45 | INFO |   log_level: 20
-2022-12-21,09:54:45 | INFO |   log_local: False
-2022-12-21,09:54:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_2.pt/out.log
-2022-12-21,09:54:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,09:54:45 | INFO |   lr: 0.0005
-2022-12-21,09:54:45 | INFO |   model: ViT-B-16
-2022-12-21,09:54:45 | INFO |   name: epoch_2.pt
-2022-12-21,09:54:45 | INFO |   no_set_device_rank: False
-2022-12-21,09:54:45 | INFO |   norm_gradient_clip: None
-2022-12-21,09:54:45 | INFO |   precision: amp_bfloat16
-2022-12-21,09:54:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_2.pt
-2022-12-21,09:54:45 | INFO |   pretrained_image: False
-2022-12-21,09:54:45 | INFO |   rank: 0
-2022-12-21,09:54:45 | INFO |   report_to: 
-2022-12-21,09:54:45 | INFO |   resume: None
-2022-12-21,09:54:45 | INFO |   save_frequency: 1
-2022-12-21,09:54:45 | INFO |   save_most_recent: False
-2022-12-21,09:54:45 | INFO |   seed: 0
-2022-12-21,09:54:45 | INFO |   skip_scheduler: False
-2022-12-21,09:54:45 | INFO |   tensorboard: False
-2022-12-21,09:54:45 | INFO |   tensorboard_path: 
-2022-12-21,09:54:45 | INFO |   torchscript: False
-2022-12-21,09:54:45 | INFO |   trace: False
-2022-12-21,09:54:45 | INFO |   train_data: None
-2022-12-21,09:54:45 | INFO |   train_num_samples: None
-2022-12-21,09:54:45 | INFO |   use_bn_sync: False
-2022-12-21,09:54:45 | INFO |   val_data: None
-2022-12-21,09:54:45 | INFO |   val_frequency: 1
-2022-12-21,09:54:45 | INFO |   val_num_samples: None
-2022-12-21,09:54:45 | INFO |   wandb: False
-2022-12-21,09:54:45 | INFO |   wandb_notes: 
-2022-12-21,09:54:45 | INFO |   warmup: 10000
-2022-12-21,09:54:45 | INFO |   wd: 0.2
-2022-12-21,09:54:45 | INFO |   workers: 1
-2022-12-21,09:54:45 | INFO |   world_size: 1
-2022-12-21,09:54:45 | INFO |   zeroshot_frequency: 2
-2022-12-21,09:54:45 | INFO | Starting zero-shot imagenet.
-2022-12-21,09:54:45 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:00,  1.39it/s]
  1%|          | 6/1000 [00:00<01:46,  9.35it/s]
  1%|          | 12/1000 [00:00<00:53, 18.46it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.52it/s]
  2%|▏         | 24/1000 [00:01<00:28, 33.79it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.78it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.44it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.02it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.56it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.54it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.92it/s]
  7%|▋         | 66/1000 [00:01<00:16, 54.99it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.74it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.13it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.71it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.07it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.48it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.68it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.94it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.08it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.07it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.00it/s]
 13%|█▎        | 132/1000 [00:03<00:14, 58.05it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.99it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.03it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.11it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.11it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.97it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.81it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.79it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.70it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.65it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.59it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.42it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.31it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.41it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.36it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.59it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.60it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.54it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.59it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.68it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.78it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.80it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.53it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.59it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.77it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.88it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.10it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.10it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.07it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.19it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.09it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.93it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.02it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.13it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.22it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.02it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.99it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.08it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.17it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.75it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.79it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.90it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.03it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.10it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.12it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.03it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.14it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.12it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.08it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.21it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.12it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.10it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.10it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.00it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.05it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.96it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.03it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.07it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.16it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.11it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.17it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.18it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.09it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.11it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.14it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.99it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.09it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.02it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.17it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.16it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.27it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.10it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.01it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.17it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.13it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.22it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.14it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.17it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.18it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.11it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.02it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.13it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.13it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.10it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.10it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.19it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.28it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.11it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.21it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.15it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.17it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.21it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.99it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.98it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.03it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.00it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.98it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.01it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.12it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.30it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.21it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.22it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.24it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.20it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.17it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.05it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.08it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.09it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.17it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.23it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.23it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.23it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.26it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.25it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.34it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.37it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.36it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.22it/s]
-2022-12-21,09:55:03 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:29, 128.16it/s]
  1%|          | 512/50176 [00:03<06:09, 134.26it/s]
  2%|▏         | 768/50176 [00:05<05:53, 139.93it/s]
  2%|▏         | 1024/50176 [00:07<05:44, 142.66it/s]
  3%|▎         | 1280/50176 [00:09<05:45, 141.67it/s]
  3%|▎         | 1536/50176 [00:11<05:58, 135.79it/s]
  4%|▎         | 1792/50176 [00:13<05:56, 135.58it/s]
  4%|▍         | 2048/50176 [00:14<05:46, 138.98it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 138.79it/s]
  5%|▌         | 2560/50176 [00:18<05:37, 140.95it/s]
  6%|▌         | 2816/50176 [00:20<05:39, 139.66it/s]
  6%|▌         | 3072/50176 [00:22<05:30, 142.63it/s]
  7%|▋         | 3328/50176 [00:23<05:26, 143.39it/s]
  7%|▋         | 3584/50176 [00:25<05:20, 145.45it/s]
  8%|▊         | 3840/50176 [00:27<05:31, 139.98it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.42it/s]
  9%|▊         | 4352/50176 [00:31<05:22, 142.03it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.51it/s]
 10%|▉         | 4864/50176 [00:34<05:13, 144.40it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.59it/s]
 11%|█         | 5376/50176 [00:37<05:05, 146.62it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.38it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 147.69it/s]
 12%|█▏        | 6144/50176 [00:43<05:23, 136.31it/s]
 13%|█▎        | 6400/50176 [00:45<05:11, 140.66it/s]
 13%|█▎        | 6656/50176 [00:47<05:06, 141.93it/s]
 14%|█▍        | 6912/50176 [00:48<05:02, 142.81it/s]
 14%|█▍        | 7168/50176 [00:50<04:58, 144.20it/s]
 15%|█▍        | 7424/50176 [00:52<04:51, 146.71it/s]
 15%|█▌        | 7680/50176 [00:53<04:51, 145.94it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 149.00it/s]
 16%|█▋        | 8192/50176 [00:57<04:42, 148.52it/s]
 17%|█▋        | 8448/50176 [00:59<04:38, 149.78it/s]
 17%|█▋        | 8704/50176 [01:00<04:35, 150.55it/s]
 18%|█▊        | 8960/50176 [01:02<04:43, 145.45it/s]
 18%|█▊        | 9216/50176 [01:04<04:36, 147.91it/s]
 19%|█▉        | 9472/50176 [01:05<04:32, 149.62it/s]
 19%|█▉        | 9728/50176 [01:07<04:36, 146.44it/s]
 20%|█▉        | 9984/50176 [01:09<04:30, 148.33it/s]
 20%|██        | 10240/50176 [01:11<04:26, 149.95it/s]
 21%|██        | 10496/50176 [01:12<04:21, 151.84it/s]
 21%|██▏       | 10752/50176 [01:14<04:27, 147.37it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.56it/s]
 22%|██▏       | 11264/50176 [01:18<04:23, 147.61it/s]
 23%|██▎       | 11520/50176 [01:19<04:17, 149.89it/s]
 23%|██▎       | 11776/50176 [01:21<04:17, 149.02it/s]
 24%|██▍       | 12032/50176 [01:23<04:14, 149.78it/s]
 24%|██▍       | 12288/50176 [01:24<04:12, 149.80it/s]
 25%|██▌       | 12544/50176 [01:26<04:10, 150.25it/s]
 26%|██▌       | 12800/50176 [01:28<04:07, 150.85it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.36it/s]
 27%|██▋       | 13312/50176 [01:31<04:04, 150.77it/s]
 27%|██▋       | 13568/50176 [01:33<04:12, 144.86it/s]
 28%|██▊       | 13824/50176 [01:35<04:14, 142.57it/s]
 28%|██▊       | 14080/50176 [01:37<04:12, 142.91it/s]
 29%|██▊       | 14336/50176 [01:39<04:15, 140.35it/s]
 29%|██▉       | 14592/50176 [01:40<04:05, 145.18it/s]
 30%|██▉       | 14848/50176 [01:42<04:18, 136.83it/s]
 30%|███       | 15104/50176 [01:44<04:17, 136.01it/s]
 31%|███       | 15360/50176 [01:46<04:10, 139.03it/s]
 31%|███       | 15616/50176 [01:48<04:04, 141.59it/s]
 32%|███▏      | 15872/50176 [01:50<04:09, 137.72it/s]
 32%|███▏      | 16128/50176 [01:51<04:02, 140.22it/s]
 33%|███▎      | 16384/50176 [01:53<04:00, 140.49it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.27it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.63it/s]
 34%|███▍      | 17152/50176 [01:59<03:49, 144.08it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.24it/s]
 35%|███▌      | 17664/50176 [02:02<03:40, 147.28it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.20it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.51it/s]
 37%|███▋      | 18432/50176 [02:07<03:34, 148.10it/s]
 37%|███▋      | 18688/50176 [02:09<03:32, 147.92it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.28it/s]
 38%|███▊      | 19200/50176 [02:13<03:51, 133.81it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.35it/s]
 39%|███▉      | 19712/50176 [02:17<03:52, 131.17it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.51it/s]
 40%|████      | 20224/50176 [02:20<03:32, 140.81it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.95it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 143.95it/s]
 42%|████▏     | 20992/50176 [02:26<03:18, 146.70it/s]
 42%|████▏     | 21248/50176 [02:28<03:24, 141.55it/s]
 43%|████▎     | 21504/50176 [02:29<03:16, 145.98it/s]
 43%|████▎     | 21760/50176 [02:31<03:13, 147.09it/s]
 44%|████▍     | 22016/50176 [02:32<03:07, 150.36it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.70it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.38it/s]
 45%|████▌     | 22784/50176 [02:38<03:09, 144.77it/s]
 46%|████▌     | 23040/50176 [02:40<03:09, 143.46it/s]
 46%|████▋     | 23296/50176 [02:42<03:11, 140.13it/s]
 47%|████▋     | 23552/50176 [02:44<03:15, 136.50it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.90it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 142.35it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 142.99it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.48it/s]
 49%|████▉     | 24832/50176 [02:52<02:51, 148.07it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 149.20it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.68it/s]
 51%|█████     | 25600/50176 [02:57<02:47, 146.59it/s]
 52%|█████▏    | 25856/50176 [02:59<02:41, 151.01it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.54it/s]
 53%|█████▎    | 26368/50176 [03:03<02:46, 143.33it/s]
 53%|█████▎    | 26624/50176 [03:04<02:40, 146.52it/s]
 54%|█████▎    | 26880/50176 [03:06<02:39, 146.15it/s]
 54%|█████▍    | 27136/50176 [03:08<02:36, 147.59it/s]
 55%|█████▍    | 27392/50176 [03:09<02:34, 147.93it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.85it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.40it/s]
 56%|█████▌    | 28160/50176 [03:15<02:33, 143.79it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.36it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.06it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.03it/s]
 58%|█████▊    | 29184/50176 [03:22<02:24, 145.33it/s]
 59%|█████▊    | 29440/50176 [03:24<02:31, 137.16it/s]
 59%|█████▉    | 29696/50176 [03:26<02:32, 134.37it/s]
 60%|█████▉    | 29952/50176 [03:28<02:30, 134.48it/s]
 60%|██████    | 30208/50176 [03:30<02:30, 132.56it/s]
 61%|██████    | 30464/50176 [03:31<02:20, 140.31it/s]
 61%|██████    | 30720/50176 [03:33<02:18, 140.55it/s]
 62%|██████▏   | 30976/50176 [03:35<02:13, 143.40it/s]
 62%|██████▏   | 31232/50176 [03:37<02:09, 146.60it/s]
 63%|██████▎   | 31488/50176 [03:38<02:06, 148.13it/s]
 63%|██████▎   | 31744/50176 [03:40<02:04, 148.46it/s]
 64%|██████▍   | 32000/50176 [03:42<02:03, 146.86it/s]
 64%|██████▍   | 32256/50176 [03:44<02:01, 147.32it/s]
 65%|██████▍   | 32512/50176 [03:45<02:03, 142.50it/s]
 65%|██████▌   | 32768/50176 [03:47<01:59, 145.62it/s]
 66%|██████▌   | 33024/50176 [03:49<01:58, 144.35it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 145.81it/s]
 67%|██████▋   | 33536/50176 [03:53<01:56, 142.32it/s]
 67%|██████▋   | 33792/50176 [03:54<01:52, 145.47it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 149.63it/s]
 68%|██████▊   | 34304/50176 [03:57<01:44, 151.68it/s]
 69%|██████▉   | 34560/50176 [03:59<01:47, 145.12it/s]
 69%|██████▉   | 34816/50176 [04:01<01:45, 145.15it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.14it/s]
 70%|███████   | 35328/50176 [04:05<01:40, 148.03it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 146.23it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 140.95it/s]
 72%|███████▏  | 36096/50176 [04:10<01:37, 144.02it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.44it/s]
 73%|███████▎  | 36608/50176 [04:14<01:33, 144.54it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 142.61it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.70it/s]
 74%|███████▍  | 37376/50176 [04:19<01:30, 141.58it/s]
 75%|███████▌  | 37632/50176 [04:21<01:27, 144.07it/s]
 76%|███████▌  | 37888/50176 [04:23<01:25, 143.61it/s]
 76%|███████▌  | 38144/50176 [04:24<01:22, 145.20it/s]
 77%|███████▋  | 38400/50176 [04:26<01:20, 145.54it/s]
 77%|███████▋  | 38656/50176 [04:28<01:19, 144.22it/s]
 78%|███████▊  | 38912/50176 [04:30<01:18, 142.77it/s]
 78%|███████▊  | 39168/50176 [04:32<01:19, 139.02it/s]
 79%|███████▊  | 39424/50176 [04:33<01:16, 140.55it/s]
 79%|███████▉  | 39680/50176 [04:35<01:13, 143.39it/s]
 80%|███████▉  | 39936/50176 [04:37<01:10, 145.34it/s]
 80%|████████  | 40192/50176 [04:39<01:07, 147.22it/s]
 81%|████████  | 40448/50176 [04:41<01:08, 140.99it/s]
 81%|████████  | 40704/50176 [04:43<01:10, 135.13it/s]
 82%|████████▏ | 40960/50176 [04:44<01:05, 140.47it/s]
 82%|████████▏ | 41216/50176 [04:46<01:02, 144.20it/s]
 83%|████████▎ | 41472/50176 [04:48<00:59, 146.67it/s]
 83%|████████▎ | 41728/50176 [04:49<00:56, 148.58it/s]
 84%|████████▎ | 41984/50176 [04:51<00:54, 149.34it/s]
 84%|████████▍ | 42240/50176 [04:53<00:54, 146.14it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 148.18it/s]
 85%|████████▌ | 42752/50176 [04:56<00:50, 147.91it/s]
 86%|████████▌ | 43008/50176 [04:58<00:47, 149.50it/s]
 86%|████████▌ | 43264/50176 [05:00<00:46, 149.17it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.11it/s]
 87%|████████▋ | 43776/50176 [05:06<01:03, 101.54it/s]
 88%|████████▊ | 44032/50176 [05:07<00:54, 113.63it/s]
 88%|████████▊ | 44288/50176 [05:09<00:47, 124.51it/s]
 89%|████████▉ | 44544/50176 [05:11<00:43, 129.94it/s]
 89%|████████▉ | 44800/50176 [05:13<00:40, 134.12it/s]
 90%|████████▉ | 45056/50176 [05:14<00:36, 140.30it/s]
 90%|█████████ | 45312/50176 [05:17<00:41, 117.24it/s]
 91%|█████████ | 45568/50176 [05:19<00:35, 128.11it/s]
 91%|█████████▏| 45824/50176 [05:20<00:32, 135.59it/s]
 92%|█████████▏| 46080/50176 [05:22<00:29, 140.58it/s]
 92%|█████████▏| 46336/50176 [05:24<00:26, 144.96it/s]
 93%|█████████▎| 46592/50176 [05:25<00:24, 147.88it/s]
 93%|█████████▎| 46848/50176 [05:27<00:22, 148.58it/s]
 94%|█████████▍| 47104/50176 [05:29<00:21, 145.11it/s]
 94%|█████████▍| 47360/50176 [05:31<00:19, 146.39it/s]
 95%|█████████▍| 47616/50176 [05:32<00:17, 145.33it/s]
 95%|█████████▌| 47872/50176 [05:34<00:15, 148.21it/s]
 96%|█████████▌| 48128/50176 [05:36<00:13, 150.23it/s]
 96%|█████████▋| 48384/50176 [05:37<00:11, 152.00it/s]
 97%|█████████▋| 48640/50176 [05:39<00:09, 154.64it/s]
 97%|█████████▋| 48896/50176 [05:41<00:08, 154.76it/s]
 98%|█████████▊| 49152/50176 [05:42<00:06, 150.73it/s]
 98%|█████████▊| 49408/50176 [05:44<00:05, 149.25it/s]
 99%|█████████▉| 49664/50176 [05:46<00:03, 149.88it/s]
 99%|█████████▉| 49920/50176 [05:48<00:01, 142.13it/s]
-2022-12-21,10:00:52 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:00:52 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.2664	imagenet-zeroshot-val-top5: 0.5124

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_20.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt).
-2022-12-21,10:01:05 | INFO | Model:
-2022-12-21,10:01:05 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:01:05 | INFO | Params:
-2022-12-21,10:01:05 | INFO |   batch_size: 256
-2022-12-21,10:01:05 | INFO |   beta1: 0.9
-2022-12-21,10:01:05 | INFO |   beta2: 0.98
-2022-12-21,10:01:05 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/checkpoints
-2022-12-21,10:01:05 | INFO |   copy_codebase: False
-2022-12-21,10:01:05 | INFO |   csv_caption_key: title
-2022-12-21,10:01:05 | INFO |   csv_img_key: filepath
-2022-12-21,10:01:05 | INFO |   csv_separator: 	
-2022-12-21,10:01:05 | INFO |   dataset_resampled: False
-2022-12-21,10:01:05 | INFO |   dataset_type: auto
-2022-12-21,10:01:05 | INFO |   ddp_static_graph: False
-2022-12-21,10:01:05 | INFO |   debug: False
-2022-12-21,10:01:05 | INFO |   device: cuda:0
-2022-12-21,10:01:05 | INFO |   dist_backend: nccl
-2022-12-21,10:01:05 | INFO |   dist_url: env://
-2022-12-21,10:01:05 | INFO |   distributed: False
-2022-12-21,10:01:05 | INFO |   epochs: 32
-2022-12-21,10:01:05 | INFO |   eps: 1e-06
-2022-12-21,10:01:05 | INFO |   force_quick_gelu: False
-2022-12-21,10:01:05 | INFO |   gather_with_grad: False
-2022-12-21,10:01:05 | INFO |   grad_checkpointing: False
-2022-12-21,10:01:05 | INFO |   horovod: False
-2022-12-21,10:01:05 | INFO |   image_mean: None
-2022-12-21,10:01:05 | INFO |   image_std: None
-2022-12-21,10:01:05 | INFO |   imagenet_v2: None
-2022-12-21,10:01:05 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:01:05 | INFO |   local_loss: False
-2022-12-21,10:01:05 | INFO |   local_rank: 0
-2022-12-21,10:01:05 | INFO |   lock_image: False
-2022-12-21,10:01:05 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:01:05 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:01:05 | INFO |   log_level: 20
-2022-12-21,10:01:05 | INFO |   log_local: False
-2022-12-21,10:01:05 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_20.pt/out.log
-2022-12-21,10:01:05 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:01:05 | INFO |   lr: 0.0005
-2022-12-21,10:01:05 | INFO |   model: ViT-B-16
-2022-12-21,10:01:05 | INFO |   name: epoch_20.pt
-2022-12-21,10:01:05 | INFO |   no_set_device_rank: False
-2022-12-21,10:01:05 | INFO |   norm_gradient_clip: None
-2022-12-21,10:01:05 | INFO |   precision: amp_bfloat16
-2022-12-21,10:01:05 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_20.pt
-2022-12-21,10:01:05 | INFO |   pretrained_image: False
-2022-12-21,10:01:05 | INFO |   rank: 0
-2022-12-21,10:01:05 | INFO |   report_to: 
-2022-12-21,10:01:05 | INFO |   resume: None
-2022-12-21,10:01:05 | INFO |   save_frequency: 1
-2022-12-21,10:01:05 | INFO |   save_most_recent: False
-2022-12-21,10:01:05 | INFO |   seed: 0
-2022-12-21,10:01:05 | INFO |   skip_scheduler: False
-2022-12-21,10:01:05 | INFO |   tensorboard: False
-2022-12-21,10:01:05 | INFO |   tensorboard_path: 
-2022-12-21,10:01:05 | INFO |   torchscript: False
-2022-12-21,10:01:05 | INFO |   trace: False
-2022-12-21,10:01:05 | INFO |   train_data: None
-2022-12-21,10:01:05 | INFO |   train_num_samples: None
-2022-12-21,10:01:05 | INFO |   use_bn_sync: False
-2022-12-21,10:01:05 | INFO |   val_data: None
-2022-12-21,10:01:05 | INFO |   val_frequency: 1
-2022-12-21,10:01:05 | INFO |   val_num_samples: None
-2022-12-21,10:01:05 | INFO |   wandb: False
-2022-12-21,10:01:05 | INFO |   wandb_notes: 
-2022-12-21,10:01:05 | INFO |   warmup: 10000
-2022-12-21,10:01:05 | INFO |   wd: 0.2
-2022-12-21,10:01:05 | INFO |   workers: 1
-2022-12-21,10:01:05 | INFO |   world_size: 1
-2022-12-21,10:01:05 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:01:05 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:01:05 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<15:32,  1.07it/s]
  1%|          | 6/1000 [00:01<02:12,  7.52it/s]
  1%|          | 11/1000 [00:01<01:09, 14.19it/s]
  2%|▏         | 17/1000 [00:01<00:44, 22.14it/s]
  2%|▏         | 23/1000 [00:01<00:33, 29.27it/s]
  3%|▎         | 29/1000 [00:01<00:27, 35.39it/s]
  4%|▎         | 35/1000 [00:01<00:24, 39.83it/s]
  4%|▍         | 41/1000 [00:01<00:21, 44.30it/s]
  5%|▍         | 47/1000 [00:01<00:19, 47.75it/s]
  5%|▌         | 53/1000 [00:01<00:18, 49.91it/s]
  6%|▌         | 59/1000 [00:02<00:18, 51.28it/s]
  6%|▋         | 65/1000 [00:02<00:17, 52.20it/s]
  7%|▋         | 71/1000 [00:02<00:17, 52.85it/s]
  8%|▊         | 77/1000 [00:02<00:17, 53.42it/s]
  8%|▊         | 83/1000 [00:02<00:17, 53.89it/s]
  9%|▉         | 89/1000 [00:02<00:16, 55.18it/s]
 10%|▉         | 95/1000 [00:02<00:16, 56.05it/s]
 10%|█         | 101/1000 [00:02<00:15, 56.66it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.23it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.55it/s]
 12%|█▏        | 119/1000 [00:03<00:15, 57.68it/s]
 12%|█▎        | 125/1000 [00:03<00:15, 57.75it/s]
 13%|█▎        | 131/1000 [00:03<00:15, 57.85it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.83it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.81it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.87it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.96it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.88it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.54it/s]
 17%|█▋        | 173/1000 [00:04<00:14, 57.60it/s]
 18%|█▊        | 179/1000 [00:04<00:14, 57.66it/s]
 18%|█▊        | 185/1000 [00:04<00:14, 57.56it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.59it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.56it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.46it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.39it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.41it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.59it/s]
 23%|██▎       | 233/1000 [00:05<00:13, 57.65it/s]
 24%|██▍       | 239/1000 [00:05<00:13, 57.50it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.53it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.61it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.79it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.77it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.54it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.77it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.87it/s]
 29%|██▉       | 293/1000 [00:06<00:12, 58.10it/s]
 30%|██▉       | 299/1000 [00:06<00:12, 58.07it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.08it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.18it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.16it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.12it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.99it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.03it/s]
 35%|███▍      | 347/1000 [00:07<00:11, 58.15it/s]
 35%|███▌      | 353/1000 [00:07<00:11, 57.87it/s]
 36%|███▌      | 359/1000 [00:07<00:11, 58.00it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 57.95it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.10it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.94it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.74it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.71it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.94it/s]
 41%|████      | 407/1000 [00:08<00:10, 58.04it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 58.08it/s]
 42%|████▏     | 419/1000 [00:08<00:09, 58.12it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.06it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.20it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.20it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.00it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.02it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.04it/s]
 47%|████▋     | 467/1000 [00:09<00:09, 58.02it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 58.16it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.06it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.04it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.99it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.00it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.90it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.96it/s]
 52%|█████▏    | 521/1000 [00:10<00:08, 58.00it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 58.11it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 58.02it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.12it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.05it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.03it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.11it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.01it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.93it/s]
 58%|█████▊    | 581/1000 [00:11<00:07, 58.08it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 58.00it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.95it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.07it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.99it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.05it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.10it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.14it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.99it/s]
 64%|██████▍   | 641/1000 [00:12<00:06, 57.87it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 58.09it/s]
 65%|██████▌   | 653/1000 [00:12<00:05, 58.03it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.11it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.12it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.13it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.15it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.19it/s]
 70%|██████▉   | 695/1000 [00:13<00:05, 58.06it/s]
 70%|███████   | 701/1000 [00:13<00:05, 58.13it/s]
 71%|███████   | 707/1000 [00:13<00:05, 58.14it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.13it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.14it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.09it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.06it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.09it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.04it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.12it/s]
 76%|███████▌  | 755/1000 [00:14<00:04, 58.25it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 58.06it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 58.12it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.10it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.15it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.18it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.98it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.98it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.01it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.00it/s]
 82%|████████▏ | 815/1000 [00:15<00:03, 57.97it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 58.03it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 58.04it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.22it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.29it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.15it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.20it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.12it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.15it/s]
 87%|████████▋ | 869/1000 [00:16<00:02, 58.03it/s]
 88%|████████▊ | 875/1000 [00:16<00:02, 58.04it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 58.09it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.96it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.06it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.11it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.09it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.06it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.95it/s]
 93%|█████████▎| 929/1000 [00:17<00:01, 58.08it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 58.17it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 58.16it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.17it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.20it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.16it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.21it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.28it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.30it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.32it/s]
 99%|█████████▉| 989/1000 [00:18<00:00, 58.25it/s]
-2022-12-21,10:01:23 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:26, 111.74it/s]
  1%|          | 512/50176 [00:04<06:33, 126.35it/s]
  2%|▏         | 768/50176 [00:06<06:25, 128.24it/s]
  2%|▏         | 1024/50176 [00:07<06:08, 133.46it/s]
  3%|▎         | 1280/50176 [00:09<05:55, 137.49it/s]
  3%|▎         | 1536/50176 [00:11<06:03, 133.74it/s]
  4%|▎         | 1792/50176 [00:13<05:56, 135.68it/s]
  4%|▍         | 2048/50176 [00:15<05:47, 138.54it/s]
  5%|▍         | 2304/50176 [00:17<05:43, 139.31it/s]
  5%|▌         | 2560/50176 [00:18<05:37, 141.22it/s]
  6%|▌         | 2816/50176 [00:20<05:37, 140.52it/s]
  6%|▌         | 3072/50176 [00:22<05:28, 143.30it/s]
  7%|▋         | 3328/50176 [00:24<05:24, 144.37it/s]
  7%|▋         | 3584/50176 [00:25<05:17, 146.88it/s]
  8%|▊         | 3840/50176 [00:27<05:26, 141.86it/s]
  8%|▊         | 4096/50176 [00:29<05:28, 140.32it/s]
  9%|▊         | 4352/50176 [00:31<05:19, 143.37it/s]
  9%|▉         | 4608/50176 [00:33<05:15, 144.58it/s]
 10%|▉         | 4864/50176 [00:34<05:09, 146.58it/s]
 10%|█         | 5120/50176 [00:36<05:05, 147.56it/s]
 11%|█         | 5376/50176 [00:38<05:02, 148.10it/s]
 11%|█         | 5632/50176 [00:39<04:57, 149.73it/s]
 12%|█▏        | 5888/50176 [00:41<04:55, 149.80it/s]
 12%|█▏        | 6144/50176 [00:43<05:20, 137.49it/s]
 13%|█▎        | 6400/50176 [00:45<05:10, 140.93it/s]
 13%|█▎        | 6656/50176 [00:47<05:03, 143.29it/s]
 14%|█▍        | 6912/50176 [00:48<05:01, 143.68it/s]
 14%|█▍        | 7168/50176 [00:50<04:56, 145.03it/s]
 15%|█▍        | 7424/50176 [00:52<04:50, 147.39it/s]
 15%|█▌        | 7680/50176 [00:54<04:50, 146.04it/s]
 16%|█▌        | 7936/50176 [00:55<04:43, 148.98it/s]
 16%|█▋        | 8192/50176 [00:57<04:44, 147.66it/s]
 17%|█▋        | 8448/50176 [00:59<04:40, 148.67it/s]
 17%|█▋        | 8704/50176 [01:00<04:37, 149.35it/s]
 18%|█▊        | 8960/50176 [01:02<04:44, 144.75it/s]
 18%|█▊        | 9216/50176 [01:04<04:37, 147.50it/s]
 19%|█▉        | 9472/50176 [01:06<04:36, 147.23it/s]
 19%|█▉        | 9728/50176 [01:08<04:38, 145.08it/s]
 20%|█▉        | 9984/50176 [01:09<04:32, 147.66it/s]
 20%|██        | 10240/50176 [01:11<04:26, 149.59it/s]
 21%|██        | 10496/50176 [01:13<04:24, 150.02it/s]
 21%|██▏       | 10752/50176 [01:14<04:30, 145.99it/s]
 22%|██▏       | 11008/50176 [01:16<04:24, 148.08it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.45it/s]
 23%|██▎       | 11520/50176 [01:19<04:15, 151.50it/s]
 23%|██▎       | 11776/50176 [01:21<04:14, 151.09it/s]
 24%|██▍       | 12032/50176 [01:23<04:12, 151.09it/s]
 24%|██▍       | 12288/50176 [01:24<04:08, 152.20it/s]
 25%|██▌       | 12544/50176 [01:26<04:06, 152.73it/s]
 26%|██▌       | 12800/50176 [01:28<04:04, 152.76it/s]
 26%|██▌       | 13056/50176 [01:29<04:02, 152.86it/s]
 27%|██▋       | 13312/50176 [01:31<04:06, 149.79it/s]
 27%|██▋       | 13568/50176 [01:33<04:14, 143.89it/s]
 28%|██▊       | 13824/50176 [01:35<04:19, 140.21it/s]
 28%|██▊       | 14080/50176 [01:37<04:15, 141.30it/s]
 29%|██▊       | 14336/50176 [01:39<04:19, 138.19it/s]
 29%|██▉       | 14592/50176 [01:41<04:08, 143.05it/s]
 30%|██▉       | 14848/50176 [01:43<04:20, 135.61it/s]
 30%|███       | 15104/50176 [01:45<04:18, 135.42it/s]
 31%|███       | 15360/50176 [01:46<04:12, 137.86it/s]
 31%|███       | 15616/50176 [01:48<04:15, 135.09it/s]
 32%|███▏      | 15872/50176 [01:50<04:15, 134.28it/s]
 32%|███▏      | 16128/50176 [01:52<04:07, 137.69it/s]
 33%|███▎      | 16384/50176 [01:54<04:04, 138.15it/s]
 33%|███▎      | 16640/50176 [01:56<04:09, 134.15it/s]
 34%|███▎      | 16896/50176 [01:58<03:58, 139.35it/s]
 34%|███▍      | 17152/50176 [01:59<03:50, 142.99it/s]
 35%|███▍      | 17408/50176 [02:01<03:46, 144.52it/s]
 35%|███▌      | 17664/50176 [02:03<03:41, 146.62it/s]
 36%|███▌      | 17920/50176 [02:04<03:37, 148.15it/s]
 36%|███▌      | 18176/50176 [02:06<03:38, 146.69it/s]
 37%|███▋      | 18432/50176 [02:08<03:34, 147.71it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.05it/s]
 38%|███▊      | 18944/50176 [02:12<03:40, 141.69it/s]
 38%|███▊      | 19200/50176 [02:14<03:50, 134.59it/s]
 39%|███▉      | 19456/50176 [02:16<03:52, 132.38it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.27it/s]
 40%|███▉      | 19968/50176 [02:19<03:43, 135.17it/s]
 40%|████      | 20224/50176 [02:21<03:32, 140.94it/s]
 41%|████      | 20480/50176 [02:23<03:26, 143.65it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 143.88it/s]
 42%|████▏     | 20992/50176 [02:26<03:19, 146.25it/s]
 42%|████▏     | 21248/50176 [02:28<03:23, 142.30it/s]
 43%|████▎     | 21504/50176 [02:30<03:16, 146.11it/s]
 43%|████▎     | 21760/50176 [02:31<03:12, 147.39it/s]
 44%|████▍     | 22016/50176 [02:33<03:07, 150.32it/s]
 44%|████▍     | 22272/50176 [02:35<03:06, 149.77it/s]
 45%|████▍     | 22528/50176 [02:36<03:03, 150.91it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.09it/s]
 46%|████▌     | 23040/50176 [02:40<03:09, 142.90it/s]
 46%|████▋     | 23296/50176 [02:42<03:12, 139.73it/s]
 47%|████▋     | 23552/50176 [02:44<03:15, 136.46it/s]
 47%|████▋     | 23808/50176 [02:46<03:07, 140.99it/s]
 48%|████▊     | 24064/50176 [02:48<03:02, 142.78it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.24it/s]
 49%|████▉     | 24576/50176 [02:51<02:56, 145.34it/s]
 49%|████▉     | 24832/50176 [02:53<02:50, 148.48it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 148.98it/s]
 51%|█████     | 25344/50176 [02:56<02:46, 149.54it/s]
 51%|█████     | 25600/50176 [02:58<02:48, 145.81it/s]
 52%|█████▏    | 25856/50176 [03:00<02:41, 150.42it/s]
 52%|█████▏    | 26112/50176 [03:01<02:43, 147.31it/s]
 53%|█████▎    | 26368/50176 [03:03<02:46, 143.35it/s]
 53%|█████▎    | 26624/50176 [03:05<02:42, 145.04it/s]
 54%|█████▎    | 26880/50176 [03:07<02:40, 144.91it/s]
 54%|█████▍    | 27136/50176 [03:08<02:38, 145.67it/s]
 55%|█████▍    | 27392/50176 [03:10<02:35, 146.37it/s]
 55%|█████▌    | 27648/50176 [03:12<02:28, 152.15it/s]
 56%|█████▌    | 27904/50176 [03:14<02:36, 142.06it/s]
 56%|█████▌    | 28160/50176 [03:16<02:33, 143.63it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.15it/s]
 57%|█████▋    | 28672/50176 [03:19<02:33, 140.26it/s]
 58%|█████▊    | 28928/50176 [03:21<02:28, 142.75it/s]
 58%|█████▊    | 29184/50176 [03:23<02:25, 144.73it/s]
 59%|█████▊    | 29440/50176 [03:25<02:32, 136.16it/s]
 59%|█████▉    | 29696/50176 [03:27<02:34, 132.86it/s]
 60%|█████▉    | 29952/50176 [03:29<02:32, 133.02it/s]
 60%|██████    | 30208/50176 [03:30<02:25, 137.46it/s]
 61%|██████    | 30464/50176 [03:32<02:17, 143.09it/s]
 61%|██████    | 30720/50176 [03:34<02:16, 142.73it/s]
 62%|██████▏   | 30976/50176 [03:36<02:11, 145.61it/s]
 62%|██████▏   | 31232/50176 [03:37<02:08, 148.00it/s]
 63%|██████▎   | 31488/50176 [03:39<02:05, 149.34it/s]
 63%|██████▎   | 31744/50176 [03:41<02:03, 149.49it/s]
 64%|██████▍   | 32000/50176 [03:42<02:02, 148.35it/s]
 64%|██████▍   | 32256/50176 [03:44<02:00, 148.30it/s]
 65%|██████▍   | 32512/50176 [03:46<02:02, 144.68it/s]
 65%|██████▌   | 32768/50176 [03:48<01:58, 147.23it/s]
 66%|██████▌   | 33024/50176 [03:49<01:57, 145.46it/s]
 66%|██████▋   | 33280/50176 [03:51<01:55, 146.85it/s]
 67%|██████▋   | 33536/50176 [03:53<01:56, 143.28it/s]
 67%|██████▋   | 33792/50176 [03:55<01:51, 146.33it/s]
 68%|██████▊   | 34048/50176 [03:56<01:47, 150.28it/s]
 68%|██████▊   | 34304/50176 [03:58<01:45, 150.97it/s]
 69%|██████▉   | 34560/50176 [04:00<01:47, 145.52it/s]
 69%|██████▉   | 34816/50176 [04:02<01:45, 145.78it/s]
 70%|██████▉   | 35072/50176 [04:03<01:44, 144.98it/s]
 70%|███████   | 35328/50176 [04:05<01:40, 147.73it/s]
 71%|███████   | 35584/50176 [04:07<01:40, 145.40it/s]
 71%|███████▏  | 35840/50176 [04:09<01:41, 140.95it/s]
 72%|███████▏  | 36096/50176 [04:11<01:38, 143.41it/s]
 72%|███████▏  | 36352/50176 [04:12<01:37, 141.55it/s]
 73%|███████▎  | 36608/50176 [04:14<01:33, 144.52it/s]
 73%|███████▎  | 36864/50176 [04:16<01:32, 143.19it/s]
 74%|███████▍  | 37120/50176 [04:18<01:30, 144.56it/s]
 74%|███████▍  | 37376/50176 [04:19<01:26, 148.77it/s]
 75%|███████▌  | 37632/50176 [04:21<01:23, 151.01it/s]
 76%|███████▌  | 37888/50176 [04:23<01:22, 148.45it/s]
 76%|███████▌  | 38144/50176 [04:24<01:20, 149.90it/s]
 77%|███████▋  | 38400/50176 [04:26<01:19, 148.40it/s]
 77%|███████▋  | 38656/50176 [04:28<01:18, 146.98it/s]
 78%|███████▊  | 38912/50176 [04:30<01:17, 145.40it/s]
 78%|███████▊  | 39168/50176 [04:32<01:17, 141.76it/s]
 79%|███████▊  | 39424/50176 [04:33<01:14, 143.67it/s]
 79%|███████▉  | 39680/50176 [04:35<01:12, 145.26it/s]
 80%|███████▉  | 39936/50176 [04:37<01:09, 147.77it/s]
 80%|████████  | 40192/50176 [04:38<01:06, 150.03it/s]
 81%|████████  | 40448/50176 [04:40<01:07, 143.25it/s]
 81%|████████  | 40704/50176 [04:42<01:08, 137.99it/s]
 82%|████████▏ | 40960/50176 [04:44<01:04, 143.47it/s]
 82%|████████▏ | 41216/50176 [04:46<01:01, 146.80it/s]
 83%|████████▎ | 41472/50176 [04:47<00:58, 148.91it/s]
 83%|████████▎ | 41728/50176 [04:49<00:55, 151.66it/s]
 84%|████████▎ | 41984/50176 [04:51<00:53, 151.82it/s]
 84%|██████���█▍ | 42240/50176 [04:52<00:53, 148.69it/s]
 85%|████████▍ | 42496/50176 [04:54<00:51, 149.79it/s]
 85%|████████▌ | 42752/50176 [04:56<00:49, 149.32it/s]
 86%|████████▌ | 43008/50176 [04:57<00:47, 150.81it/s]
 86%|████████▌ | 43264/50176 [04:59<00:45, 150.53it/s]
 87%|████████▋ | 43520/50176 [05:01<00:44, 148.75it/s]
 87%|████████▋ | 43776/50176 [05:03<00:43, 146.35it/s]
 88%|████████▊ | 44032/50176 [05:04<00:41, 147.98it/s]
 88%|████████▊ | 44288/50176 [05:06<00:39, 150.48it/s]
 89%|████████▉ | 44544/50176 [05:08<00:38, 147.77it/s]
 89%|████████▉ | 44800/50176 [05:10<00:36, 146.37it/s]
 90%|████████▉ | 45056/50176 [05:11<00:34, 148.45it/s]
 90%|█████████ | 45312/50176 [05:13<00:34, 141.80it/s]
 91%|█████████ | 45568/50176 [05:15<00:31, 146.60it/s]
 91%|█████████▏| 45824/50176 [05:17<00:29, 148.24it/s]
 92%|█████████▏| 46080/50176 [05:18<00:27, 148.54it/s]
 92%|█████████▏| 46336/50176 [05:20<00:25, 149.42it/s]
 93%|█████████▎| 46592/50176 [05:22<00:24, 148.74it/s]
 93%|█████████▎| 46848/50176 [05:24<00:22, 148.18it/s]
 94%|█████████▍| 47104/50176 [05:25<00:21, 143.71it/s]
 94%|█████████▍| 47360/50176 [05:27<00:19, 145.42it/s]
 95%|█████████▍| 47616/50176 [05:29<00:17, 143.66it/s]
 95%|█████████▌| 47872/50176 [05:31<00:15, 146.55it/s]
 96%|█████████▌| 48128/50176 [05:32<00:13, 146.99it/s]
 96%|█████████▋| 48384/50176 [05:34<00:11, 149.73it/s]
 97%|█████████▋| 48640/50176 [05:36<00:10, 151.90it/s]
 97%|█████████▋| 48896/50176 [05:37<00:08, 151.92it/s]
 98%|█████████▊| 49152/50176 [05:39<00:06, 147.66it/s]
 98%|█████████▊| 49408/50176 [05:41<00:05, 145.41it/s]
 99%|█████████▉| 49664/50176 [05:43<00:03, 145.51it/s]
 99%|█████████▉| 49920/50176 [05:45<00:01, 138.63it/s]
-2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5645	imagenet-zeroshot-val-top5: 0.8353

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_21.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt).
-2022-12-21,10:01:07 | INFO | Model:
-2022-12-21,10:01:07 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:01:07 | INFO | Params:
-2022-12-21,10:01:07 | INFO |   batch_size: 256
-2022-12-21,10:01:07 | INFO |   beta1: 0.9
-2022-12-21,10:01:07 | INFO |   beta2: 0.98
-2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/checkpoints
-2022-12-21,10:01:07 | INFO |   copy_codebase: False
-2022-12-21,10:01:07 | INFO |   csv_caption_key: title
-2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
-2022-12-21,10:01:07 | INFO |   csv_separator: 	
-2022-12-21,10:01:07 | INFO |   dataset_resampled: False
-2022-12-21,10:01:07 | INFO |   dataset_type: auto
-2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
-2022-12-21,10:01:07 | INFO |   debug: False
-2022-12-21,10:01:07 | INFO |   device: cuda:0
-2022-12-21,10:01:07 | INFO |   dist_backend: nccl
-2022-12-21,10:01:07 | INFO |   dist_url: env://
-2022-12-21,10:01:07 | INFO |   distributed: False
-2022-12-21,10:01:07 | INFO |   epochs: 32
-2022-12-21,10:01:07 | INFO |   eps: 1e-06
-2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
-2022-12-21,10:01:07 | INFO |   gather_with_grad: False
-2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
-2022-12-21,10:01:07 | INFO |   horovod: False
-2022-12-21,10:01:07 | INFO |   image_mean: None
-2022-12-21,10:01:07 | INFO |   image_std: None
-2022-12-21,10:01:07 | INFO |   imagenet_v2: None
-2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:01:07 | INFO |   local_loss: False
-2022-12-21,10:01:07 | INFO |   local_rank: 0
-2022-12-21,10:01:07 | INFO |   lock_image: False
-2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:01:07 | INFO |   log_level: 20
-2022-12-21,10:01:07 | INFO |   log_local: False
-2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_21.pt/out.log
-2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:01:07 | INFO |   lr: 0.0005
-2022-12-21,10:01:07 | INFO |   model: ViT-B-16
-2022-12-21,10:01:07 | INFO |   name: epoch_21.pt
-2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
-2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
-2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
-2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_21.pt
-2022-12-21,10:01:07 | INFO |   pretrained_image: False
-2022-12-21,10:01:07 | INFO |   rank: 0
-2022-12-21,10:01:07 | INFO |   report_to: 
-2022-12-21,10:01:07 | INFO |   resume: None
-2022-12-21,10:01:07 | INFO |   save_frequency: 1
-2022-12-21,10:01:07 | INFO |   save_most_recent: False
-2022-12-21,10:01:07 | INFO |   seed: 0
-2022-12-21,10:01:07 | INFO |   skip_scheduler: False
-2022-12-21,10:01:07 | INFO |   tensorboard: False
-2022-12-21,10:01:07 | INFO |   tensorboard_path: 
-2022-12-21,10:01:07 | INFO |   torchscript: False
-2022-12-21,10:01:07 | INFO |   trace: False
-2022-12-21,10:01:07 | INFO |   train_data: None
-2022-12-21,10:01:07 | INFO |   train_num_samples: None
-2022-12-21,10:01:07 | INFO |   use_bn_sync: False
-2022-12-21,10:01:07 | INFO |   val_data: None
-2022-12-21,10:01:07 | INFO |   val_frequency: 1
-2022-12-21,10:01:07 | INFO |   val_num_samples: None
-2022-12-21,10:01:07 | INFO |   wandb: False
-2022-12-21,10:01:07 | INFO |   wandb_notes: 
-2022-12-21,10:01:07 | INFO |   warmup: 10000
-2022-12-21,10:01:07 | INFO |   wd: 0.2
-2022-12-21,10:01:07 | INFO |   workers: 1
-2022-12-21,10:01:07 | INFO |   world_size: 1
-2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:01:07 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:56,  1.52it/s]
  1%|          | 7/1000 [00:00<01:25, 11.56it/s]
  1%|▏         | 13/1000 [00:00<00:48, 20.38it/s]
  2%|▏         | 19/1000 [00:00<00:34, 28.69it/s]
  2%|▎         | 25/1000 [00:01<00:27, 35.66it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.36it/s]
  4%|▎         | 37/1000 [00:01<00:21, 45.75it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.05it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.46it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.17it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.41it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.36it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.05it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.64it/s]
  8%|▊         | 85/1000 [00:02<00:17, 51.56it/s]
  9%|▉         | 91/1000 [00:02<00:17, 53.35it/s]
 10%|▉         | 97/1000 [00:02<00:16, 54.85it/s]
 10%|█         | 103/1000 [00:02<00:16, 55.84it/s]
 11%|█         | 109/1000 [00:02<00:15, 56.44it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 57.08it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.36it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 57.58it/s]
 13%|█▎        | 133/1000 [00:03<00:15, 57.78it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.88it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.00it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.09it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.06it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.11it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.91it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.94it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.88it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.89it/s]
 19%|█▉        | 193/1000 [00:04<00:13, 57.88it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.81it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.64it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.65it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.65it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.82it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.79it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.61it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.78it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.65it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.83it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.85it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.72it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.86it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 58.03it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.17it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.34it/s]
 30%|███       | 301/1000 [00:05<00:11, 58.30it/s]
 31%|███       | 307/1000 [00:06<00:11, 58.24it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.43it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.45it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.26it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.11it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.25it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.31it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.27it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 58.19it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.24it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.29it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.37it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.13it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.93it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 58.00it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.07it/s]
 40%|████      | 403/1000 [00:07<00:10, 58.19it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.26it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.32it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.33it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.34it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.47it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.44it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.16it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.27it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.29it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.34it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.26it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.41it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.22it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.33it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.27it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.19it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.25it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.29it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.26it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.28it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.34it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.43it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.44it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.32it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.37it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.16it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.22it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.31it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.32it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.23it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.35it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.22it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.27it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.40it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.34it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.34it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.25it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.23it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.32it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.32it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.36it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.34it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.39it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.39it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.31it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.36it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.38it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 58.35it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.17it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.17it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.24it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.27it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.35it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.39it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.25it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.33it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.32it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.34it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.36it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.21it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.15it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.21it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.22it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.33it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.48it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.50it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.37it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.38it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.37it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.33it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.37it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.32it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.25it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.36it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.20it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 57.97it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.19it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.32it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.37it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.32it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.32it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.46it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.34it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.44it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.55it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.56it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.54it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.42it/s]
-2022-12-21,10:01:25 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:08, 135.42it/s]
  1%|          | 512/50176 [00:03<05:15, 157.34it/s]
  2%|▏         | 768/50176 [00:04<04:50, 170.14it/s]
  2%|▏         | 1024/50176 [00:06<04:50, 169.46it/s]
  3%|▎         | 1280/50176 [00:08<05:08, 158.58it/s]
  3%|▎         | 1536/50176 [00:10<05:34, 145.37it/s]
  4%|▎         | 1792/50176 [00:11<05:36, 143.96it/s]
  4%|▍         | 2048/50176 [00:13<05:33, 144.35it/s]
  5%|▍         | 2304/50176 [00:15<05:33, 143.46it/s]
  5%|▌         | 2560/50176 [00:17<05:30, 143.86it/s]
  6%|▌         | 2816/50176 [00:19<05:32, 142.64it/s]
  6%|▌         | 3072/50176 [00:20<05:25, 144.65it/s]
  7%|▋         | 3328/50176 [00:22<05:22, 145.26it/s]
  7%|▋         | 3584/50176 [00:24<05:15, 147.59it/s]
  8%|▊         | 3840/50176 [00:26<05:27, 141.65it/s]
  8%|▊         | 4096/50176 [00:27<05:25, 141.78it/s]
  9%|▊         | 4352/50176 [00:29<05:19, 143.35it/s]
  9%|▉         | 4608/50176 [00:31<05:16, 144.07it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.65it/s]
 10%|█         | 5120/50176 [00:34<05:06, 147.08it/s]
 11%|█         | 5376/50176 [00:36<05:01, 148.83it/s]
 11%|█         | 5632/50176 [00:38<04:57, 149.76it/s]
 12%|█▏        | 5888/50176 [00:39<04:55, 149.82it/s]
 12%|█▏        | 6144/50176 [00:42<05:19, 137.67it/s]
 13%|█▎        | 6400/50176 [00:43<05:09, 141.22it/s]
 13%|█▎        | 6656/50176 [00:45<05:03, 143.27it/s]
 14%|█▍        | 6912/50176 [00:47<05:01, 143.58it/s]
 14%|█▍        | 7168/50176 [00:49<04:56, 145.03it/s]
 15%|█▍        | 7424/50176 [00:50<04:51, 146.60it/s]
 15%|█▌        | 7680/50176 [00:52<04:50, 146.22it/s]
 16%|█▌        | 7936/50176 [00:54<04:43, 149.07it/s]
 16%|█▋        | 8192/50176 [00:55<04:44, 147.80it/s]
 17%|█▋        | 8448/50176 [00:57<04:40, 148.69it/s]
 17%|█▋        | 8704/50176 [00:59<04:37, 149.48it/s]
 18%|█▊        | 8960/50176 [01:01<04:46, 144.07it/s]
 18%|█▊        | 9216/50176 [01:02<04:37, 147.71it/s]
 19%|█▉        | 9472/50176 [01:04<04:35, 147.98it/s]
 19%|█▉        | 9728/50176 [01:06<04:40, 144.30it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 147.78it/s]
 20%|██        | 10240/50176 [01:09<04:26, 149.71it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.10it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.41it/s]
 22%|██▏       | 11008/50176 [01:14<04:24, 148.31it/s]
 22%|██▏       | 11264/50176 [01:16<04:21, 148.75it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.85it/s]
 23%|██▎       | 11776/50176 [01:20<04:13, 151.37it/s]
 24%|██▍       | 12032/50176 [01:21<04:11, 151.88it/s]
 24%|██▍       | 12288/50176 [01:23<04:09, 152.16it/s]
 25%|██▌       | 12544/50176 [01:25<04:06, 152.67it/s]
 26%|██▌       | 12800/50176 [01:26<04:04, 152.79it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.83it/s]
 27%|██▋       | 13312/50176 [01:30<04:06, 149.77it/s]
 27%|██▋       | 13568/50176 [01:32<04:15, 143.31it/s]
 28%|██▊       | 13824/50176 [01:34<04:17, 141.08it/s]
 28%|██▊       | 14080/50176 [01:35<04:15, 141.24it/s]
 29%|██▊       | 14336/50176 [01:37<04:20, 137.63it/s]
 29%|██▉       | 14592/50176 [01:39<04:08, 143.17it/s]
 30%|██▉       | 14848/50176 [01:41<04:20, 135.80it/s]
 30%|███       | 15104/50176 [01:43<04:19, 134.89it/s]
 31%|███       | 15360/50176 [01:45<04:11, 138.52it/s]
 31%|███       | 15616/50176 [01:47<04:14, 135.58it/s]
 32%|███▏      | 15872/50176 [01:49<04:15, 134.05it/s]
 32%|███▏      | 16128/50176 [01:50<04:07, 137.45it/s]
 33%|███▎      | 16384/50176 [01:52<04:04, 138.24it/s]
 33%|███▎      | 16640/50176 [01:54<04:10, 134.07it/s]
 34%|███▎      | 16896/50176 [01:56<03:58, 139.42it/s]
 34%|███▍      | 17152/50176 [01:58<03:51, 142.89it/s]
 35%|███▍      | 17408/50176 [01:59<03:46, 144.47it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.60it/s]
 36%|███▌      | 17920/50176 [02:03<03:37, 148.17it/s]
 36%|███▌      | 18176/50176 [02:04<03:37, 147.15it/s]
 37%|███▋      | 18432/50176 [02:06<03:35, 147.62it/s]
 37%|███▋      | 18688/50176 [02:08<03:34, 146.88it/s]
 38%|███▊      | 18944/50176 [02:10<03:40, 141.95it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.48it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.21it/s]
 39%|███▉      | 19712/50176 [02:16<03:50, 132.07it/s]
 40%|███▉      | 19968/50176 [02:18<03:43, 135.21it/s]
 40%|████      | 20224/50176 [02:19<03:32, 140.93it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.76it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.78it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.51it/s]
 42%|████▏     | 21248/50176 [02:27<03:24, 141.56it/s]
 43%|████▎     | 21504/50176 [02:28<03:15, 146.30it/s]
 43%|████▎     | 21760/50176 [02:30<03:12, 147.47it/s]
 44%|████▍     | 22016/50176 [02:31<03:07, 150.00it/s]
 44%|████▍     | 22272/50176 [02:33<03:06, 149.72it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.17it/s]
 45%|████▌     | 22784/50176 [02:37<03:09, 144.52it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.35it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.61it/s]
 47%|████▋     | 23552/50176 [02:42<03:14, 137.22it/s]
 47%|████▋     | 23808/50176 [02:44<03:07, 140.94it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 142.73it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.22it/s]
 49%|████▉     | 24576/50176 [02:49<02:56, 145.45it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.49it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.97it/s]
 51%|█████     | 25344/50176 [02:54<02:46, 149.46it/s]
 51%|█████     | 25600/50176 [02:56<02:48, 145.81it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.33it/s]
 52%|█████▏    | 26112/50176 [03:00<02:43, 147.15it/s]
 53%|█████▎    | 26368/50176 [03:02<02:46, 143.33it/s]
 53%|█████▎    | 26624/50176 [03:03<02:42, 145.16it/s]
 54%|█████▎    | 26880/50176 [03:05<02:41, 144.37it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 145.90it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.46it/s]
 55%|█████▌    | 27648/50176 [03:10<02:28, 151.54it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 142.36it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.89it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.22it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.83it/s]
 58%|█████▊    | 28928/50176 [03:19<02:29, 142.53it/s]
 58%|█████▊    | 29184/50176 [03:21<02:24, 144.99it/s]
 59%|█████▊    | 29440/50176 [03:23<02:32, 135.88it/s]
 59%|█████▉    | 29696/50176 [03:25<02:34, 132.72it/s]
 60%|█████▉    | 29952/50176 [03:27<02:31, 133.52it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.39it/s]
 61%|██████    | 30464/50176 [03:30<02:17, 143.31it/s]
 61%|██████    | 30720/50176 [03:32<02:16, 142.68it/s]
 62%|██████▏   | 30976/50176 [03:34<02:12, 145.31it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.91it/s]
 63%|██████▎   | 31488/50176 [03:37<02:05, 149.25it/s]
 63%|██████▎   | 31744/50176 [03:39<02:03, 149.49it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.29it/s]
 64%|██████▍   | 32256/50176 [03:42<02:00, 149.11it/s]
 65%|██████▍   | 32512/50176 [03:44<02:02, 143.71it/s]
 65%|██████▌   | 32768/50176 [03:46<01:58, 147.41it/s]
 66%|██████▌   | 33024/50176 [03:48<01:58, 145.35it/s]
 66%|██████▋   | 33280/50176 [03:50<01:54, 146.94it/s]
 67%|██████▋   | 33536/50176 [03:51<01:56, 142.62it/s]
 67%|██████▋   | 33792/50176 [03:53<01:51, 146.67it/s]
 68%|██████▊   | 34048/50176 [03:55<01:47, 150.30it/s]
 68%|██████▊   | 34304/50176 [03:56<01:44, 151.84it/s]
 69%|██████▉   | 34560/50176 [03:58<01:47, 144.68it/s]
 69%|██████▉   | 34816/50176 [04:00<01:45, 145.92it/s]
 70%|██████▉   | 35072/50176 [04:02<01:44, 144.94it/s]
 70%|███████   | 35328/50176 [04:03<01:40, 147.87it/s]
 71%|███████   | 35584/50176 [04:05<01:40, 144.73it/s]
 71%|███████▏  | 35840/50176 [04:07<01:41, 141.61it/s]
 72%|███████▏  | 36096/50176 [04:09<01:38, 143.27it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 141.67it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 144.49it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.36it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 144.50it/s]
 74%|███████▍  | 37376/50176 [04:18<01:25, 148.86it/s]
 75%|███████▌  | 37632/50176 [04:19<01:23, 151.04it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 148.89it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.65it/s]
 77%|███████▋  | 38400/50176 [04:24<01:19, 148.43it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.22it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.30it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.81it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.53it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.13it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.88it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.89it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.79it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 137.73it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.35it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.64it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.89it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.46it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 151.77it/s]
 84%|███��████▍ | 42240/50176 [04:51<00:53, 148.64it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.85it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.25it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.91it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.36it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.52it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 145.94it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 148.03it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.67it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 148.03it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 146.47it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 148.54it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 141.90it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.81it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 148.14it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.63it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.29it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.42it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 147.51it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.42it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.39it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.81it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 146.56it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 147.24it/s]
 96%|█████████▋| 48384/50176 [05:32<00:11, 149.68it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 151.98it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 151.73it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.72it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.30it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.63it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.84it/s]
-2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5724	imagenet-zeroshot-val-top5: 0.8406

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_22.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:00:58 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:00:58 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt).
-2022-12-21,10:01:07 | INFO | Model:
-2022-12-21,10:01:07 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:01:07 | INFO | Params:
-2022-12-21,10:01:07 | INFO |   batch_size: 256
-2022-12-21,10:01:07 | INFO |   beta1: 0.9
-2022-12-21,10:01:07 | INFO |   beta2: 0.98
-2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/checkpoints
-2022-12-21,10:01:07 | INFO |   copy_codebase: False
-2022-12-21,10:01:07 | INFO |   csv_caption_key: title
-2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
-2022-12-21,10:01:07 | INFO |   csv_separator: 	
-2022-12-21,10:01:07 | INFO |   dataset_resampled: False
-2022-12-21,10:01:07 | INFO |   dataset_type: auto
-2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
-2022-12-21,10:01:07 | INFO |   debug: False
-2022-12-21,10:01:07 | INFO |   device: cuda:0
-2022-12-21,10:01:07 | INFO |   dist_backend: nccl
-2022-12-21,10:01:07 | INFO |   dist_url: env://
-2022-12-21,10:01:07 | INFO |   distributed: False
-2022-12-21,10:01:07 | INFO |   epochs: 32
-2022-12-21,10:01:07 | INFO |   eps: 1e-06
-2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
-2022-12-21,10:01:07 | INFO |   gather_with_grad: False
-2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
-2022-12-21,10:01:07 | INFO |   horovod: False
-2022-12-21,10:01:07 | INFO |   image_mean: None
-2022-12-21,10:01:07 | INFO |   image_std: None
-2022-12-21,10:01:07 | INFO |   imagenet_v2: None
-2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:01:07 | INFO |   local_loss: False
-2022-12-21,10:01:07 | INFO |   local_rank: 0
-2022-12-21,10:01:07 | INFO |   lock_image: False
-2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:01:07 | INFO |   log_level: 20
-2022-12-21,10:01:07 | INFO |   log_local: False
-2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_22.pt/out.log
-2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:01:07 | INFO |   lr: 0.0005
-2022-12-21,10:01:07 | INFO |   model: ViT-B-16
-2022-12-21,10:01:07 | INFO |   name: epoch_22.pt
-2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
-2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
-2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
-2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_22.pt
-2022-12-21,10:01:07 | INFO |   pretrained_image: False
-2022-12-21,10:01:07 | INFO |   rank: 0
-2022-12-21,10:01:07 | INFO |   report_to: 
-2022-12-21,10:01:07 | INFO |   resume: None
-2022-12-21,10:01:07 | INFO |   save_frequency: 1
-2022-12-21,10:01:07 | INFO |   save_most_recent: False
-2022-12-21,10:01:07 | INFO |   seed: 0
-2022-12-21,10:01:07 | INFO |   skip_scheduler: False
-2022-12-21,10:01:07 | INFO |   tensorboard: False
-2022-12-21,10:01:07 | INFO |   tensorboard_path: 
-2022-12-21,10:01:07 | INFO |   torchscript: False
-2022-12-21,10:01:07 | INFO |   trace: False
-2022-12-21,10:01:07 | INFO |   train_data: None
-2022-12-21,10:01:07 | INFO |   train_num_samples: None
-2022-12-21,10:01:07 | INFO |   use_bn_sync: False
-2022-12-21,10:01:07 | INFO |   val_data: None
-2022-12-21,10:01:07 | INFO |   val_frequency: 1
-2022-12-21,10:01:07 | INFO |   val_num_samples: None
-2022-12-21,10:01:07 | INFO |   wandb: False
-2022-12-21,10:01:07 | INFO |   wandb_notes: 
-2022-12-21,10:01:07 | INFO |   warmup: 10000
-2022-12-21,10:01:07 | INFO |   wd: 0.2
-2022-12-21,10:01:07 | INFO |   workers: 1
-2022-12-21,10:01:07 | INFO |   world_size: 1
-2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:01:07 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:57,  1.39it/s]
  1%|          | 6/1000 [00:00<01:45,  9.44it/s]
  1%|          | 12/1000 [00:00<00:53, 18.47it/s]
  2%|▏         | 18/1000 [00:01<00:36, 26.91it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.15it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.05it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.66it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.07it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.73it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.62it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.04it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.05it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.78it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.29it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.86it/s]
  9%|▉         | 90/1000 [00:02<00:17, 52.46it/s]
 10%|▉         | 96/1000 [00:02<00:16, 54.13it/s]
 10%|█         | 102/1000 [00:02<00:16, 55.20it/s]
 11%|█         | 108/1000 [00:02<00:15, 56.18it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 56.76it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.11it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.33it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.56it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.66it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.79it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.92it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.01it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.90it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.79it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.80it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.76it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.68it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.66it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.42it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.40it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.62it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.60it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.49it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.56it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.65it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.76it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.78it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.66it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.86it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.91it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.11it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.10it/s]
 31%|███       | 306/1000 [00:06<00:11, 58.09it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.22it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.13it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.09it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.18it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.96it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.02it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.11it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.66it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.74it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.85it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.00it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.09it/s]
 42%|████▏     | 420/1000 [00:08<00:09, 58.11it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.26it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.14it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.08it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.22it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.12it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.11it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.03it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.09it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.00it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.09it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.17it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.10it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.21it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.21it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.10it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.11it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.03it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.97it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.12it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 594/1000 [00:11<00:06, 58.02it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.13it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.05it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.17it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.26it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.08it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.01it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.15it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.10it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.18it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.19it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.21it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.10it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.18it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.13it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.11it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.12it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.09it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.11it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.13it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.11it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.20it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.06it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.93it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.09it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.06it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.11it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.18it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.99it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.99it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.05it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.02it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.03it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.13it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.27it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.29it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.20it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.21it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.21it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.14it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.20it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.18it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.05it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.16it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.11it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.94it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.97it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.99it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.12it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.19it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.17it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.21it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.24it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.19it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.32it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.34it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.33it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.28it/s]
-2022-12-21,10:01:25 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:04, 137.06it/s]
  1%|          | 512/50176 [00:03<05:12, 158.79it/s]
  2%|▏         | 768/50176 [00:04<04:47, 172.08it/s]
  2%|▏         | 1024/50176 [00:06<04:52, 167.78it/s]
  3%|▎         | 1280/50176 [00:08<05:10, 157.46it/s]
  3%|▎         | 1536/50176 [00:10<05:36, 144.69it/s]
  4%|▎         | 1792/50176 [00:11<05:37, 143.24it/s]
  4%|▍         | 2048/50176 [00:13<05:34, 143.97it/s]
  5%|▍         | 2304/50176 [00:15<05:34, 143.13it/s]
  5%|▌         | 2560/50176 [00:17<05:30, 144.20it/s]
  6%|▌         | 2816/50176 [00:19<05:32, 142.39it/s]
  6%|▌         | 3072/50176 [00:20<05:25, 144.52it/s]
  7%|▋         | 3328/50176 [00:22<05:22, 145.21it/s]
  7%|▋         | 3584/50176 [00:24<05:15, 147.50it/s]
  8%|▊         | 3840/50176 [00:26<05:25, 142.19it/s]
  8%|▊         | 4096/50176 [00:27<05:25, 141.36it/s]
  9%|▊         | 4352/50176 [00:29<05:19, 143.35it/s]
  9%|▉         | 4608/50176 [00:31<05:14, 144.76it/s]
 10%|▉         | 4864/50176 [00:33<05:09, 146.53it/s]
 10%|█         | 5120/50176 [00:34<05:05, 147.57it/s]
 11%|█         | 5376/50176 [00:36<05:01, 148.58it/s]
 11%|█         | 5632/50176 [00:38<04:57, 149.53it/s]
 12%|█▏        | 5888/50176 [00:39<04:56, 149.61it/s]
 12%|█▏        | 6144/50176 [00:42<05:20, 137.47it/s]
 13%|█▎        | 6400/50176 [00:43<05:10, 141.12it/s]
 13%|█▎        | 6656/50176 [00:45<05:04, 143.08it/s]
 14%|█▍        | 6912/50176 [00:47<05:01, 143.62it/s]
 14%|█▍        | 7168/50176 [00:49<04:56, 145.08it/s]
 15%|█▍        | 7424/50176 [00:50<04:50, 147.35it/s]
 15%|█▌        | 7680/50176 [00:52<04:51, 146.03it/s]
 16%|█▌        | 7936/50176 [00:54<04:43, 148.91it/s]
 16%|█▋        | 8192/50176 [00:55<04:44, 147.76it/s]
 17%|█▋        | 8448/50176 [00:57<04:40, 148.69it/s]
 17%|█▋        | 8704/50176 [00:59<04:37, 149.41it/s]
 18%|█▊        | 8960/50176 [01:01<04:44, 144.68it/s]
 18%|█▊        | 9216/50176 [01:02<04:37, 147.43it/s]
 19%|█▉        | 9472/50176 [01:04<04:36, 147.19it/s]
 19%|█▉        | 9728/50176 [01:06<04:38, 145.04it/s]
 20%|█▉        | 9984/50176 [01:08<04:32, 147.61it/s]
 20%|██        | 10240/50176 [01:09<04:26, 149.58it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.06it/s]
 21%|██▏       | 10752/50176 [01:13<04:30, 146.00it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 148.11it/s]
 22%|██▏       | 11264/50176 [01:16<04:21, 148.66it/s]
 23%|██▎       | 11520/50176 [01:18<04:15, 151.40it/s]
 23%|██▎       | 11776/50176 [01:20<04:14, 151.09it/s]
 24%|██▍       | 12032/50176 [01:21<04:11, 151.77it/s]
 24%|██▍       | 12288/50176 [01:23<04:09, 152.03it/s]
 25%|██▌       | 12544/50176 [01:25<04:06, 152.61it/s]
 26%|██▌       | 12800/50176 [01:26<04:04, 152.70it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.80it/s]
 27%|██▋       | 13312/50176 [01:30<04:06, 149.63it/s]
 27%|██▋       | 13568/50176 [01:32<04:15, 143.55it/s]
 28%|██▊       | 13824/50176 [01:34<04:18, 140.88it/s]
 28%|██▊       | 14080/50176 [01:35<04:15, 141.19it/s]
 29%|██▊       | 14336/50176 [01:37<04:20, 137.78it/s]
 29%|██▉       | 14592/50176 [01:39<04:08, 142.98it/s]
 30%|██▉       | 14848/50176 [01:41<04:20, 135.71it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.04it/s]
 31%|███       | 15360/50176 [01:45<04:11, 138.31it/s]
 31%|███       | 15616/50176 [01:47<04:15, 135.23it/s]
 32%|███▏      | 15872/50176 [01:49<04:15, 134.05it/s]
 32%|███▏      | 16128/50176 [01:50<04:07, 137.67it/s]
 33%|███▎      | 16384/50176 [01:52<04:04, 138.29it/s]
 33%|███▎      | 16640/50176 [01:54<04:09, 134.23it/s]
 34%|███▎      | 16896/50176 [01:56<03:58, 139.45it/s]
 34%|███▍      | 17152/50176 [01:58<03:51, 142.89it/s]
 35%|███▍      | 17408/50176 [01:59<03:46, 144.51it/s]
 35%|███▌      | 17664/50176 [02:01<03:42, 146.43it/s]
 36%|███▌      | 17920/50176 [02:03<03:37, 148.24it/s]
 36%|███▌      | 18176/50176 [02:04<03:37, 147.09it/s]
 37%|███▋      | 18432/50176 [02:06<03:35, 147.03it/s]
 37%|███▋      | 18688/50176 [02:08<03:34, 147.00it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.52it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.43it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.16it/s]
 39%|███▉      | 19712/50176 [02:16<03:50, 132.43it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.61it/s]
 40%|████      | 20224/50176 [02:19<03:32, 140.74it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.78it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.97it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.38it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 141.83it/s]
 43%|████▎     | 21504/50176 [02:28<03:16, 146.21it/s]
 43%|████▎     | 21760/50176 [02:30<03:12, 147.54it/s]
 44%|████▍     | 22016/50176 [02:31<03:07, 150.50it/s]
 44%|████▍     | 22272/50176 [02:33<03:06, 149.75it/s]
 45%|████▍     | 22528/50176 [02:35<03:03, 150.94it/s]
 45%|████▌     | 22784/50176 [02:37<03:09, 144.53it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.24it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.85it/s]
 47%|████▋     | 23552/50176 [02:43<03:15, 136.52it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.03it/s]
 48%|████▊     | 24064/50176 [02:46<03:03, 142.22it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.51it/s]
 49%|████▉     | 24576/50176 [02:49<02:55, 145.58it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.55it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 149.22it/s]
 51%|█████     | 25344/50176 [02:54<02:46, 149.50it/s]
 51%|█████     | 25600/50176 [02:56<02:48, 145.89it/s]
 52%|█████▏    | 25856/50176 [02:58<02:42, 149.96it/s]
 52%|█████▏    | 26112/50176 [03:00<02:43, 147.41it/s]
 53%|█████▎    | 26368/50176 [03:02<02:46, 142.94it/s]
 53%|█████▎    | 26624/50176 [03:03<02:42, 145.38it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.01it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 145.93it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.11it/s]
 55%|█████▌    | 27648/50176 [03:10<02:28, 152.19it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 141.93it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.76it/s]
 57%|█████▋    | 28416/50176 [03:16<02:31, 144.10it/s]
 57%|█████▋    | 28672/50176 [03:18<02:33, 140.13it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 142.69it/s]
 58%|█████▊    | 29184/50176 [03:21<02:24, 145.04it/s]
 59%|█████▊    | 29440/50176 [03:23<02:32, 136.01it/s]
 59%|█████▉    | 29696/50176 [03:25<02:34, 132.82it/s]
 60%|█████▉    | 29952/50176 [03:27<02:31, 133.45it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.43it/s]
 61%|██████    | 30464/50176 [03:30<02:17, 143.42it/s]
 61%|██████    | 30720/50176 [03:32<02:16, 142.74it/s]
 62%|██████▏   | 30976/50176 [03:34<02:12, 145.44it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.82it/s]
 63%|██████▎   | 31488/50176 [03:37<02:05, 149.10it/s]
 63%|██████▎   | 31744/50176 [03:39<02:03, 149.59it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.39it/s]
 64%|██████▍   | 32256/50176 [03:42<02:00, 149.11it/s]
 65%|██████▍   | 32512/50176 [03:44<02:02, 143.99it/s]
 65%|██████▌   | 32768/50176 [03:46<01:58, 147.32it/s]
 66%|██████▌   | 33024/50176 [03:48<01:58, 145.31it/s]
 66%|██████▋   | 33280/50176 [03:50<01:55, 146.23it/s]
 67%|██████▋   | 33536/50176 [03:51<01:57, 142.14it/s]
 67%|██████▋   | 33792/50176 [03:53<01:52, 146.13it/s]
 68%|██████▊   | 34048/50176 [03:55<01:47, 149.97it/s]
 68%|██████▊   | 34304/50176 [03:56<01:44, 151.59it/s]
 69%|██████▉   | 34560/50176 [03:58<01:47, 145.36it/s]
 69%|██████▉   | 34816/50176 [04:00<01:45, 145.60it/s]
 70%|██████▉   | 35072/50176 [04:02<01:44, 144.90it/s]
 70%|███████   | 35328/50176 [04:03<01:40, 148.13it/s]
 71%|███████   | 35584/50176 [04:05<01:40, 145.61it/s]
 71%|███████▏  | 35840/50176 [04:07<01:41, 141.13it/s]
 72%|███████▏  | 36096/50176 [04:09<01:38, 143.46it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 141.59it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 144.60it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.28it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 144.91it/s]
 74%|███████▍  | 37376/50176 [04:18<01:26, 148.75it/s]
 75%|███████▌  | 37632/50176 [04:19<01:23, 150.91it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 148.82it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.60it/s]
 77%|███████▋  | 38400/50176 [04:25<01:19, 148.60it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.14it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.23it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.73it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.49it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.13it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.71it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.90it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.86it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 137.74it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.44it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.73it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.22it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.74it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 151.92it/s]
 84%|██████���█▍ | 42240/50176 [04:51<00:53, 149.05it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.85it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.27it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.77it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.37it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.07it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.34it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 147.97it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.69it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 148.05it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 146.39it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 148.41it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 142.15it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.65it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 148.07it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.66it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.21it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.49it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 148.05it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.25it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.19it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.71it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 146.69it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 146.65it/s]
 96%|█████████▋| 48384/50176 [05:32<00:11, 149.85it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 152.06it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 151.97it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.77it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.41it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.65it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.57it/s]
-2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5654	imagenet-zeroshot-val-top5: 0.8378

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_23.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:00:57 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:00:57 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:00:59 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt).
-2022-12-21,10:01:07 | INFO | Model:
-2022-12-21,10:01:07 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:01:07 | INFO | Params:
-2022-12-21,10:01:07 | INFO |   batch_size: 256
-2022-12-21,10:01:07 | INFO |   beta1: 0.9
-2022-12-21,10:01:07 | INFO |   beta2: 0.98
-2022-12-21,10:01:07 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/checkpoints
-2022-12-21,10:01:07 | INFO |   copy_codebase: False
-2022-12-21,10:01:07 | INFO |   csv_caption_key: title
-2022-12-21,10:01:07 | INFO |   csv_img_key: filepath
-2022-12-21,10:01:07 | INFO |   csv_separator: 	
-2022-12-21,10:01:07 | INFO |   dataset_resampled: False
-2022-12-21,10:01:07 | INFO |   dataset_type: auto
-2022-12-21,10:01:07 | INFO |   ddp_static_graph: False
-2022-12-21,10:01:07 | INFO |   debug: False
-2022-12-21,10:01:07 | INFO |   device: cuda:0
-2022-12-21,10:01:07 | INFO |   dist_backend: nccl
-2022-12-21,10:01:07 | INFO |   dist_url: env://
-2022-12-21,10:01:07 | INFO |   distributed: False
-2022-12-21,10:01:07 | INFO |   epochs: 32
-2022-12-21,10:01:07 | INFO |   eps: 1e-06
-2022-12-21,10:01:07 | INFO |   force_quick_gelu: False
-2022-12-21,10:01:07 | INFO |   gather_with_grad: False
-2022-12-21,10:01:07 | INFO |   grad_checkpointing: False
-2022-12-21,10:01:07 | INFO |   horovod: False
-2022-12-21,10:01:07 | INFO |   image_mean: None
-2022-12-21,10:01:07 | INFO |   image_std: None
-2022-12-21,10:01:07 | INFO |   imagenet_v2: None
-2022-12-21,10:01:07 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:01:07 | INFO |   local_loss: False
-2022-12-21,10:01:07 | INFO |   local_rank: 0
-2022-12-21,10:01:07 | INFO |   lock_image: False
-2022-12-21,10:01:07 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:01:07 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:01:07 | INFO |   log_level: 20
-2022-12-21,10:01:07 | INFO |   log_local: False
-2022-12-21,10:01:07 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_23.pt/out.log
-2022-12-21,10:01:07 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:01:07 | INFO |   lr: 0.0005
-2022-12-21,10:01:07 | INFO |   model: ViT-B-16
-2022-12-21,10:01:07 | INFO |   name: epoch_23.pt
-2022-12-21,10:01:07 | INFO |   no_set_device_rank: False
-2022-12-21,10:01:07 | INFO |   norm_gradient_clip: None
-2022-12-21,10:01:07 | INFO |   precision: amp_bfloat16
-2022-12-21,10:01:07 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_23.pt
-2022-12-21,10:01:07 | INFO |   pretrained_image: False
-2022-12-21,10:01:07 | INFO |   rank: 0
-2022-12-21,10:01:07 | INFO |   report_to: 
-2022-12-21,10:01:07 | INFO |   resume: None
-2022-12-21,10:01:07 | INFO |   save_frequency: 1
-2022-12-21,10:01:07 | INFO |   save_most_recent: False
-2022-12-21,10:01:07 | INFO |   seed: 0
-2022-12-21,10:01:07 | INFO |   skip_scheduler: False
-2022-12-21,10:01:07 | INFO |   tensorboard: False
-2022-12-21,10:01:07 | INFO |   tensorboard_path: 
-2022-12-21,10:01:07 | INFO |   torchscript: False
-2022-12-21,10:01:07 | INFO |   trace: False
-2022-12-21,10:01:07 | INFO |   train_data: None
-2022-12-21,10:01:07 | INFO |   train_num_samples: None
-2022-12-21,10:01:07 | INFO |   use_bn_sync: False
-2022-12-21,10:01:07 | INFO |   val_data: None
-2022-12-21,10:01:07 | INFO |   val_frequency: 1
-2022-12-21,10:01:07 | INFO |   val_num_samples: None
-2022-12-21,10:01:07 | INFO |   wandb: False
-2022-12-21,10:01:07 | INFO |   wandb_notes: 
-2022-12-21,10:01:07 | INFO |   warmup: 10000
-2022-12-21,10:01:07 | INFO |   wd: 0.2
-2022-12-21,10:01:07 | INFO |   workers: 1
-2022-12-21,10:01:07 | INFO |   world_size: 1
-2022-12-21,10:01:07 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:01:07 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:01:07 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:23,  1.46it/s]
  1%|          | 6/1000 [00:00<01:41,  9.84it/s]
  1%|          | 11/1000 [00:00<00:55, 17.77it/s]
  2%|▏         | 17/1000 [00:00<00:37, 26.53it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.01it/s]
  3%|▎         | 29/1000 [00:01<00:24, 40.03it/s]
  4%|▎         | 35/1000 [00:01<00:21, 44.64it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.19it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.62it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.60it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.84it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.86it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.46it/s]
  8%|▊         | 77/1000 [00:02<00:16, 55.97it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.41it/s]
  9%|▉         | 89/1000 [00:02<00:16, 56.80it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.01it/s]
 10%|█         | 101/1000 [00:02<00:17, 51.99it/s]
 11%|█         | 107/1000 [00:02<00:16, 53.67it/s]
 11%|█▏        | 113/1000 [00:02<00:16, 54.87it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 55.66it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 56.16it/s]
 13%|█▎        | 131/1000 [00:03<00:15, 56.63it/s]
 14%|█▎        | 137/1000 [00:03<00:15, 56.82it/s]
 14%|█▍        | 143/1000 [00:03<00:15, 57.06it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.08it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.29it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.24it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.21it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.26it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.30it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.17it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.17it/s]
 20%|█▉        | 197/1000 [00:04<00:14, 57.13it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.01it/s]
 21%|██        | 209/1000 [00:04<00:13, 56.86it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 56.91it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 56.92it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.11it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.19it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.02it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.04it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 56.94it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.17it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.24it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.16it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.29it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.61it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.60it/s]
 30%|███       | 305/1000 [00:06<00:12, 57.61it/s]
 31%|███       | 311/1000 [00:06<00:11, 57.72it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 57.72it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 57.67it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.46it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.51it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 57.57it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 57.70it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.51it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 57.63it/s]
 36%|███▋      | 365/1000 [00:07<00:11, 57.59it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 57.71it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.55it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.13it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 56.88it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.06it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.25it/s]
 41%|████      | 407/1000 [00:07<00:10, 57.42it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 57.50it/s]
 42%|████▏     | 419/1000 [00:08<00:10, 57.54it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.52it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.69it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 57.72it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.52it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 57.57it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 57.65it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 57.63it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 57.61it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 57.73it/s]
 48%|████▊     | 479/1000 [00:09<00:09, 57.63it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 57.62it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.60it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.50it/s]
 50%|█████     | 503/1000 [00:09<00:08, 57.64it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.51it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 57.61it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 57.72it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 57.63it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 57.73it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 57.76it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.63it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.55it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 57.65it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.55it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.48it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 57.64it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 57.56it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.50it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 57.61it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.51it/s]
 61%|██████    | 611/1000 [00:11<00:06, 57.58it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 57.66it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 57.67it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 57.74it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.60it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.47it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 57.69it/s]
 65%|██████▌   | 653/1000 [00:12<00:06, 57.65it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 57.76it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 57.74it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 57.70it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 57.74it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 57.72it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.77it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.65it/s]
 70%|███████   | 701/1000 [00:12<00:05, 57.76it/s]
 71%|███████   | 707/1000 [00:13<00:05, 57.75it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 57.75it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 57.77it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 57.69it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 57.63it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 57.69it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 57.64it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 57.73it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 57.85it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 57.69it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 57.77it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 57.77it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 57.60it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 57.68it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.52it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.54it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.58it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.55it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 57.56it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 57.62it/s]
 83%|████████▎ | 827/1000 [00:15<00:03, 57.63it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 57.82it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 57.87it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 57.72it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 57.78it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 57.72it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 57.79it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 57.71it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 57.72it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 57.76it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.61it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 57.71it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 57.74it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 57.73it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 57.43it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 57.53it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.55it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 57.71it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 57.82it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 57.82it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 57.81it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 57.84it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 57.78it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 57.54it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 57.71it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 57.82it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 57.89it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 57.85it/s]
-2022-12-21,10:01:25 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:12, 133.90it/s]
  1%|          | 512/50176 [00:03<05:25, 152.35it/s]
  2%|▏         | 768/50176 [00:04<04:54, 167.53it/s]
  2%|▏         | 1024/50176 [00:06<04:47, 171.03it/s]
  3%|▎         | 1280/50176 [00:08<05:07, 159.08it/s]
  3%|▎         | 1536/50176 [00:10<05:33, 145.88it/s]
  4%|▎         | 1792/50176 [00:11<05:35, 144.10it/s]
  4%|▍         | 2048/50176 [00:13<05:32, 144.65it/s]
  5%|▍         | 2304/50176 [00:15<05:34, 143.28it/s]
  5%|▌         | 2560/50176 [00:17<05:30, 144.08it/s]
  6%|▌         | 2816/50176 [00:19<05:31, 142.69it/s]
  6%|▌         | 3072/50176 [00:20<05:25, 144.54it/s]
  7%|▋         | 3328/50176 [00:22<05:22, 145.25it/s]
  7%|▋         | 3584/50176 [00:24<05:15, 147.57it/s]
  8%|▊         | 3840/50176 [00:26<05:27, 141.62it/s]
  8%|▊         | 4096/50176 [00:27<05:25, 141.71it/s]
  9%|▊         | 4352/50176 [00:29<05:19, 143.57it/s]
  9%|▉         | 4608/50176 [00:31<05:14, 144.88it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.68it/s]
 10%|█         | 5120/50176 [00:34<05:06, 147.01it/s]
 11%|█         | 5376/50176 [00:36<05:00, 148.86it/s]
 11%|█         | 5632/50176 [00:38<04:57, 149.82it/s]
 12%|█▏        | 5888/50176 [00:39<04:55, 149.71it/s]
 12%|█▏        | 6144/50176 [00:42<05:20, 137.54it/s]
 13%|█▎        | 6400/50176 [00:43<05:10, 141.09it/s]
 13%|█▎        | 6656/50176 [00:45<05:04, 143.03it/s]
 14%|█▍        | 6912/50176 [00:47<05:01, 143.72it/s]
 14%|█▍        | 7168/50176 [00:49<04:56, 145.13it/s]
 15%|█▍        | 7424/50176 [00:50<04:51, 146.69it/s]
 15%|█▌        | 7680/50176 [00:52<04:50, 146.26it/s]
 16%|█▌        | 7936/50176 [00:54<04:43, 149.07it/s]
 16%|█▋        | 8192/50176 [00:55<04:44, 147.81it/s]
 17%|█▋        | 8448/50176 [00:57<04:40, 148.74it/s]
 17%|█▋        | 8704/50176 [00:59<04:37, 149.42it/s]
 18%|█▊        | 8960/50176 [01:01<04:45, 144.22it/s]
 18%|█▊        | 9216/50176 [01:02<04:37, 147.56it/s]
 19%|█▉        | 9472/50176 [01:04<04:35, 147.92it/s]
 19%|█▉        | 9728/50176 [01:06<04:40, 144.35it/s]
 20%|█▉        | 9984/50176 [01:08<04:32, 147.72it/s]
 20%|██        | 10240/50176 [01:09<04:26, 149.75it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.23it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.31it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 148.21it/s]
 22%|██▏       | 11264/50176 [01:16<04:21, 148.76it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.91it/s]
 23%|██▎       | 11776/50176 [01:20<04:13, 151.38it/s]
 24%|██▍       | 12032/50176 [01:21<04:11, 151.83it/s]
 24%|██▍       | 12288/50176 [01:23<04:09, 152.14it/s]
 25%|██▌       | 12544/50176 [01:25<04:06, 152.61it/s]
 26%|██▌       | 12800/50176 [01:26<04:04, 152.87it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.85it/s]
 27%|██▋       | 13312/50176 [01:30<04:06, 149.75it/s]
 27%|██▋       | 13568/50176 [01:32<04:15, 143.22it/s]
 28%|██▊       | 13824/50176 [01:34<04:17, 141.08it/s]
 28%|██▊       | 14080/50176 [01:35<04:15, 141.25it/s]
 29%|██▊       | 14336/50176 [01:37<04:20, 137.57it/s]
 29%|██▉       | 14592/50176 [01:39<04:08, 143.07it/s]
 30%|██▉       | 14848/50176 [01:41<04:20, 135.77it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.13it/s]
 31%|███       | 15360/50176 [01:45<04:11, 138.37it/s]
 31%|███       | 15616/50176 [01:47<04:14, 135.66it/s]
 32%|███▏      | 15872/50176 [01:49<04:16, 133.95it/s]
 32%|███▏      | 16128/50176 [01:50<04:07, 137.54it/s]
 33%|███▎      | 16384/50176 [01:52<04:04, 138.21it/s]
 33%|███▎      | 16640/50176 [01:54<04:10, 134.12it/s]
 34%|███▎      | 16896/50176 [01:56<03:58, 139.43it/s]
 34%|███▍      | 17152/50176 [01:58<03:51, 142.85it/s]
 35%|███▍      | 17408/50176 [01:59<03:46, 144.43it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.54it/s]
 36%|███▌      | 17920/50176 [02:03<03:37, 148.10it/s]
 36%|███▌      | 18176/50176 [02:04<03:37, 147.20it/s]
 37%|███▋      | 18432/50176 [02:06<03:35, 147.51it/s]
 37%|███▋      | 18688/50176 [02:08<03:34, 146.86it/s]
 38%|███▊      | 18944/50176 [02:10<03:40, 141.91it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.57it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.17it/s]
 39%|███▉      | 19712/50176 [02:16<03:50, 132.01it/s]
 40%|███▉      | 19968/50176 [02:18<03:43, 135.21it/s]
 40%|████      | 20224/50176 [02:19<03:32, 141.02it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.70it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.90it/s]
 42%|████▏     | 20992/50176 [02:25<03:19, 146.59it/s]
 42%|████▏     | 21248/50176 [02:27<03:24, 141.53it/s]
 43%|████▎     | 21504/50176 [02:28<03:16, 146.26it/s]
 43%|████▎     | 21760/50176 [02:30<03:12, 147.70it/s]
 44%|████▍     | 22016/50176 [02:31<03:07, 150.46it/s]
 44%|████▍     | 22272/50176 [02:33<03:07, 149.06it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.15it/s]
 45%|████▌     | 22784/50176 [02:37<03:09, 144.43it/s]
 46%|████▌     | 23040/50176 [02:39<03:09, 143.36it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.56it/s]
 47%|████▋     | 23552/50176 [02:43<03:13, 137.29it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.06it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 142.77it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.28it/s]
 49%|████▉     | 24576/50176 [02:49<02:56, 145.39it/s]
 49%|████▉     | 24832/50176 [02:51<02:51, 147.70it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 149.21it/s]
 51%|█████     | 25344/50176 [02:54<02:45, 149.67it/s]
 51%|█████     | 25600/50176 [02:56<02:49, 145.31it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.61it/s]
 52%|█████▏    | 26112/50176 [03:00<02:43, 146.80it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 143.59it/s]
 53%|█████▎    | 26624/50176 [03:03<02:41, 145.41it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.03it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 145.91it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.27it/s]
 55%|█████▌    | 27648/50176 [03:10<02:28, 152.17it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 141.94it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.74it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.16it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.69it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 142.63it/s]
 58%|█████▊    | 29184/50176 [03:21<02:24, 145.01it/s]
 59%|█████▊    | 29440/50176 [03:23<02:32, 135.84it/s]
 59%|█████▉    | 29696/50176 [03:25<02:34, 132.60it/s]
 60%|█████▉    | 29952/50176 [03:27<02:31, 133.47it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.49it/s]
 61%|██████    | 30464/50176 [03:30<02:17, 143.41it/s]
 61%|██████    | 30720/50176 [03:32<02:16, 142.74it/s]
 62%|██████▏   | 30976/50176 [03:34<02:12, 145.43it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.75it/s]
 63%|██████▎   | 31488/50176 [03:37<02:05, 148.39it/s]
 63%|██████▎   | 31744/50176 [03:39<02:03, 149.83it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.47it/s]
 64%|██████▍   | 32256/50176 [03:42<02:00, 149.18it/s]
 65%|██████▍   | 32512/50176 [03:44<02:02, 143.87it/s]
 65%|██████▌   | 32768/50176 [03:46<01:58, 147.45it/s]
 66%|██████▌   | 33024/50176 [03:48<01:57, 145.42it/s]
 66%|██████▋   | 33280/50176 [03:50<01:55, 146.49it/s]
 67%|██████▋   | 33536/50176 [03:51<01:56, 142.66it/s]
 67%|██████▋   | 33792/50176 [03:53<01:52, 146.15it/s]
 68%|██████▊   | 34048/50176 [03:55<01:47, 149.94it/s]
 68%|██████▊   | 34304/50176 [03:56<01:44, 151.58it/s]
 69%|██████▉   | 34560/50176 [03:58<01:47, 145.11it/s]
 69%|██████▉   | 34816/50176 [04:00<01:45, 145.68it/s]
 70%|██████▉   | 35072/50176 [04:02<01:44, 144.24it/s]
 70%|███████   | 35328/50176 [04:03<01:40, 147.96it/s]
 71%|███████   | 35584/50176 [04:05<01:40, 145.71it/s]
 71%|███████▏  | 35840/50176 [04:07<01:41, 141.82it/s]
 72%|███████▏  | 36096/50176 [04:09<01:38, 143.26it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 141.56it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 144.68it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 143.09it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 144.82it/s]
 74%|███████▍  | 37376/50176 [04:18<01:26, 148.79it/s]
 75%|███████▌  | 37632/50176 [04:19<01:23, 150.95it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 148.90it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.60it/s]
 77%|███████▋  | 38400/50176 [04:25<01:19, 148.59it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.15it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.26it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.63it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.50it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.15it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.73it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.27it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 144.05it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 137.92it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.54it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.67it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.99it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.46it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 151.80it/s]
 84%|██████���█▍ | 42240/50176 [04:51<00:53, 148.99it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.83it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 148.56it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.95it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.50it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.16it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.24it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 147.45it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.90it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 148.17it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 146.64it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 148.44it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 141.90it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.75it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 148.21it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.78it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.56it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.44it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 148.04it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.06it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.28it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.64it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 146.68it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 147.21it/s]
 96%|█████████▋| 48384/50176 [05:32<00:11, 149.68it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 152.02it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 151.78it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.63it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.31it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.55it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.68it/s]
-2022-12-21,10:07:09 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:07:09 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5731	imagenet-zeroshot-val-top5: 0.8431

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_24.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt).
-2022-12-21,10:07:24 | INFO | Model:
-2022-12-21,10:07:24 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:07:24 | INFO | Params:
-2022-12-21,10:07:24 | INFO |   batch_size: 256
-2022-12-21,10:07:24 | INFO |   beta1: 0.9
-2022-12-21,10:07:24 | INFO |   beta2: 0.98
-2022-12-21,10:07:24 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/checkpoints
-2022-12-21,10:07:24 | INFO |   copy_codebase: False
-2022-12-21,10:07:24 | INFO |   csv_caption_key: title
-2022-12-21,10:07:24 | INFO |   csv_img_key: filepath
-2022-12-21,10:07:24 | INFO |   csv_separator: 	
-2022-12-21,10:07:24 | INFO |   dataset_resampled: False
-2022-12-21,10:07:24 | INFO |   dataset_type: auto
-2022-12-21,10:07:24 | INFO |   ddp_static_graph: False
-2022-12-21,10:07:24 | INFO |   debug: False
-2022-12-21,10:07:24 | INFO |   device: cuda:0
-2022-12-21,10:07:24 | INFO |   dist_backend: nccl
-2022-12-21,10:07:24 | INFO |   dist_url: env://
-2022-12-21,10:07:24 | INFO |   distributed: False
-2022-12-21,10:07:24 | INFO |   epochs: 32
-2022-12-21,10:07:24 | INFO |   eps: 1e-06
-2022-12-21,10:07:24 | INFO |   force_quick_gelu: False
-2022-12-21,10:07:24 | INFO |   gather_with_grad: False
-2022-12-21,10:07:24 | INFO |   grad_checkpointing: False
-2022-12-21,10:07:24 | INFO |   horovod: False
-2022-12-21,10:07:24 | INFO |   image_mean: None
-2022-12-21,10:07:24 | INFO |   image_std: None
-2022-12-21,10:07:24 | INFO |   imagenet_v2: None
-2022-12-21,10:07:24 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:07:24 | INFO |   local_loss: False
-2022-12-21,10:07:24 | INFO |   local_rank: 0
-2022-12-21,10:07:24 | INFO |   lock_image: False
-2022-12-21,10:07:24 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:07:24 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:07:24 | INFO |   log_level: 20
-2022-12-21,10:07:24 | INFO |   log_local: False
-2022-12-21,10:07:24 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_24.pt/out.log
-2022-12-21,10:07:24 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:07:24 | INFO |   lr: 0.0005
-2022-12-21,10:07:24 | INFO |   model: ViT-B-16
-2022-12-21,10:07:24 | INFO |   name: epoch_24.pt
-2022-12-21,10:07:24 | INFO |   no_set_device_rank: False
-2022-12-21,10:07:24 | INFO |   norm_gradient_clip: None
-2022-12-21,10:07:24 | INFO |   precision: amp_bfloat16
-2022-12-21,10:07:24 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_24.pt
-2022-12-21,10:07:24 | INFO |   pretrained_image: False
-2022-12-21,10:07:24 | INFO |   rank: 0
-2022-12-21,10:07:24 | INFO |   report_to: 
-2022-12-21,10:07:24 | INFO |   resume: None
-2022-12-21,10:07:24 | INFO |   save_frequency: 1
-2022-12-21,10:07:24 | INFO |   save_most_recent: False
-2022-12-21,10:07:24 | INFO |   seed: 0
-2022-12-21,10:07:24 | INFO |   skip_scheduler: False
-2022-12-21,10:07:24 | INFO |   tensorboard: False
-2022-12-21,10:07:24 | INFO |   tensorboard_path: 
-2022-12-21,10:07:24 | INFO |   torchscript: False
-2022-12-21,10:07:24 | INFO |   trace: False
-2022-12-21,10:07:24 | INFO |   train_data: None
-2022-12-21,10:07:24 | INFO |   train_num_samples: None
-2022-12-21,10:07:24 | INFO |   use_bn_sync: False
-2022-12-21,10:07:24 | INFO |   val_data: None
-2022-12-21,10:07:24 | INFO |   val_frequency: 1
-2022-12-21,10:07:24 | INFO |   val_num_samples: None
-2022-12-21,10:07:24 | INFO |   wandb: False
-2022-12-21,10:07:24 | INFO |   wandb_notes: 
-2022-12-21,10:07:24 | INFO |   warmup: 10000
-2022-12-21,10:07:24 | INFO |   wd: 0.2
-2022-12-21,10:07:24 | INFO |   workers: 1
-2022-12-21,10:07:24 | INFO |   world_size: 1
-2022-12-21,10:07:24 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:07:25 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:29,  1.59it/s]
  1%|          | 6/1000 [00:00<01:35, 10.39it/s]
  1%|          | 12/1000 [00:00<00:49, 19.83it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.32it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.47it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.21it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.43it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.67it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.11it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.90it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.18it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.14it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.78it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.24it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.76it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.02it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.39it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.52it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.74it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.88it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.79it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.84it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.81it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.84it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.91it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.96it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.69it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.62it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.58it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.52it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.49it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.43it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.26it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.16it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.33it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.28it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.46it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.45it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.39it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.45it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.54it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.59it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.41it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.65it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.88it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.87it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.84it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.01it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.99it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.92it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.74it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.83it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.95it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.04it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.82it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.80it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.88it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.96it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.52it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.58it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.69it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.78it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.80it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.84it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.88it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.89it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.01it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.02it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.81it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.89it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.96it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.94it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.88it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.00it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.91it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.90it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.92it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.80it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.86it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.79it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.86it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.91it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.99it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.94it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.04it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.05it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.94it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.94it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.02it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.88it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.59it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.77it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.79it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.74it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.87it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.79it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.81it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.93it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.03it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.86it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.79it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.95it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 57.91it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.98it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.04it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.97it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.96it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.97it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.89it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.97it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.98it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.97it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.99it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.05it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.01it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.08it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.09it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.94it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.05it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.97it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.95it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.01it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.81it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.80it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.85it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.83it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.83it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.85it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 57.95it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.05it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.12it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.04it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.10it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.04it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.07it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.01it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.04it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.03it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.90it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.00it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.02it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.94it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.04it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.08it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.08it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.11it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.85it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.81it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.76it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.91it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.02it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.08it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.07it/s]
-2022-12-21,10:07:43 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:57, 139.57it/s]
  1%|          | 512/50176 [00:03<05:18, 155.78it/s]
  2%|▏         | 768/50176 [00:04<05:14, 156.94it/s]
  2%|▏         | 1024/50176 [00:06<05:25, 151.21it/s]
  3%|▎         | 1280/50176 [00:08<05:32, 147.06it/s]
  3%|▎         | 1536/50176 [00:10<05:51, 138.46it/s]
  4%|▎         | 1792/50176 [00:12<05:51, 137.75it/s]
  4%|▍         | 2048/50176 [00:14<05:44, 139.80it/s]
  5%|▍         | 2304/50176 [00:16<05:43, 139.22it/s]
  5%|▌         | 2560/50176 [00:17<05:36, 141.69it/s]
  6%|▌         | 2816/50176 [00:19<05:36, 140.65it/s]
  6%|▌         | 3072/50176 [00:21<05:31, 141.92it/s]
  7%|▋         | 3328/50176 [00:23<05:25, 143.91it/s]
  7%|▋         | 3584/50176 [00:24<05:18, 146.31it/s]
  8%|▊         | 3840/50176 [00:26<05:30, 140.39it/s]
  8%|▊         | 4096/50176 [00:28<05:27, 140.49it/s]
  9%|▊         | 4352/50176 [00:30<05:21, 142.40it/s]
  9%|▉         | 4608/50176 [00:32<05:16, 144.01it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.87it/s]
 10%|█         | 5120/50176 [00:35<05:05, 147.59it/s]
 11%|█         | 5376/50176 [00:37<05:03, 147.84it/s]
 11%|█         | 5632/50176 [00:39<05:01, 147.70it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.42it/s]
 12%|█▏        | 6144/50176 [00:42<05:22, 136.50it/s]
 13%|█▎        | 6400/50176 [00:44<05:12, 140.30it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.57it/s]
 14%|█▍        | 6912/50176 [00:48<05:02, 142.98it/s]
 14%|█▍        | 7168/50176 [00:49<04:57, 144.73it/s]
 15%|█▍        | 7424/50176 [00:51<04:50, 147.05it/s]
 15%|█▌        | 7680/50176 [00:53<04:49, 147.00it/s]
 16%|█▌        | 7936/50176 [00:54<04:39, 150.98it/s]
 16%|█▋        | 8192/50176 [00:56<04:38, 151.00it/s]
 17%|█▋        | 8448/50176 [00:58<04:35, 151.64it/s]
 17%|█▋        | 8704/50176 [00:59<04:32, 152.10it/s]
 18%|█▊        | 8960/50176 [01:01<04:39, 147.24it/s]
 18%|█▊        | 9216/50176 [01:03<04:32, 150.36it/s]
 19%|█▉        | 9472/50176 [01:05<04:31, 149.99it/s]
 19%|█▉        | 9728/50176 [01:06<04:36, 146.15it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 148.27it/s]
 20%|██        | 10240/50176 [01:10<04:27, 149.48it/s]
 21%|██        | 10496/50176 [01:12<04:24, 150.29it/s]
 21%|██▏       | 10752/50176 [01:13<04:32, 144.93it/s]
 22%|██▏       | 11008/50176 [01:15<04:27, 146.54it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.12it/s]
 23%|██▎       | 11520/50176 [01:18<04:17, 150.39it/s]
 23%|██▎       | 11776/50176 [01:20<04:15, 150.23it/s]
 24%|██▍       | 12032/50176 [01:22<04:12, 150.94it/s]
 24%|██▍       | 12288/50176 [01:23<04:08, 152.64it/s]
 25%|██▌       | 12544/50176 [01:25<04:04, 153.73it/s]
 26%|██▌       | 12800/50176 [01:27<04:02, 154.06it/s]
 26%|██▌       | 13056/50176 [01:28<03:59, 154.98it/s]
 27%|██▋       | 13312/50176 [01:30<04:02, 152.08it/s]
 27%|██▋       | 13568/50176 [01:32<04:10, 145.89it/s]
 28%|██▊       | 13824/50176 [01:34<04:14, 142.90it/s]
 28%|██▊       | 14080/50176 [01:36<04:43, 127.51it/s]
 29%|██▊       | 14336/50176 [01:38<04:38, 128.82it/s]
 29%|██▉       | 14592/50176 [01:40<04:19, 137.20it/s]
 30%|██▉       | 14848/50176 [01:42<04:27, 131.96it/s]
 30%|███       | 15104/50176 [01:44<04:26, 131.69it/s]
 31%|███       | 15360/50176 [01:46<04:15, 136.42it/s]
 31%|███       | 15616/50176 [01:48<04:08, 138.90it/s]
 32%|███▏      | 15872/50176 [01:49<04:11, 136.26it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.68it/s]
 33%|███▎      | 16384/50176 [01:53<04:02, 139.23it/s]
 33%|███▎      | 16640/50176 [01:55<04:06, 136.19it/s]
 34%|███▎      | 16896/50176 [01:57<03:57, 140.15it/s]
 34%|███▍      | 17152/50176 [01:58<03:48, 144.33it/s]
 35%|███▍      | 17408/50176 [02:00<03:44, 145.96it/s]
 35%|███▌      | 17664/50176 [02:02<03:39, 147.79it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.44it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.24it/s]
 37%|███▋      | 18432/50176 [02:07<03:32, 149.51it/s]
 37%|███▋      | 18688/50176 [02:09<03:31, 148.89it/s]
 38%|███▊      | 18944/50176 [02:11<03:38, 142.99it/s]
 38%|███▊      | 19200/50176 [02:13<03:49, 134.82it/s]
 39%|███▉      | 19456/50176 [02:15<03:52, 132.19it/s]
 39%|███▉      | 19712/50176 [02:17<03:50, 132.32it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.63it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.54it/s]
 41%|████      | 20480/50176 [02:22<03:25, 144.84it/s]
 41%|████▏     | 20736/50176 [02:24<03:23, 145.02it/s]
 42%|████▏     | 20992/50176 [02:25<03:17, 147.53it/s]
 42%|████▏     | 21248/50176 [02:27<03:21, 143.28it/s]
 43%|████▎     | 21504/50176 [02:29<03:14, 147.57it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.31it/s]
 44%|████▍     | 22016/50176 [02:32<03:06, 150.97it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.59it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.61it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.69it/s]
 46%|████▌     | 23040/50176 [02:39<03:07, 144.40it/s]
 46%|████▋     | 23296/50176 [02:41<03:10, 140.75it/s]
 47%|████▋     | 23552/50176 [02:43<03:12, 138.37it/s]
 47%|████▋     | 23808/50176 [02:45<03:05, 142.01it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 143.14it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.54it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.84it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 149.04it/s]
 50%|█████     | 25088/50176 [02:53<02:47, 149.64it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.88it/s]
 51%|█████     | 25600/50176 [02:57<02:46, 147.53it/s]
 52%|█████▏    | 25856/50176 [02:58<02:40, 151.16it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.39it/s]
 53%|█████▎    | 26368/50176 [03:02<02:44, 144.80it/s]
 53%|█████▎    | 26624/50176 [03:04<02:40, 146.84it/s]
 54%|█████▎    | 26880/50176 [03:05<02:38, 147.03it/s]
 54%|█████▍    | 27136/50176 [03:07<02:35, 147.77it/s]
 55%|█████▍    | 27392/50176 [03:09<02:33, 148.16it/s]
 55%|█████▌    | 27648/50176 [03:10<02:25, 155.23it/s]
 56%|█████▌    | 27904/50176 [03:12<02:34, 143.81it/s]
 56%|█████▌    | 28160/50176 [03:14<02:30, 145.82it/s]
 57%|█████▋    | 28416/50176 [03:16<02:29, 145.56it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.29it/s]
 58%|█████▊    | 28928/50176 [03:20<02:27, 144.10it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 146.43it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.01it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.39it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.22it/s]
 60%|██████    | 30208/50176 [03:29<02:24, 138.23it/s]
 61%|██████    | 30464/50176 [03:30<02:16, 144.56it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 143.61it/s]
 62%|██████▏   | 30976/50176 [03:34<02:11, 145.79it/s]
 62%|██████▏   | 31232/50176 [03:36<02:08, 147.98it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.13it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.55it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.66it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 149.86it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 145.40it/s]
 65%|██████▌   | 32768/50176 [03:46<01:57, 148.49it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 146.66it/s]
 66%|██████▋   | 33280/50176 [03:49<01:54, 148.10it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 144.90it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.80it/s]
 68%|██████▊   | 34048/50176 [03:55<01:45, 152.31it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 155.39it/s]
 69%|██████▉   | 34560/50176 [03:58<01:46, 147.03it/s]
 69%|██████▉   | 34816/50176 [04:00<01:44, 146.75it/s]
 70%|██████▉   | 35072/50176 [04:02<01:43, 146.10it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 148.64it/s]
 71%|███████   | 35584/50176 [04:05<01:39, 147.23it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.57it/s]
 72%|███████▏  | 36096/50176 [04:09<01:36, 145.58it/s]
 72%|███████▏  | 36352/50176 [04:10<01:37, 142.49it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.24it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 142.43it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 143.62it/s]
 74%|███████▍  | 37376/50176 [04:17<01:26, 148.53it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.29it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 149.19it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.81it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.34it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.44it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 144.79it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.48it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.64it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.33it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 146.97it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.65it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 144.15it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.57it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.10it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.65it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.88it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 152.06it/s]
 84%|████████▎ | 41984/50176 [04:49<00:53, 152.42it/s]
 84%|██████���█▍ | 42240/50176 [04:50<00:53, 148.90it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 150.10it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.20it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 151.39it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 151.03it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.10it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.28it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.08it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.31it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.30it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 145.63it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.36it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 140.62it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.55it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 147.39it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 147.30it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.50it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.69it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 148.32it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.21it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 145.58it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.74it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.32it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 147.04it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 147.49it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.11it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 150.29it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 146.50it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 144.81it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.52it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.22it/s]
-2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5730	imagenet-zeroshot-val-top5: 0.8418

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_25.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt).
-2022-12-21,10:07:24 | INFO | Model:
-2022-12-21,10:07:24 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:07:24 | INFO | Params:
-2022-12-21,10:07:24 | INFO |   batch_size: 256
-2022-12-21,10:07:24 | INFO |   beta1: 0.9
-2022-12-21,10:07:24 | INFO |   beta2: 0.98
-2022-12-21,10:07:24 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/checkpoints
-2022-12-21,10:07:24 | INFO |   copy_codebase: False
-2022-12-21,10:07:24 | INFO |   csv_caption_key: title
-2022-12-21,10:07:24 | INFO |   csv_img_key: filepath
-2022-12-21,10:07:24 | INFO |   csv_separator: 	
-2022-12-21,10:07:24 | INFO |   dataset_resampled: False
-2022-12-21,10:07:24 | INFO |   dataset_type: auto
-2022-12-21,10:07:24 | INFO |   ddp_static_graph: False
-2022-12-21,10:07:24 | INFO |   debug: False
-2022-12-21,10:07:24 | INFO |   device: cuda:0
-2022-12-21,10:07:24 | INFO |   dist_backend: nccl
-2022-12-21,10:07:24 | INFO |   dist_url: env://
-2022-12-21,10:07:24 | INFO |   distributed: False
-2022-12-21,10:07:24 | INFO |   epochs: 32
-2022-12-21,10:07:24 | INFO |   eps: 1e-06
-2022-12-21,10:07:24 | INFO |   force_quick_gelu: False
-2022-12-21,10:07:24 | INFO |   gather_with_grad: False
-2022-12-21,10:07:24 | INFO |   grad_checkpointing: False
-2022-12-21,10:07:24 | INFO |   horovod: False
-2022-12-21,10:07:24 | INFO |   image_mean: None
-2022-12-21,10:07:24 | INFO |   image_std: None
-2022-12-21,10:07:24 | INFO |   imagenet_v2: None
-2022-12-21,10:07:24 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:07:24 | INFO |   local_loss: False
-2022-12-21,10:07:24 | INFO |   local_rank: 0
-2022-12-21,10:07:24 | INFO |   lock_image: False
-2022-12-21,10:07:24 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:07:24 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:07:24 | INFO |   log_level: 20
-2022-12-21,10:07:24 | INFO |   log_local: False
-2022-12-21,10:07:24 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_25.pt/out.log
-2022-12-21,10:07:24 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:07:24 | INFO |   lr: 0.0005
-2022-12-21,10:07:24 | INFO |   model: ViT-B-16
-2022-12-21,10:07:24 | INFO |   name: epoch_25.pt
-2022-12-21,10:07:24 | INFO |   no_set_device_rank: False
-2022-12-21,10:07:24 | INFO |   norm_gradient_clip: None
-2022-12-21,10:07:24 | INFO |   precision: amp_bfloat16
-2022-12-21,10:07:24 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_25.pt
-2022-12-21,10:07:24 | INFO |   pretrained_image: False
-2022-12-21,10:07:24 | INFO |   rank: 0
-2022-12-21,10:07:24 | INFO |   report_to: 
-2022-12-21,10:07:24 | INFO |   resume: None
-2022-12-21,10:07:24 | INFO |   save_frequency: 1
-2022-12-21,10:07:24 | INFO |   save_most_recent: False
-2022-12-21,10:07:24 | INFO |   seed: 0
-2022-12-21,10:07:24 | INFO |   skip_scheduler: False
-2022-12-21,10:07:24 | INFO |   tensorboard: False
-2022-12-21,10:07:24 | INFO |   tensorboard_path: 
-2022-12-21,10:07:24 | INFO |   torchscript: False
-2022-12-21,10:07:24 | INFO |   trace: False
-2022-12-21,10:07:24 | INFO |   train_data: None
-2022-12-21,10:07:24 | INFO |   train_num_samples: None
-2022-12-21,10:07:24 | INFO |   use_bn_sync: False
-2022-12-21,10:07:24 | INFO |   val_data: None
-2022-12-21,10:07:24 | INFO |   val_frequency: 1
-2022-12-21,10:07:24 | INFO |   val_num_samples: None
-2022-12-21,10:07:24 | INFO |   wandb: False
-2022-12-21,10:07:24 | INFO |   wandb_notes: 
-2022-12-21,10:07:24 | INFO |   warmup: 10000
-2022-12-21,10:07:24 | INFO |   wd: 0.2
-2022-12-21,10:07:24 | INFO |   workers: 1
-2022-12-21,10:07:24 | INFO |   world_size: 1
-2022-12-21,10:07:24 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:07:24 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:07:24 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<09:58,  1.67it/s]
  1%|          | 7/1000 [00:00<01:19, 12.43it/s]
  1%|▏         | 13/1000 [00:00<00:45, 21.67it/s]
  2%|▏         | 19/1000 [00:00<00:32, 30.06it/s]
  2%|▎         | 25/1000 [00:01<00:26, 36.91it/s]
  3%|▎         | 31/1000 [00:01<00:22, 42.46it/s]
  4%|▎         | 37/1000 [00:01<00:20, 46.63it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.82it/s]
  5%|▍         | 49/1000 [00:01<00:18, 52.13it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.70it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.92it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.68it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.21it/s]
  8%|▊         | 79/1000 [00:01<00:16, 56.50it/s]
  8%|▊         | 85/1000 [00:02<00:16, 56.93it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.16it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.40it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.58it/s]
 11%|█         | 109/1000 [00:02<00:15, 57.82it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.01it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 57.93it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 57.93it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 57.93it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 57.90it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 57.95it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 57.99it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 57.75it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 57.79it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.58it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.60it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.62it/s]
 19%|█▉        | 193/1000 [00:03<00:14, 57.60it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.51it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.35it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.17it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.36it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.36it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.55it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.52it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.36it/s]
 25%|██▌       | 253/1000 [00:05<00:15, 49.55it/s]
 26%|██▌       | 259/1000 [00:05<00:14, 51.80it/s]
 26%|██▋       | 265/1000 [00:05<00:13, 53.43it/s]
 27%|██▋       | 271/1000 [00:05<00:13, 54.50it/s]
 28%|██▊       | 277/1000 [00:05<00:13, 55.50it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 56.27it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 56.84it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 57.30it/s]
 30%|███       | 301/1000 [00:05<00:12, 57.52it/s]
 31%|███       | 307/1000 [00:05<00:12, 57.61it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 57.91it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.01it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 57.89it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 57.78it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 57.94it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 57.97it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 57.91it/s]
 36%|███▌      | 361/1000 [00:06<00:11, 57.91it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 57.97it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 57.85it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.69it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.75it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 57.82it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.93it/s]
 41%|████      | 409/1000 [00:07<00:10, 57.99it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.08it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.09it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.21it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.17it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 57.91it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.05it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.10it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.02it/s]
 48%|████▊     | 475/1000 [00:08<00:09, 58.19it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.01it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.05it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 57.96it/s]
 50%|█████     | 505/1000 [00:09<00:08, 57.92it/s]
 51%|█████     | 511/1000 [00:09<00:08, 57.85it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.00it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 57.95it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.02it/s]
 54%|█████▎    | 535/1000 [00:09<00:08, 58.08it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.13it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.15it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.05it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.03it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 57.94it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.12it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.02it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.14it/s]
 61%|██████    | 607/1000 [00:11<00:06, 57.99it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.03it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.17it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.14it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.12it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.03it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.11it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.11it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.14it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.20it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.18it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.10it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.16it/s]
 71%|███████   | 709/1000 [00:12<00:05, 58.18it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 58.15it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.07it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.05it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.11it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.16it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.20it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.06it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.15it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.14it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.15it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.14it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 57.92it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 57.90it/s]
 80%|████████  | 805/1000 [00:14<00:03, 57.96it/s]
 81%|████████  | 811/1000 [00:14<00:03, 57.93it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 57.99it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.11it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.27it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.14it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.20it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.18it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.15it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.19it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.15it/s]
 89%|████████▉ | 889/1000 [00:16<00:01, 58.07it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.16it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.08it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.12it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.26it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.19it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.18it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.18it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.35it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.19it/s]
-2022-12-21,10:07:42 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:23, 112.65it/s]
  1%|          | 512/50176 [00:04<06:27, 128.24it/s]
  2%|▏         | 768/50176 [00:05<06:05, 135.17it/s]
  2%|▏         | 1024/50176 [00:07<05:54, 138.77it/s]
  3%|▎         | 1280/50176 [00:09<05:50, 139.32it/s]
  3%|▎         | 1536/50176 [00:11<06:04, 133.59it/s]
  4%|▎         | 1792/50176 [00:13<05:58, 134.91it/s]
  4%|▍         | 2048/50176 [00:15<05:48, 137.92it/s]
  5%|▍         | 2304/50176 [00:16<05:47, 137.89it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.17it/s]
  6%|▌         | 2816/50176 [00:20<05:39, 139.61it/s]
  6%|▌         | 3072/50176 [00:22<05:31, 141.99it/s]
  7%|▋         | 3328/50176 [00:24<05:27, 143.14it/s]
  7%|▋         | 3584/50176 [00:25<05:19, 145.78it/s]
  8%|▊         | 3840/50176 [00:27<05:29, 140.56it/s]
  8%|▊         | 4096/50176 [00:29<05:29, 139.97it/s]
  9%|▊         | 4352/50176 [00:31<05:22, 142.16it/s]
  9%|▉         | 4608/50176 [00:33<05:16, 143.83it/s]
 10%|▉         | 4864/50176 [00:34<05:08, 146.72it/s]
 10%|█         | 5120/50176 [00:36<05:05, 147.39it/s]
 11%|█         | 5376/50176 [00:38<05:04, 147.14it/s]
 11%|█         | 5632/50176 [00:39<04:59, 148.77it/s]
 12%|█▏        | 5888/50176 [00:41<04:58, 148.26it/s]
 12%|█▏        | 6144/50176 [00:43<05:22, 136.40it/s]
 13%|█▎        | 6400/50176 [00:45<05:11, 140.33it/s]
 13%|█▎        | 6656/50176 [00:47<05:05, 142.63it/s]
 14%|█▍        | 6912/50176 [00:49<05:02, 142.90it/s]
 14%|█▍        | 7168/50176 [00:50<04:57, 144.69it/s]
 15%|█▍        | 7424/50176 [00:52<04:50, 147.03it/s]
 15%|█▌        | 7680/50176 [00:54<04:49, 146.95it/s]
 16%|█▌        | 7936/50176 [00:55<04:39, 150.94it/s]
 16%|█▋        | 8192/50176 [00:57<04:38, 151.02it/s]
 17%|█▋        | 8448/50176 [00:59<04:35, 151.68it/s]
 17%|█▋        | 8704/50176 [01:00<04:31, 152.75it/s]
 18%|█▊        | 8960/50176 [01:02<04:40, 146.90it/s]
 18%|█▊        | 9216/50176 [01:04<04:32, 150.17it/s]
 19%|█▉        | 9472/50176 [01:06<04:31, 149.84it/s]
 19%|█▉        | 9728/50176 [01:07<04:36, 146.13it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.29it/s]
 20%|██        | 10240/50176 [01:11<04:28, 148.77it/s]
 21%|██        | 10496/50176 [01:12<04:23, 150.53it/s]
 21%|██▏       | 10752/50176 [01:14<04:32, 144.64it/s]
 22%|██▏       | 11008/50176 [01:16<04:26, 146.79it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.29it/s]
 23%|██▎       | 11520/50176 [01:19<04:16, 150.46it/s]
 23%|██▎       | 11776/50176 [01:21<04:15, 150.33it/s]
 24%|██▍       | 12032/50176 [01:23<04:12, 150.99it/s]
 24%|██▍       | 12288/50176 [01:24<04:08, 152.66it/s]
 25%|██▌       | 12544/50176 [01:26<04:04, 153.84it/s]
 26%|██▌       | 12800/50176 [01:28<04:02, 154.15it/s]
 26%|██▌       | 13056/50176 [01:29<03:59, 154.94it/s]
 27%|██▋       | 13312/50176 [01:31<04:02, 152.05it/s]
 27%|██▋       | 13568/50176 [01:33<04:10, 146.12it/s]
 28%|██▊       | 13824/50176 [01:35<04:14, 142.84it/s]
 28%|██▊       | 14080/50176 [01:37<04:43, 127.32it/s]
 29%|██▊       | 14336/50176 [01:39<04:37, 129.31it/s]
 29%|██▉       | 14592/50176 [01:41<04:19, 137.02it/s]
 30%|██▉       | 14848/50176 [01:43<04:28, 131.75it/s]
 30%|███       | 15104/50176 [01:45<04:25, 132.27it/s]
 31%|███       | 15360/50176 [01:47<04:15, 136.25it/s]
 31%|███       | 15616/50176 [01:48<04:08, 139.21it/s]
 32%|███▏      | 15872/50176 [01:50<04:11, 136.55it/s]
 32%|███▏      | 16128/50176 [01:52<04:04, 139.25it/s]
 33%|███▎      | 16384/50176 [01:54<04:02, 139.54it/s]
 33%|███▎      | 16640/50176 [01:56<04:08, 135.22it/s]
 34%|███▎      | 16896/50176 [01:58<03:56, 140.73it/s]
 34%|███▍      | 17152/50176 [01:59<03:49, 144.15it/s]
 35%|███▍      | 17408/50176 [02:01<03:44, 145.81it/s]
 35%|███▌      | 17664/50176 [02:03<03:40, 147.71it/s]
 36%|███▌      | 17920/50176 [02:04<03:35, 149.50it/s]
 36%|███▌      | 18176/50176 [02:06<03:35, 148.45it/s]
 37%|███▋      | 18432/50176 [02:08<03:32, 149.33it/s]
 37%|███▋      | 18688/50176 [02:10<03:31, 148.67it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.35it/s]
 38%|███▊      | 19200/50176 [02:14<03:49, 134.91it/s]
 39%|███▉      | 19456/50176 [02:16<03:51, 132.55it/s]
 39%|███▉      | 19712/50176 [02:18<03:49, 132.67it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.50it/s]
 40%|████      | 20224/50176 [02:21<03:31, 141.43it/s]
 41%|████      | 20480/50176 [02:23<03:25, 144.76it/s]
 41%|████▏     | 20736/50176 [02:24<03:23, 144.48it/s]
 42%|████▏     | 20992/50176 [02:26<03:17, 147.64it/s]
 42%|████▏     | 21248/50176 [02:28<03:22, 142.76it/s]
 43%|████▎     | 21504/50176 [02:30<03:13, 147.82it/s]
 43%|████▎     | 21760/50176 [02:31<03:11, 148.50it/s]
 44%|████▍     | 22016/50176 [02:33<03:06, 151.17it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.68it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.70it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.64it/s]
 46%|████▌     | 23040/50176 [02:40<03:07, 144.51it/s]
 46%|████▋     | 23296/50176 [02:42<03:10, 141.21it/s]
 47%|████▋     | 23552/50176 [02:44<03:13, 137.94it/s]
 47%|████▋     | 23808/50176 [02:46<03:05, 142.13it/s]
 48%|████▊     | 24064/50176 [02:47<03:02, 143.21it/s]
 48%|████▊     | 24320/50176 [02:49<03:00, 143.62it/s]
 49%|████▉     | 24576/50176 [02:51<02:55, 145.91it/s]
 49%|████▉     | 24832/50176 [02:52<02:49, 149.14it/s]
 50%|█████     | 25088/50176 [02:54<02:47, 149.67it/s]
 51%|█████     | 25344/50176 [02:56<02:45, 149.91it/s]
 51%|█████     | 25600/50176 [02:58<02:46, 147.47it/s]
 52%|█████▏    | 25856/50176 [02:59<02:40, 151.24it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.53it/s]
 53%|█████▎    | 26368/50176 [03:03<02:44, 144.71it/s]
 53%|█████▎    | 26624/50176 [03:05<02:40, 147.00it/s]
 54%|█████▎    | 26880/50176 [03:06<02:38, 146.92it/s]
 54%|█████▍    | 27136/50176 [03:08<02:35, 147.76it/s]
 55%|█████▍    | 27392/50176 [03:10<02:33, 148.18it/s]
 55%|█████▌    | 27648/50176 [03:11<02:25, 155.15it/s]
 56%|█████▌    | 27904/50176 [03:13<02:34, 143.76it/s]
 56%|█████▌    | 28160/50176 [03:15<02:30, 145.87it/s]
 57%|█████▋    | 28416/50176 [03:17<02:29, 145.51it/s]
 57%|█████▋    | 28672/50176 [03:19<02:31, 141.61it/s]
 58%|█████▊    | 28928/50176 [03:20<02:27, 143.97it/s]
 58%|█████▊    | 29184/50176 [03:22<02:23, 146.35it/s]
 59%|█████▊    | 29440/50176 [03:24<02:30, 138.07it/s]
 59%|█████▉    | 29696/50176 [03:26<02:30, 135.72it/s]
 60%|█████▉    | 29952/50176 [03:28<02:28, 135.95it/s]
 60%|██████    | 30208/50176 [03:30<02:24, 138.15it/s]
 61%|██████    | 30464/50176 [03:31<02:16, 144.55it/s]
 61%|██████    | 30720/50176 [03:33<02:15, 143.59it/s]
 62%|██████▏   | 30976/50176 [03:35<02:11, 145.79it/s]
 62%|██████▏   | 31232/50176 [03:37<02:08, 147.97it/s]
 63%|██████▎   | 31488/50176 [03:38<02:04, 150.17it/s]
 63%|██████▎   | 31744/50176 [03:40<02:02, 150.49it/s]
 64%|██████▍   | 32000/50176 [03:42<02:02, 148.65it/s]
 64%|██████▍   | 32256/50176 [03:43<01:59, 150.00it/s]
 65%|██████▍   | 32512/50176 [03:45<02:01, 144.89it/s]
 65%|██████▌   | 32768/50176 [03:47<01:57, 148.63it/s]
 66%|██████▌   | 33024/50176 [03:49<01:56, 147.06it/s]
 66%|██████▋   | 33280/50176 [03:50<01:53, 148.41it/s]
 67%|██████▋   | 33536/50176 [03:52<01:54, 145.07it/s]
 67%|██████▋   | 33792/50176 [03:54<01:50, 147.94it/s]
 68%|██████▊   | 34048/50176 [03:55<01:45, 152.52it/s]
 68%|██████▊   | 34304/50176 [03:57<01:42, 154.98it/s]
 69%|██████▉   | 34560/50176 [03:59<01:45, 147.82it/s]
 69%|██████▉   | 34816/50176 [04:01<01:44, 146.80it/s]
 70%|██████▉   | 35072/50176 [04:02<01:43, 145.60it/s]
 70%|███████   | 35328/50176 [04:04<01:40, 148.26it/s]
 71%|███████   | 35584/50176 [04:06<01:39, 147.12it/s]
 71%|███████▏  | 35840/50176 [04:08<01:41, 141.94it/s]
 72%|███████▏  | 36096/50176 [04:09<01:36, 145.50it/s]
 72%|███████▏  | 36352/50176 [04:11<01:37, 142.51it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 145.24it/s]
 73%|███████▎  | 36864/50176 [04:15<01:33, 142.88it/s]
 74%|███████▍  | 37120/50176 [04:17<01:30, 143.58it/s]
 74%|███████▍  | 37376/50176 [04:18<01:25, 149.58it/s]
 75%|███████▌  | 37632/50176 [04:20<01:22, 151.35it/s]
 76%|███████▌  | 37888/50176 [04:22<01:22, 148.67it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 149.95it/s]
 77%|███████▋  | 38400/50176 [04:25<01:19, 148.96it/s]
 77%|███████▋  | 38656/50176 [04:27<01:18, 147.25it/s]
 78%|███████▊  | 38912/50176 [04:29<01:17, 144.62it/s]
 78%|███████▊  | 39168/50176 [04:31<01:17, 141.47it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 143.55it/s]
 79%|███████▉  | 39680/50176 [04:34<01:12, 145.19it/s]
 80%|███████▉  | 39936/50176 [04:36<01:09, 146.99it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.67it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.60it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 138.73it/s]
 82%|████████▏ | 40960/50176 [04:43<01:04, 143.36it/s]
 82%|████████▏ | 41216/50176 [04:45<01:00, 146.94it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.76it/s]
 83%|████████▎ | 41728/50176 [04:48<00:55, 152.17it/s]
 84%|████████▎ | 41984/50176 [04:50<00:53, 152.52it/s]
 84%|███��████▍ | 42240/50176 [04:51<00:53, 148.94it/s]
 85%|████████▍ | 42496/50176 [04:53<00:51, 150.07it/s]
 85%|████████▌ | 42752/50176 [04:55<00:49, 149.20it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 151.27it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.29it/s]
 87%|████████▋ | 43520/50176 [05:00<00:44, 149.26it/s]
 87%|████████▋ | 43776/50176 [05:02<00:43, 146.34it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 148.19it/s]
 88%|████████▊ | 44288/50176 [05:05<00:39, 150.35it/s]
 89%|████████▉ | 44544/50176 [05:07<00:38, 147.31it/s]
 89%|████████▉ | 44800/50176 [05:09<00:36, 145.61it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 147.46it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 140.64it/s]
 91%|█████████ | 45568/50176 [05:14<00:31, 145.46it/s]
 91%|█████████▏| 45824/50176 [05:16<00:29, 146.78it/s]
 92%|█████████▏| 46080/50176 [05:17<00:27, 148.18it/s]
 92%|█████████▏| 46336/50176 [05:19<00:25, 149.45it/s]
 93%|█████████▎| 46592/50176 [05:21<00:23, 149.68it/s]
 93%|█████████▎| 46848/50176 [05:23<00:22, 148.37it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 143.93it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 145.41it/s]
 95%|█████████▍| 47616/50176 [05:28<00:17, 142.98it/s]
 95%|█████████▌| 47872/50176 [05:30<00:15, 145.03it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 146.96it/s]
 96%|█████████▋| 48384/50176 [05:33<00:12, 148.23it/s]
 97%|█████████▋| 48640/50176 [05:35<00:10, 149.93it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 150.16it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 146.40it/s]
 98%|█████████▊| 49408/50176 [05:40<00:05, 144.69it/s]
 99%|█████████▉| 49664/50176 [05:42<00:03, 145.43it/s]
 99%|█████████▉| 49920/50176 [05:44<00:01, 137.97it/s]
-2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5733	imagenet-zeroshot-val-top5: 0.8413

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_26.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt).
-2022-12-21,10:07:25 | INFO | Model:
-2022-12-21,10:07:25 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:07:25 | INFO | Params:
-2022-12-21,10:07:25 | INFO |   batch_size: 256
-2022-12-21,10:07:25 | INFO |   beta1: 0.9
-2022-12-21,10:07:25 | INFO |   beta2: 0.98
-2022-12-21,10:07:25 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/checkpoints
-2022-12-21,10:07:25 | INFO |   copy_codebase: False
-2022-12-21,10:07:25 | INFO |   csv_caption_key: title
-2022-12-21,10:07:25 | INFO |   csv_img_key: filepath
-2022-12-21,10:07:25 | INFO |   csv_separator: 	
-2022-12-21,10:07:25 | INFO |   dataset_resampled: False
-2022-12-21,10:07:25 | INFO |   dataset_type: auto
-2022-12-21,10:07:25 | INFO |   ddp_static_graph: False
-2022-12-21,10:07:25 | INFO |   debug: False
-2022-12-21,10:07:25 | INFO |   device: cuda:0
-2022-12-21,10:07:25 | INFO |   dist_backend: nccl
-2022-12-21,10:07:25 | INFO |   dist_url: env://
-2022-12-21,10:07:25 | INFO |   distributed: False
-2022-12-21,10:07:25 | INFO |   epochs: 32
-2022-12-21,10:07:25 | INFO |   eps: 1e-06
-2022-12-21,10:07:25 | INFO |   force_quick_gelu: False
-2022-12-21,10:07:25 | INFO |   gather_with_grad: False
-2022-12-21,10:07:25 | INFO |   grad_checkpointing: False
-2022-12-21,10:07:25 | INFO |   horovod: False
-2022-12-21,10:07:25 | INFO |   image_mean: None
-2022-12-21,10:07:25 | INFO |   image_std: None
-2022-12-21,10:07:25 | INFO |   imagenet_v2: None
-2022-12-21,10:07:25 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:07:25 | INFO |   local_loss: False
-2022-12-21,10:07:25 | INFO |   local_rank: 0
-2022-12-21,10:07:25 | INFO |   lock_image: False
-2022-12-21,10:07:25 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:07:25 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:07:25 | INFO |   log_level: 20
-2022-12-21,10:07:25 | INFO |   log_local: False
-2022-12-21,10:07:25 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_26.pt/out.log
-2022-12-21,10:07:25 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:07:25 | INFO |   lr: 0.0005
-2022-12-21,10:07:25 | INFO |   model: ViT-B-16
-2022-12-21,10:07:25 | INFO |   name: epoch_26.pt
-2022-12-21,10:07:25 | INFO |   no_set_device_rank: False
-2022-12-21,10:07:25 | INFO |   norm_gradient_clip: None
-2022-12-21,10:07:25 | INFO |   precision: amp_bfloat16
-2022-12-21,10:07:25 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_26.pt
-2022-12-21,10:07:25 | INFO |   pretrained_image: False
-2022-12-21,10:07:25 | INFO |   rank: 0
-2022-12-21,10:07:25 | INFO |   report_to: 
-2022-12-21,10:07:25 | INFO |   resume: None
-2022-12-21,10:07:25 | INFO |   save_frequency: 1
-2022-12-21,10:07:25 | INFO |   save_most_recent: False
-2022-12-21,10:07:25 | INFO |   seed: 0
-2022-12-21,10:07:25 | INFO |   skip_scheduler: False
-2022-12-21,10:07:25 | INFO |   tensorboard: False
-2022-12-21,10:07:25 | INFO |   tensorboard_path: 
-2022-12-21,10:07:25 | INFO |   torchscript: False
-2022-12-21,10:07:25 | INFO |   trace: False
-2022-12-21,10:07:25 | INFO |   train_data: None
-2022-12-21,10:07:25 | INFO |   train_num_samples: None
-2022-12-21,10:07:25 | INFO |   use_bn_sync: False
-2022-12-21,10:07:25 | INFO |   val_data: None
-2022-12-21,10:07:25 | INFO |   val_frequency: 1
-2022-12-21,10:07:25 | INFO |   val_num_samples: None
-2022-12-21,10:07:25 | INFO |   wandb: False
-2022-12-21,10:07:25 | INFO |   wandb_notes: 
-2022-12-21,10:07:25 | INFO |   warmup: 10000
-2022-12-21,10:07:25 | INFO |   wd: 0.2
-2022-12-21,10:07:25 | INFO |   workers: 1
-2022-12-21,10:07:25 | INFO |   world_size: 1
-2022-12-21,10:07:25 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:07:25 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:38,  1.22it/s]
  1%|          | 6/1000 [00:00<01:57,  8.45it/s]
  1%|          | 12/1000 [00:01<00:58, 16.90it/s]
  2%|▏         | 18/1000 [00:01<00:39, 24.95it/s]
  2%|▏         | 24/1000 [00:01<00:30, 32.18it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.29it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.07it/s]
  4%|▍         | 42/1000 [00:01<00:20, 46.86it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.66it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.80it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.33it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.29it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.89it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.48it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.81it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.18it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.37it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.66it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.75it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.55it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.51it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.61it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.58it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.64it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.73it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.80it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.71it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.61it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.59it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.51it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.38it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.36it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.33it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.16it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.08it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.19it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.15it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.37it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.32it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.19it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.30it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.39it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.47it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.53it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.38it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.43it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.56it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.67it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.85it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.79it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.75it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.89it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.91it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.79it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.60it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.71it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.81it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.89it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.68it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.72it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.82it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.90it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.78it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.54it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.78it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.81it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.81it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.82it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.85it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.95it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.94it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.72it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.79it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.86it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.82it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.78it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.92it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.85it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.86it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.85it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.78it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.81it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.69it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.77it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.79it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.88it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.76it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.87it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.87it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.78it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.77it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.85it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.75it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.72it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.87it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.85it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.74it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.88it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.76it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.77it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.89it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.90it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.96it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.80it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.74it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.89it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.82it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.89it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.85it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.74it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.83it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.88it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.85it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.86it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.88it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.90it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.93it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.86it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.89it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.93it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.91it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.00it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.03it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.87it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.97it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.95it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.00it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.04it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.80it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.81it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.87it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.84it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.79it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.81it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.94it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.10it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.14it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.05it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.07it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.02it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.03it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.95it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 58.01it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.00it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.87it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.99it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.98it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.89it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.92it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.93it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.92it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.99it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 58.02it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.02it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.04it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.03it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.93it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.97it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.03it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.04it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.05it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.03it/s]
-2022-12-21,10:07:43 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:04, 137.14it/s]
  1%|          | 512/50176 [00:03<05:12, 159.02it/s]
  2%|▏         | 768/50176 [00:04<04:46, 172.60it/s]
  2%|▏         | 1024/50176 [00:06<05:01, 162.76it/s]
  3%|▎         | 1280/50176 [00:08<05:17, 153.97it/s]
  3%|▎         | 1536/50176 [00:10<05:41, 142.25it/s]
  4%|▎         | 1792/50176 [00:12<05:44, 140.27it/s]
  4%|▍         | 2048/50176 [00:13<05:39, 141.58it/s]
  5%|▍         | 2304/50176 [00:15<05:39, 141.05it/s]
  5%|▌         | 2560/50176 [00:17<05:34, 142.29it/s]
  6%|▌         | 2816/50176 [00:19<05:36, 140.83it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 143.00it/s]
  7%|▋         | 3328/50176 [00:22<05:25, 144.00it/s]
  7%|▋         | 3584/50176 [00:24<05:18, 146.46it/s]
  8%|▊         | 3840/50176 [00:26<05:29, 140.56it/s]
  8%|▊         | 4096/50176 [00:28<05:27, 140.52it/s]
  9%|▊         | 4352/50176 [00:30<05:21, 142.48it/s]
  9%|▉         | 4608/50176 [00:31<05:16, 144.18it/s]
 10%|▉         | 4864/50176 [00:33<05:09, 146.57it/s]
 10%|█         | 5120/50176 [00:35<05:05, 147.34it/s]
 11%|█         | 5376/50176 [00:36<05:03, 147.79it/s]
 11%|█         | 5632/50176 [00:38<04:59, 148.65it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.42it/s]
 12%|█▏        | 6144/50176 [00:42<05:22, 136.36it/s]
 13%|█▎        | 6400/50176 [00:44<05:12, 140.26it/s]
 13%|█▎        | 6656/50176 [00:45<05:06, 142.01it/s]
 14%|█▍        | 6912/50176 [00:47<05:02, 143.02it/s]
 14%|█▍        | 7168/50176 [00:49<04:57, 144.79it/s]
 15%|█▍        | 7424/50176 [00:51<04:50, 147.08it/s]
 15%|█▌        | 7680/50176 [00:52<04:50, 146.41it/s]
 16%|█▌        | 7936/50176 [00:54<04:39, 151.24it/s]
 16%|█▋        | 8192/50176 [00:56<04:37, 151.13it/s]
 17%|█▋        | 8448/50176 [00:57<04:34, 151.77it/s]
 17%|█▋        | 8704/50176 [00:59<04:31, 152.81it/s]
 18%|█▊        | 8960/50176 [01:01<04:40, 147.11it/s]
 18%|█▊        | 9216/50176 [01:03<04:32, 150.21it/s]
 19%|█▉        | 9472/50176 [01:04<04:31, 149.89it/s]
 19%|█▉        | 9728/50176 [01:06<04:36, 146.15it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 148.15it/s]
 20%|██        | 10240/50176 [01:09<04:27, 149.34it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.00it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.02it/s]
 22%|██▏       | 11008/50176 [01:15<04:26, 146.73it/s]
 22%|██▏       | 11264/50176 [01:16<04:22, 148.28it/s]
 23%|██▎       | 11520/50176 [01:18<04:17, 150.27it/s]
 23%|██▎       | 11776/50176 [01:20<04:15, 150.22it/s]
 24%|██▍       | 12032/50176 [01:21<04:12, 150.93it/s]
 24%|██▍       | 12288/50176 [01:23<04:08, 152.63it/s]
 25%|██▌       | 12544/50176 [01:25<04:04, 153.79it/s]
 26%|██▌       | 12800/50176 [01:26<04:02, 154.08it/s]
 26%|██▌       | 13056/50176 [01:28<03:59, 155.08it/s]
 27%|██▋       | 13312/50176 [01:30<04:02, 152.08it/s]
 27%|██▋       | 13568/50176 [01:32<04:11, 145.63it/s]
 28%|██▊       | 13824/50176 [01:34<04:14, 142.78it/s]
 28%|██▊       | 14080/50176 [01:36<04:43, 127.48it/s]
 29%|██▊       | 14336/50176 [01:38<04:37, 129.36it/s]
 29%|██▉       | 14592/50176 [01:40<04:19, 137.14it/s]
 30%|██▉       | 14848/50176 [01:42<04:28, 131.80it/s]
 30%|███       | 15104/50176 [01:44<04:25, 132.03it/s]
 31%|███       | 15360/50176 [01:45<04:15, 136.27it/s]
 31%|███       | 15616/50176 [01:47<04:08, 139.11it/s]
 32%|███▏      | 15872/50176 [01:49<04:11, 136.21it/s]
 32%|███▏      | 16128/50176 [01:51<04:04, 139.31it/s]
 33%|███▎      | 16384/50176 [01:53<04:01, 139.82it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.48it/s]
 34%|███▎      | 16896/50176 [01:56<03:56, 140.78it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.16it/s]
 35%|███▍      | 17408/50176 [02:00<03:44, 145.80it/s]
 35%|███▌      | 17664/50176 [02:01<03:40, 147.62it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.56it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.85it/s]
 37%|███▋      | 18432/50176 [02:06<03:32, 149.56it/s]
 37%|███▋      | 18688/50176 [02:08<03:31, 148.63it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.59it/s]
 38%|███▊      | 19200/50176 [02:12<03:49, 135.11it/s]
 39%|███▉      | 19456/50176 [02:14<03:52, 132.19it/s]
 39%|███▉      | 19712/50176 [02:16<03:49, 132.63it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.68it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.52it/s]
 41%|████      | 20480/50176 [02:21<03:24, 144.92it/s]
 41%|████▏     | 20736/50176 [02:23<03:23, 145.01it/s]
 42%|████▏     | 20992/50176 [02:25<03:17, 147.52it/s]
 42%|████▏     | 21248/50176 [02:27<03:22, 142.71it/s]
 43%|████▎     | 21504/50176 [02:28<03:14, 147.68it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.42it/s]
 44%|████▍     | 22016/50176 [02:32<03:06, 151.09it/s]
 44%|████▍     | 22272/50176 [02:33<03:05, 150.58it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.69it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.28it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 144.17it/s]
 46%|████▋     | 23296/50176 [02:41<03:10, 140.84it/s]
 47%|████▋     | 23552/50176 [02:43<03:13, 137.55it/s]
 47%|████▋     | 23808/50176 [02:44<03:05, 141.89it/s]
 48%|████▊     | 24064/50176 [02:46<03:02, 143.21it/s]
 48%|████▊     | 24320/50176 [02:48<03:00, 143.56it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 145.92it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 149.02it/s]
 50%|█████     | 25088/50176 [02:53<02:47, 149.70it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.20it/s]
 51%|█████     | 25600/50176 [02:56<02:46, 147.72it/s]
 52%|█████▏    | 25856/50176 [02:58<02:40, 151.32it/s]
 52%|█████▏    | 26112/50176 [03:00<02:41, 148.55it/s]
 53%|█████▎    | 26368/50176 [03:02<02:44, 144.87it/s]
 53%|█████▎    | 26624/50176 [03:03<02:40, 146.99it/s]
 54%|█████▎    | 26880/50176 [03:05<02:38, 146.85it/s]
 54%|█████▍    | 27136/50176 [03:07<02:36, 147.62it/s]
 55%|█████▍    | 27392/50176 [03:08<02:33, 148.10it/s]
 55%|█████▌    | 27648/50176 [03:10<02:25, 155.31it/s]
 56%|█████▌    | 27904/50176 [03:12<02:34, 143.83it/s]
 56%|█████▌    | 28160/50176 [03:14<02:31, 145.46it/s]
 57%|█████▋    | 28416/50176 [03:15<02:29, 145.75it/s]
 57%|█████▋    | 28672/50176 [03:17<02:31, 141.84it/s]
 58%|█████▊    | 28928/50176 [03:19<02:27, 144.01it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 146.45it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.21it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 135.81it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.70it/s]
 60%|██████    | 30208/50176 [03:29<02:25, 137.68it/s]
 61%|██████    | 30464/50176 [03:30<02:16, 144.54it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 143.59it/s]
 62%|██████▏   | 30976/50176 [03:34<02:11, 145.77it/s]
 62%|██████▏   | 31232/50176 [03:35<02:08, 147.33it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.33it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.68it/s]
 64%|██████▍   | 32000/50176 [03:40<02:02, 148.73it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 150.04it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 144.89it/s]
 65%|██████▌   | 32768/50176 [03:46<01:57, 148.76it/s]
 66%|██████▌   | 33024/50176 [03:47<01:56, 146.90it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 148.33it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 144.96it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.86it/s]
 68%|██████▊   | 34048/50176 [03:54<01:46, 151.92it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 155.35it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.08it/s]
 69%|██████▉   | 34816/50176 [03:59<01:44, 147.16it/s]
 70%|██████▉   | 35072/50176 [04:01<01:43, 146.17it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 148.81it/s]
 71%|███████   | 35584/50176 [04:05<01:39, 147.19it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.27it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.49it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 142.80it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.35it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 142.62it/s]
 74%|███████▍  | 37120/50176 [04:15<01:30, 143.66it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 149.30it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.30it/s]
 76%|███████▌  | 37888/50176 [04:20<01:22, 148.62it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.88it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.26it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.42it/s]
 78%|███████▊  | 38912/50176 [04:27<01:17, 144.81it/s]
 78%|███████▊  | 39168/50176 [04:29<01:18, 140.93it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.78it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 144.99it/s]
 80%|███████▉  | 39936/50176 [04:34<01:09, 147.25it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.86it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.70it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.79it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.16it/s]
 82%|████████▏ | 41216/50176 [04:43<01:01, 146.58it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 149.01it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.90it/s]
 84%|████████▎ | 41984/50176 [04:48<00:53, 152.30it/s]
 84%|██████���█▍ | 42240/50176 [04:50<00:53, 148.84it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 150.11it/s]
 85%|████████▌ | 42752/50176 [04:53<00:49, 149.17it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 151.15it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.80it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.91it/s]
 87%|████████▋ | 43776/50176 [05:00<00:43, 146.33it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.10it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.32it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.24it/s]
 89%|████████▉ | 44800/50176 [05:07<00:37, 145.15it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.08it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 141.07it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.50it/s]
 91%|█████████▏| 45824/50176 [05:14<00:29, 147.56it/s]
 92%|█████████▏| 46080/50176 [05:16<00:27, 148.01it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.39it/s]
 93%|█████████▎| 46592/50176 [05:19<00:23, 149.63it/s]
 93%|█████████▎| 46848/50176 [05:21<00:22, 148.11it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 143.32it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 145.43it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.05it/s]
 95%|█████████▌| 47872/50176 [05:28<00:15, 145.06it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 146.91it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.13it/s]
 97%|█████████▋| 48640/50176 [05:33<00:10, 149.89it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 150.13it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 146.39it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 144.72it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.44it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.23it/s]
-2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5748	imagenet-zeroshot-val-top5: 0.8455

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_27.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:07:15 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:07:15 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:07:16 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt).
-2022-12-21,10:07:25 | INFO | Model:
-2022-12-21,10:07:25 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:07:25 | INFO | Params:
-2022-12-21,10:07:25 | INFO |   batch_size: 256
-2022-12-21,10:07:25 | INFO |   beta1: 0.9
-2022-12-21,10:07:25 | INFO |   beta2: 0.98
-2022-12-21,10:07:25 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/checkpoints
-2022-12-21,10:07:25 | INFO |   copy_codebase: False
-2022-12-21,10:07:25 | INFO |   csv_caption_key: title
-2022-12-21,10:07:25 | INFO |   csv_img_key: filepath
-2022-12-21,10:07:25 | INFO |   csv_separator: 	
-2022-12-21,10:07:25 | INFO |   dataset_resampled: False
-2022-12-21,10:07:25 | INFO |   dataset_type: auto
-2022-12-21,10:07:25 | INFO |   ddp_static_graph: False
-2022-12-21,10:07:25 | INFO |   debug: False
-2022-12-21,10:07:25 | INFO |   device: cuda:0
-2022-12-21,10:07:25 | INFO |   dist_backend: nccl
-2022-12-21,10:07:25 | INFO |   dist_url: env://
-2022-12-21,10:07:25 | INFO |   distributed: False
-2022-12-21,10:07:25 | INFO |   epochs: 32
-2022-12-21,10:07:25 | INFO |   eps: 1e-06
-2022-12-21,10:07:25 | INFO |   force_quick_gelu: False
-2022-12-21,10:07:25 | INFO |   gather_with_grad: False
-2022-12-21,10:07:25 | INFO |   grad_checkpointing: False
-2022-12-21,10:07:25 | INFO |   horovod: False
-2022-12-21,10:07:25 | INFO |   image_mean: None
-2022-12-21,10:07:25 | INFO |   image_std: None
-2022-12-21,10:07:25 | INFO |   imagenet_v2: None
-2022-12-21,10:07:25 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:07:25 | INFO |   local_loss: False
-2022-12-21,10:07:25 | INFO |   local_rank: 0
-2022-12-21,10:07:25 | INFO |   lock_image: False
-2022-12-21,10:07:25 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:07:25 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:07:25 | INFO |   log_level: 20
-2022-12-21,10:07:25 | INFO |   log_local: False
-2022-12-21,10:07:25 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_27.pt/out.log
-2022-12-21,10:07:25 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:07:25 | INFO |   lr: 0.0005
-2022-12-21,10:07:25 | INFO |   model: ViT-B-16
-2022-12-21,10:07:25 | INFO |   name: epoch_27.pt
-2022-12-21,10:07:25 | INFO |   no_set_device_rank: False
-2022-12-21,10:07:25 | INFO |   norm_gradient_clip: None
-2022-12-21,10:07:25 | INFO |   precision: amp_bfloat16
-2022-12-21,10:07:25 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_27.pt
-2022-12-21,10:07:25 | INFO |   pretrained_image: False
-2022-12-21,10:07:25 | INFO |   rank: 0
-2022-12-21,10:07:25 | INFO |   report_to: 
-2022-12-21,10:07:25 | INFO |   resume: None
-2022-12-21,10:07:25 | INFO |   save_frequency: 1
-2022-12-21,10:07:25 | INFO |   save_most_recent: False
-2022-12-21,10:07:25 | INFO |   seed: 0
-2022-12-21,10:07:25 | INFO |   skip_scheduler: False
-2022-12-21,10:07:25 | INFO |   tensorboard: False
-2022-12-21,10:07:25 | INFO |   tensorboard_path: 
-2022-12-21,10:07:25 | INFO |   torchscript: False
-2022-12-21,10:07:25 | INFO |   trace: False
-2022-12-21,10:07:25 | INFO |   train_data: None
-2022-12-21,10:07:25 | INFO |   train_num_samples: None
-2022-12-21,10:07:25 | INFO |   use_bn_sync: False
-2022-12-21,10:07:25 | INFO |   val_data: None
-2022-12-21,10:07:25 | INFO |   val_frequency: 1
-2022-12-21,10:07:25 | INFO |   val_num_samples: None
-2022-12-21,10:07:25 | INFO |   wandb: False
-2022-12-21,10:07:25 | INFO |   wandb_notes: 
-2022-12-21,10:07:25 | INFO |   warmup: 10000
-2022-12-21,10:07:25 | INFO |   wd: 0.2
-2022-12-21,10:07:25 | INFO |   workers: 1
-2022-12-21,10:07:25 | INFO |   world_size: 1
-2022-12-21,10:07:25 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:07:25 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:07:25 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:04,  1.50it/s]
  1%|          | 6/1000 [00:00<01:38, 10.05it/s]
  1%|          | 12/1000 [00:00<00:51, 19.37it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.75it/s]
  2%|▏         | 24/1000 [00:01<00:27, 34.91it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.65it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.03it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.37it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.01it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.97it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.37it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.05it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.69it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.08it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.58it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.88it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.25it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.44it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.70it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.79it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.55it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.52it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.10it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.22it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.19it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.40it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.20it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.20it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.20it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.20it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.27it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 56.77it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 56.91it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 56.63it/s]
 20%|██        | 204/1000 [00:04<00:14, 56.62it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.43it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.71it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.49it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 56.83it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 56.96it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.55it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 56.77it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 54.99it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 55.80it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 56.32it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 56.51it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 56.80it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.17it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.41it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.71it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.75it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.76it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.95it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.95it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.86it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.67it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.75it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.83it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.94it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.73it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.75it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.82it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.91it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.80it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.48it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.56it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.84it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.93it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.92it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.93it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.96it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.08it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.07it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.85it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.91it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.93it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.92it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.88it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.03it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.86it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.86it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.86it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.78it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.84it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.75it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.82it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.87it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.96it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.89it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.00it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.01it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.91it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.92it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.97it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.86it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.80it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.94it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.92it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.84it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.95it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.85it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.87it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.00it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.99it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.07it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.87it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.81it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.96it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.91it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.01it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.05it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.99it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.05it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.05it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.07it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.96it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.06it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.06it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.05it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.06it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.02it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.04it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.02it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.09it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.17it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.00it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.10it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.05it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.08it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.10it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.89it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.86it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.92it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.89it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.87it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.90it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.01it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.15it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.19it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.10it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.14it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.09it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.10it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.09it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.08it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.96it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.06it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.06it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.03it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.03it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.00it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.10it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.14it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.12it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.87it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.81it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.63it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.65it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.68it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.70it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.64it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.62it/s]
-2022-12-21,10:07:43 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:01, 138.14it/s]
  1%|          | 512/50176 [00:03<05:13, 158.18it/s]
  2%|▏         | 768/50176 [00:04<04:59, 165.03it/s]
  2%|▏         | 1024/50176 [00:06<05:16, 155.36it/s]
  3%|▎         | 1280/50176 [00:08<05:28, 148.83it/s]
  3%|▎         | 1536/50176 [00:10<05:46, 140.24it/s]
  4%|▎         | 1792/50176 [00:12<05:47, 139.04it/s]
  4%|▍         | 2048/50176 [00:14<05:43, 140.19it/s]
  5%|▍         | 2304/50176 [00:15<05:41, 140.29it/s]
  5%|▌         | 2560/50176 [00:17<05:35, 141.90it/s]
  6%|▌         | 2816/50176 [00:19<05:36, 140.90it/s]
  6%|▌         | 3072/50176 [00:21<05:30, 142.53it/s]
  7%|▋         | 3328/50176 [00:23<05:27, 143.11it/s]
  7%|▋         | 3584/50176 [00:24<05:18, 146.28it/s]
  8%|▊         | 3840/50176 [00:26<05:29, 140.59it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.38it/s]
  9%|▊         | 4352/50176 [00:30<05:21, 142.37it/s]
  9%|▉         | 4608/50176 [00:32<05:16, 143.86it/s]
 10%|▉         | 4864/50176 [00:33<05:08, 146.81it/s]
 10%|█         | 5120/50176 [00:35<05:04, 147.77it/s]
 11%|█         | 5376/50176 [00:37<05:02, 147.92it/s]
 11%|█         | 5632/50176 [00:38<04:59, 148.68it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.34it/s]
 12%|█▏        | 6144/50176 [00:42<05:23, 136.22it/s]
 13%|█▎        | 6400/50176 [00:44<05:12, 140.23it/s]
 13%|█▎        | 6656/50176 [00:46<05:05, 142.28it/s]
 14%|█▍        | 6912/50176 [00:47<05:02, 142.94it/s]
 14%|█▍        | 7168/50176 [00:49<04:57, 144.64it/s]
 15%|█▍        | 7424/50176 [00:51<04:50, 146.97it/s]
 15%|█▌        | 7680/50176 [00:53<04:49, 146.95it/s]
 16%|█▌        | 7936/50176 [00:54<04:39, 151.00it/s]
 16%|█▋        | 8192/50176 [00:56<04:38, 151.02it/s]
 17%|█▋        | 8448/50176 [00:58<04:35, 151.64it/s]
 17%|█▋        | 8704/50176 [00:59<04:31, 152.81it/s]
 18%|█▊        | 8960/50176 [01:01<04:40, 146.91it/s]
 18%|█▊        | 9216/50176 [01:03<04:32, 150.27it/s]
 19%|█▉        | 9472/50176 [01:04<04:32, 149.51it/s]
 19%|█▉        | 9728/50176 [01:06<04:36, 146.27it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 148.28it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.68it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.08it/s]
 21%|██▏       | 10752/50176 [01:13<04:32, 144.73it/s]
 22%|██▏       | 11008/50176 [01:15<04:26, 146.71it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.38it/s]
 23%|██▎       | 11520/50176 [01:18<04:17, 150.35it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.56it/s]
 24%|██▍       | 12032/50176 [01:22<04:12, 151.12it/s]
 24%|██▍       | 12288/50176 [01:23<04:08, 152.76it/s]
 25%|██▌       | 12544/50176 [01:25<04:04, 153.81it/s]
 26%|██▌       | 12800/50176 [01:27<04:02, 154.11it/s]
 26%|██▌       | 13056/50176 [01:28<03:59, 155.13it/s]
 27%|██▋       | 13312/50176 [01:30<04:02, 152.22it/s]
 27%|██▋       | 13568/50176 [01:32<04:12, 145.19it/s]
 28%|██▊       | 13824/50176 [01:34<04:14, 142.97it/s]
 28%|██▊       | 14080/50176 [01:36<04:43, 127.49it/s]
 29%|██▊       | 14336/50176 [01:38<04:36, 129.44it/s]
 29%|██▉       | 14592/50176 [01:40<04:19, 137.13it/s]
 30%|██▉       | 14848/50176 [01:42<04:28, 131.78it/s]
 30%|███       | 15104/50176 [01:44<04:25, 132.00it/s]
 31%|███       | 15360/50176 [01:46<04:15, 136.44it/s]
 31%|███       | 15616/50176 [01:47<04:08, 139.19it/s]
 32%|███▏      | 15872/50176 [01:49<04:11, 136.62it/s]
 32%|███▏      | 16128/50176 [01:51<04:04, 139.34it/s]
 33%|███▎      | 16384/50176 [01:53<04:02, 139.58it/s]
 33%|███▎      | 16640/50176 [01:55<04:07, 135.28it/s]
 34%|███▎      | 16896/50176 [01:57<03:56, 140.69it/s]
 34%|███▍      | 17152/50176 [01:58<03:49, 144.10it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.18it/s]
 35%|███▌      | 17664/50176 [02:02<03:39, 148.05it/s]
 36%|███▌      | 17920/50176 [02:03<03:35, 149.58it/s]
 36%|███▌      | 18176/50176 [02:05<03:35, 148.17it/s]
 37%|███▋      | 18432/50176 [02:07<03:32, 149.51it/s]
 37%|███▋      | 18688/50176 [02:08<03:31, 148.91it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.43it/s]
 38%|███▊      | 19200/50176 [02:13<03:49, 134.92it/s]
 39%|███▉      | 19456/50176 [02:15<03:51, 132.57it/s]
 39%|███▉      | 19712/50176 [02:17<03:49, 132.50it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.58it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.54it/s]
 41%|████      | 20480/50176 [02:22<03:25, 144.75it/s]
 41%|████▏     | 20736/50176 [02:23<03:22, 145.03it/s]
 42%|████▏     | 20992/50176 [02:25<03:17, 147.53it/s]
 42%|████▏     | 21248/50176 [02:27<03:22, 142.65it/s]
 43%|████▎     | 21504/50176 [02:29<03:14, 147.72it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.37it/s]
 44%|████▍     | 22016/50176 [02:32<03:06, 151.11it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.70it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.63it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.26it/s]
 46%|████▌     | 23040/50176 [02:39<03:07, 144.67it/s]
 46%|████▋     | 23296/50176 [02:41<03:09, 141.51it/s]
 47%|████▋     | 23552/50176 [02:43<03:12, 138.31it/s]
 47%|████▋     | 23808/50176 [02:45<03:05, 142.19it/s]
 48%|████▊     | 24064/50176 [02:46<03:01, 143.58it/s]
 48%|████▊     | 24320/50176 [02:48<02:59, 143.83it/s]
 49%|████▉     | 24576/50176 [02:50<02:55, 146.08it/s]
 49%|████▉     | 24832/50176 [02:51<02:49, 149.13it/s]
 50%|█████     | 25088/50176 [02:53<02:47, 149.57it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 150.01it/s]
 51%|█████     | 25600/50176 [02:57<02:46, 147.62it/s]
 52%|█████▏    | 25856/50176 [02:58<02:40, 151.27it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.38it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.26it/s]
 53%|█████▎    | 26624/50176 [03:03<02:40, 147.17it/s]
 54%|█████▎    | 26880/50176 [03:05<02:38, 146.99it/s]
 54%|█████▍    | 27136/50176 [03:07<02:35, 147.79it/s]
 55%|█████▍    | 27392/50176 [03:09<02:33, 148.10it/s]
 55%|█████▌    | 27648/50176 [03:10<02:25, 154.88it/s]
 56%|█████▌    | 27904/50176 [03:12<02:34, 144.14it/s]
 56%|█████▌    | 28160/50176 [03:14<02:30, 145.82it/s]
 57%|█████▋    | 28416/50176 [03:16<02:29, 145.71it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 141.20it/s]
 58%|█████▊    | 28928/50176 [03:19<02:27, 144.19it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 145.91it/s]
 59%|█████▊    | 29440/50176 [03:23<02:29, 138.35it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.04it/s]
 60%|█████▉    | 29952/50176 [03:27<02:28, 135.86it/s]
 60%|██████    | 30208/50176 [03:29<02:24, 138.17it/s]
 61%|██████    | 30464/50176 [03:30<02:16, 144.49it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 143.55it/s]
 62%|██████▏   | 30976/50176 [03:34<02:11, 145.75it/s]
 62%|██████▏   | 31232/50176 [03:35<02:08, 147.93it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.14it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.53it/s]
 64%|██████▍   | 32000/50176 [03:41<02:02, 148.62it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 149.90it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 144.96it/s]
 65%|██████▌   | 32768/50176 [03:46<01:57, 148.40it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 146.97it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 148.35it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 144.99it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.88it/s]
 68%|██████▊   | 34048/50176 [03:54<01:45, 152.48it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 155.15it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.16it/s]
 69%|██████▉   | 34816/50176 [04:00<01:44, 146.60it/s]
 70%|██████▉   | 35072/50176 [04:01<01:43, 146.28it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 148.82it/s]
 71%|███████   | 35584/50176 [04:05<01:39, 147.21it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.31it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.49it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 142.77it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.35it/s]
 73%|███████▎  | 36864/50176 [04:14<01:33, 142.60it/s]
 74%|███████▍  | 37120/50176 [04:16<01:30, 143.67it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 149.10it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.37it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 149.10it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.78it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.16it/s]
 77%|███████▋  | 38656/50176 [04:26<01:18, 147.39it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 144.78it/s]
 78%|███████▊  | 39168/50176 [04:30<01:18, 140.96it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.75it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 144.96it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.22it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.84it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.73it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.75it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.16it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 146.58it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.99it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.94it/s]
 84%|████████▎ | 41984/50176 [04:48<00:53, 152.29it/s]
 84%|██████���█▍ | 42240/50176 [04:50<00:53, 148.86it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 150.11it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.17it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 150.45it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 151.01it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 149.04it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.46it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.14it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.38it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.38it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 145.62it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.24it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 140.97it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.55it/s]
 91%|█████████▏| 45824/50176 [05:15<00:29, 147.38it/s]
 92%|█████████▏| 46080/50176 [05:16<00:27, 147.89it/s]
 92%|█████████▏| 46336/50176 [05:18<00:25, 149.29it/s]
 93%|█████████▎| 46592/50176 [05:20<00:23, 149.54it/s]
 93%|█████████▎| 46848/50176 [05:21<00:22, 148.09it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 143.39it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 145.29it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 143.02it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.09it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 146.90it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.11it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 149.19it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 150.32it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 146.52it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 144.81it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.51it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 138.21it/s]
-2022-12-21,10:13:27 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:13:27 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5813	imagenet-zeroshot-val-top5: 0.8476

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_28.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:13:33 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:13:33 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:13:34 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt).
-2022-12-21,10:13:41 | INFO | Model:
-2022-12-21,10:13:41 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:13:42 | INFO | Params:
-2022-12-21,10:13:42 | INFO |   batch_size: 256
-2022-12-21,10:13:42 | INFO |   beta1: 0.9
-2022-12-21,10:13:42 | INFO |   beta2: 0.98
-2022-12-21,10:13:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/checkpoints
-2022-12-21,10:13:42 | INFO |   copy_codebase: False
-2022-12-21,10:13:42 | INFO |   csv_caption_key: title
-2022-12-21,10:13:42 | INFO |   csv_img_key: filepath
-2022-12-21,10:13:42 | INFO |   csv_separator: 	
-2022-12-21,10:13:42 | INFO |   dataset_resampled: False
-2022-12-21,10:13:42 | INFO |   dataset_type: auto
-2022-12-21,10:13:42 | INFO |   ddp_static_graph: False
-2022-12-21,10:13:42 | INFO |   debug: False
-2022-12-21,10:13:42 | INFO |   device: cuda:0
-2022-12-21,10:13:42 | INFO |   dist_backend: nccl
-2022-12-21,10:13:42 | INFO |   dist_url: env://
-2022-12-21,10:13:42 | INFO |   distributed: False
-2022-12-21,10:13:42 | INFO |   epochs: 32
-2022-12-21,10:13:42 | INFO |   eps: 1e-06
-2022-12-21,10:13:42 | INFO |   force_quick_gelu: False
-2022-12-21,10:13:42 | INFO |   gather_with_grad: False
-2022-12-21,10:13:42 | INFO |   grad_checkpointing: False
-2022-12-21,10:13:42 | INFO |   horovod: False
-2022-12-21,10:13:42 | INFO |   image_mean: None
-2022-12-21,10:13:42 | INFO |   image_std: None
-2022-12-21,10:13:42 | INFO |   imagenet_v2: None
-2022-12-21,10:13:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:13:42 | INFO |   local_loss: False
-2022-12-21,10:13:42 | INFO |   local_rank: 0
-2022-12-21,10:13:42 | INFO |   lock_image: False
-2022-12-21,10:13:42 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:13:42 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:13:42 | INFO |   log_level: 20
-2022-12-21,10:13:42 | INFO |   log_local: False
-2022-12-21,10:13:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_28.pt/out.log
-2022-12-21,10:13:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:13:42 | INFO |   lr: 0.0005
-2022-12-21,10:13:42 | INFO |   model: ViT-B-16
-2022-12-21,10:13:42 | INFO |   name: epoch_28.pt
-2022-12-21,10:13:42 | INFO |   no_set_device_rank: False
-2022-12-21,10:13:42 | INFO |   norm_gradient_clip: None
-2022-12-21,10:13:42 | INFO |   precision: amp_bfloat16
-2022-12-21,10:13:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_28.pt
-2022-12-21,10:13:42 | INFO |   pretrained_image: False
-2022-12-21,10:13:42 | INFO |   rank: 0
-2022-12-21,10:13:42 | INFO |   report_to: 
-2022-12-21,10:13:42 | INFO |   resume: None
-2022-12-21,10:13:42 | INFO |   save_frequency: 1
-2022-12-21,10:13:42 | INFO |   save_most_recent: False
-2022-12-21,10:13:42 | INFO |   seed: 0
-2022-12-21,10:13:42 | INFO |   skip_scheduler: False
-2022-12-21,10:13:42 | INFO |   tensorboard: False
-2022-12-21,10:13:42 | INFO |   tensorboard_path: 
-2022-12-21,10:13:42 | INFO |   torchscript: False
-2022-12-21,10:13:42 | INFO |   trace: False
-2022-12-21,10:13:42 | INFO |   train_data: None
-2022-12-21,10:13:42 | INFO |   train_num_samples: None
-2022-12-21,10:13:42 | INFO |   use_bn_sync: False
-2022-12-21,10:13:42 | INFO |   val_data: None
-2022-12-21,10:13:42 | INFO |   val_frequency: 1
-2022-12-21,10:13:42 | INFO |   val_num_samples: None
-2022-12-21,10:13:42 | INFO |   wandb: False
-2022-12-21,10:13:42 | INFO |   wandb_notes: 
-2022-12-21,10:13:42 | INFO |   warmup: 10000
-2022-12-21,10:13:42 | INFO |   wd: 0.2
-2022-12-21,10:13:42 | INFO |   workers: 1
-2022-12-21,10:13:42 | INFO |   world_size: 1
-2022-12-21,10:13:42 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:13:42 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:13:42 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:53,  1.40it/s]
  1%|          | 6/1000 [00:00<01:44,  9.49it/s]
  1%|          | 12/1000 [00:00<00:53, 18.53it/s]
  2%|▏         | 18/1000 [00:01<00:36, 26.86it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.07it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.99it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.44it/s]
  4%|▍         | 42/1000 [00:01<00:19, 47.95it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.49it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.42it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.79it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.82it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.54it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.04it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.56it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.84it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.21it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.40it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.63it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.75it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.52it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.52it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.59it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.57it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.52it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.64it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.69it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.59it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.46it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.48it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.45it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.38it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.34it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.29it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.11it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.00it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.11it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.07it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.30it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.29it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.20it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.28it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.39it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.49it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.53it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.35it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.39it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.54it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.78it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.79it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.79it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.93it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.92it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.82it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.63it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.72it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.83it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.92it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.69it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.67it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.73it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.81it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.69it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.36it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.43it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.55it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.68it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.76it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.76it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.77it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.80it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.93it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.92it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.70it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.81it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.78it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.73it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.87it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.77it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.75it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.72it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.64it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.70it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.63it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.71it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.74it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.83it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.77it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.88it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.88it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.76it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.75it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.82it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.70it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.64it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.78it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.77it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.70it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.80it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.71it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.71it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.84it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.84it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.93it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.75it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.70it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.84it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.77it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.86it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.88it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.80it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.84it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.86it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.89it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.77it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.85it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.84it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.83it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.85it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.81it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.82it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.83it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.63it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.74it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.88it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.74it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.87it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.84it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.90it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.96it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.75it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.73it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.79it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.77it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.74it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.76it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.86it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.99it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.03it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.94it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.98it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.92it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.92it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.86it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.91it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.88it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.76it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.86it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.86it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.82it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.85it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.84it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.59it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.67it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.79it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.84it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.91it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.96it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.92it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.00it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.08it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.11it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.83it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.52it/s]
-2022-12-21,10:14:00 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:55, 140.40it/s]
  1%|          | 512/50176 [00:03<05:13, 158.57it/s]
  2%|▏         | 768/50176 [00:04<04:46, 172.32it/s]
  2%|▏         | 1024/50176 [00:06<04:51, 168.41it/s]
  3%|▎         | 1280/50176 [00:07<05:08, 158.44it/s]
  3%|▎         | 1536/50176 [00:10<05:34, 145.37it/s]
  4%|▎         | 1792/50176 [00:11<05:38, 143.05it/s]
  4%|▍         | 2048/50176 [00:13<05:34, 143.71it/s]
  5%|▍         | 2304/50176 [00:15<05:36, 142.32it/s]
  5%|▌         | 2560/50176 [00:17<05:33, 142.60it/s]
  6%|▌         | 2816/50176 [00:19<05:38, 139.83it/s]
  6%|▌         | 3072/50176 [00:20<05:30, 142.52it/s]
  7%|▋         | 3328/50176 [00:22<05:28, 142.52it/s]
  7%|▋         | 3584/50176 [00:24<05:21, 144.87it/s]
  8%|▊         | 3840/50176 [00:26<05:31, 139.70it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.07it/s]
  9%|▊         | 4352/50176 [00:29<05:22, 142.01it/s]
  9%|▉         | 4608/50176 [00:31<05:17, 143.43it/s]
 10%|▉         | 4864/50176 [00:33<05:11, 145.62it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.41it/s]
 11%|█         | 5376/50176 [00:36<05:03, 147.39it/s]
 11%|█         | 5632/50176 [00:38<05:00, 148.21it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.14it/s]
 12%|█▏        | 6144/50176 [00:42<05:24, 135.86it/s]
 13%|█▎        | 6400/50176 [00:44<05:14, 139.23it/s]
 13%|█▎        | 6656/50176 [00:45<05:08, 141.24it/s]
 14%|█▍        | 6912/50176 [00:47<05:04, 141.90it/s]
 14%|█▍        | 7168/50176 [00:49<05:01, 142.87it/s]
 15%|█▍        | 7424/50176 [00:51<04:53, 145.72it/s]
 15%|█▌        | 7680/50176 [00:53<04:54, 144.42it/s]
 16%|█▌        | 7936/50176 [00:54<04:45, 148.10it/s]
 16%|█▋        | 8192/50176 [00:56<04:45, 147.21it/s]
 17%|█▋        | 8448/50176 [00:58<04:41, 148.43it/s]
 17%|█▋        | 8704/50176 [00:59<04:38, 149.06it/s]
 18%|█▊        | 8960/50176 [01:01<04:47, 143.57it/s]
 18%|█▊        | 9216/50176 [01:03<04:37, 147.65it/s]
 19%|█▉        | 9472/50176 [01:05<04:36, 147.33it/s]
 19%|█▉        | 9728/50176 [01:06<04:39, 144.65it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 147.90it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.71it/s]
 21%|██        | 10496/50176 [01:11<04:24, 150.24it/s]
 21%|██▏       | 10752/50176 [01:13<04:30, 145.86it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 147.82it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.40it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.50it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.81it/s]
 24%|██▍       | 12032/50176 [01:22<04:14, 150.07it/s]
 24%|██▍       | 12288/50176 [01:23<04:11, 150.58it/s]
 25%|██▌       | 12544/50176 [01:25<04:08, 151.45it/s]
 26%|██▌       | 12800/50176 [01:27<04:06, 151.42it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.78it/s]
 27%|██▋       | 13312/50176 [01:30<04:05, 150.05it/s]
 27%|██▋       | 13568/50176 [01:32<04:12, 144.87it/s]
 28%|██▊       | 13824/50176 [01:34<04:16, 141.74it/s]
 28%|██▊       | 14080/50176 [01:36<04:13, 142.21it/s]
 29%|██▊       | 14336/50176 [01:38<04:18, 138.87it/s]
 29%|██▉       | 14592/50176 [01:39<04:05, 144.73it/s]
 30%|██▉       | 14848/50176 [01:41<04:18, 136.63it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.37it/s]
 31%|███       | 15360/50176 [01:45<04:10, 138.86it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.40it/s]
 32%|███▏      | 15872/50176 [01:49<04:10, 137.01it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.65it/s]
 33%|███▎      | 16384/50176 [01:52<04:01, 139.70it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 134.93it/s]
 34%|███▎      | 16896/50176 [01:56<03:57, 139.96it/s]
 34%|███▍      | 17152/50176 [01:58<03:50, 143.10it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.40it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.89it/s]
 36%|███▌      | 17920/50176 [02:03<03:36, 149.18it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.91it/s]
 37%|███▋      | 18432/50176 [02:06<03:34, 147.83it/s]
 37%|███▋      | 18688/50176 [02:08<03:33, 147.44it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.05it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.36it/s]
 39%|███▉      | 19456/50176 [02:14<03:53, 131.46it/s]
 39%|███▉      | 19712/50176 [02:16<03:51, 131.67it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.66it/s]
 40%|████      | 20224/50176 [02:20<03:32, 141.25it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.77it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 143.89it/s]
 42%|████▏     | 20992/50176 [02:25<03:18, 147.22it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 142.20it/s]
 43%|████▎     | 21504/50176 [02:28<03:15, 146.89it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.49it/s]
 44%|████▍     | 22016/50176 [02:32<03:05, 151.58it/s]
 44%|████▍     | 22272/50176 [02:33<03:04, 150.94it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.51it/s]
 45%|████▌     | 22784/50176 [02:37<03:07, 145.91it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 143.95it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.64it/s]
 47%|████▋     | 23552/50176 [02:43<03:14, 136.90it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.02it/s]
 48%|████▊     | 24064/50176 [02:46<03:04, 141.87it/s]
 48%|████▊     | 24320/50176 [02:48<03:01, 142.61it/s]
 49%|████▉     | 24576/50176 [02:50<02:56, 145.13it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.66it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.70it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.74it/s]
 51%|█████     | 25600/50176 [02:56<02:48, 146.16it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.59it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.25it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.14it/s]
 53%|█████▎    | 26624/50176 [03:03<02:42, 145.31it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.29it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 146.20it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.69it/s]
 55%|█████▌    | 27648/50176 [03:10<02:27, 153.24it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 142.52it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.78it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.50it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.90it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 143.13it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 146.11it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.00it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.30it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.32it/s]
 60%|██████    | 30208/50176 [03:29<02:23, 138.90it/s]
 61%|██████    | 30464/50176 [03:30<02:15, 145.36it/s]
 61%|██████    | 30720/50176 [03:32<02:15, 144.08it/s]
 62%|██████▏   | 30976/50176 [03:34<02:10, 146.95it/s]
 62%|██████▏   | 31232/50176 [03:35<02:07, 149.08it/s]
 63%|██████▎   | 31488/50176 [03:37<02:03, 150.96it/s]
 63%|██████▎   | 31744/50176 [03:39<02:01, 151.15it/s]
 64%|██████▍   | 32000/50176 [03:41<02:01, 149.96it/s]
 64%|██████▍   | 32256/50176 [03:42<01:59, 150.43it/s]
 65%|██████▍   | 32512/50176 [03:44<02:01, 145.75it/s]
 65%|██████▌   | 32768/50176 [03:46<01:56, 149.39it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 147.58it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 148.79it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 145.21it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 147.97it/s]
 68%|██████▊   | 34048/50176 [03:54<01:45, 152.19it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 154.37it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.61it/s]
 69%|██████▉   | 34816/50176 [03:59<01:43, 148.57it/s]
 70%|██████▉   | 35072/50176 [04:01<01:42, 147.34it/s]
 70%|███████   | 35328/50176 [04:03<01:39, 149.83it/s]
 71%|███████   | 35584/50176 [04:05<01:38, 147.82it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.35it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.72it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 143.38it/s]
 73%|███████▎  | 36608/50176 [04:12<01:32, 146.06it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.74it/s]
 74%|███████▍  | 37120/50176 [04:15<01:29, 146.21it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 150.11it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.58it/s]
 76%|███████▌  | 37888/50176 [04:20<01:22, 149.01it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 150.13it/s]
 77%|███████▋  | 38400/50176 [04:24<01:19, 148.91it/s]
 77%|███████▋  | 38656/50176 [04:26<01:17, 147.90it/s]
 78%|███████▊  | 38912/50176 [04:27<01:17, 145.72it/s]
 78%|███████▊  | 39168/50176 [04:29<01:17, 142.01it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.70it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.37it/s]
 80%|███████▉  | 39936/50176 [04:34<01:09, 147.11it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.53it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.79it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.77it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.25it/s]
 82%|████████▏ | 41216/50176 [04:43<01:01, 146.40it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 147.97it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.04it/s]
 84%|████████▎ | 41984/50176 [04:48<00:54, 150.71it/s]
 84%|██████���█▍ | 42240/50176 [04:50<00:53, 147.99it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.75it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.00it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 150.33it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.59it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.74it/s]
 87%|████████▋ | 43776/50176 [05:00<00:43, 146.74it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.18it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.11it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.37it/s]
 89%|████████▉ | 44800/50176 [05:07<00:36, 145.85it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.44it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 141.68it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.77it/s]
 91%|█████████▏| 45824/50176 [05:15<00:30, 144.89it/s]
 92%|█████████▏| 46080/50176 [05:16<00:28, 144.77it/s]
 92%|█████████▏| 46336/50176 [05:18<00:26, 145.65it/s]
 93%|█████████▎| 46592/50176 [05:20<00:24, 147.01it/s]
 93%|█████████▎| 46848/50176 [05:21<00:22, 146.05it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 142.00it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 144.19it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.44it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.16it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 147.35it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.41it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.28it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 149.56it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 147.50it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.63it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.23it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 137.69it/s]
-2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5849	imagenet-zeroshot-val-top5: 0.8517

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_29.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt).
-2022-12-21,10:13:40 | INFO | Model:
-2022-12-21,10:13:40 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:13:40 | INFO | Params:
-2022-12-21,10:13:40 | INFO |   batch_size: 256
-2022-12-21,10:13:40 | INFO |   beta1: 0.9
-2022-12-21,10:13:40 | INFO |   beta2: 0.98
-2022-12-21,10:13:40 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/checkpoints
-2022-12-21,10:13:40 | INFO |   copy_codebase: False
-2022-12-21,10:13:40 | INFO |   csv_caption_key: title
-2022-12-21,10:13:40 | INFO |   csv_img_key: filepath
-2022-12-21,10:13:40 | INFO |   csv_separator: 	
-2022-12-21,10:13:40 | INFO |   dataset_resampled: False
-2022-12-21,10:13:40 | INFO |   dataset_type: auto
-2022-12-21,10:13:40 | INFO |   ddp_static_graph: False
-2022-12-21,10:13:40 | INFO |   debug: False
-2022-12-21,10:13:40 | INFO |   device: cuda:0
-2022-12-21,10:13:40 | INFO |   dist_backend: nccl
-2022-12-21,10:13:40 | INFO |   dist_url: env://
-2022-12-21,10:13:40 | INFO |   distributed: False
-2022-12-21,10:13:40 | INFO |   epochs: 32
-2022-12-21,10:13:40 | INFO |   eps: 1e-06
-2022-12-21,10:13:40 | INFO |   force_quick_gelu: False
-2022-12-21,10:13:40 | INFO |   gather_with_grad: False
-2022-12-21,10:13:40 | INFO |   grad_checkpointing: False
-2022-12-21,10:13:40 | INFO |   horovod: False
-2022-12-21,10:13:40 | INFO |   image_mean: None
-2022-12-21,10:13:40 | INFO |   image_std: None
-2022-12-21,10:13:40 | INFO |   imagenet_v2: None
-2022-12-21,10:13:40 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:13:40 | INFO |   local_loss: False
-2022-12-21,10:13:40 | INFO |   local_rank: 0
-2022-12-21,10:13:40 | INFO |   lock_image: False
-2022-12-21,10:13:40 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:13:40 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:13:40 | INFO |   log_level: 20
-2022-12-21,10:13:40 | INFO |   log_local: False
-2022-12-21,10:13:40 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_29.pt/out.log
-2022-12-21,10:13:40 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:13:40 | INFO |   lr: 0.0005
-2022-12-21,10:13:40 | INFO |   model: ViT-B-16
-2022-12-21,10:13:40 | INFO |   name: epoch_29.pt
-2022-12-21,10:13:40 | INFO |   no_set_device_rank: False
-2022-12-21,10:13:40 | INFO |   norm_gradient_clip: None
-2022-12-21,10:13:40 | INFO |   precision: amp_bfloat16
-2022-12-21,10:13:40 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_29.pt
-2022-12-21,10:13:40 | INFO |   pretrained_image: False
-2022-12-21,10:13:40 | INFO |   rank: 0
-2022-12-21,10:13:40 | INFO |   report_to: 
-2022-12-21,10:13:40 | INFO |   resume: None
-2022-12-21,10:13:40 | INFO |   save_frequency: 1
-2022-12-21,10:13:40 | INFO |   save_most_recent: False
-2022-12-21,10:13:40 | INFO |   seed: 0
-2022-12-21,10:13:40 | INFO |   skip_scheduler: False
-2022-12-21,10:13:40 | INFO |   tensorboard: False
-2022-12-21,10:13:40 | INFO |   tensorboard_path: 
-2022-12-21,10:13:40 | INFO |   torchscript: False
-2022-12-21,10:13:40 | INFO |   trace: False
-2022-12-21,10:13:40 | INFO |   train_data: None
-2022-12-21,10:13:40 | INFO |   train_num_samples: None
-2022-12-21,10:13:40 | INFO |   use_bn_sync: False
-2022-12-21,10:13:40 | INFO |   val_data: None
-2022-12-21,10:13:40 | INFO |   val_frequency: 1
-2022-12-21,10:13:40 | INFO |   val_num_samples: None
-2022-12-21,10:13:40 | INFO |   wandb: False
-2022-12-21,10:13:40 | INFO |   wandb_notes: 
-2022-12-21,10:13:40 | INFO |   warmup: 10000
-2022-12-21,10:13:40 | INFO |   wd: 0.2
-2022-12-21,10:13:40 | INFO |   workers: 1
-2022-12-21,10:13:40 | INFO |   world_size: 1
-2022-12-21,10:13:40 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:13:40 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:13:40 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<14:33,  1.14it/s]
  1%|          | 6/1000 [00:00<02:05,  7.94it/s]
  1%|          | 12/1000 [00:01<01:01, 16.11it/s]
  2%|▏         | 18/1000 [00:01<00:40, 24.18it/s]
  2%|▏         | 24/1000 [00:01<00:31, 31.40it/s]
  3%|▎         | 30/1000 [00:01<00:26, 36.99it/s]
  4%|▎         | 36/1000 [00:01<00:23, 41.31it/s]
  4%|▍         | 42/1000 [00:01<00:21, 44.77it/s]
  5%|▍         | 48/1000 [00:01<00:20, 47.24it/s]
  5%|▌         | 54/1000 [00:01<00:19, 49.22it/s]
  6%|▌         | 60/1000 [00:01<00:18, 51.34it/s]
  7%|▋         | 66/1000 [00:02<00:17, 53.19it/s]
  7%|▋         | 72/1000 [00:02<00:17, 54.53it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.41it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.10it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.55it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.03it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.28it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.56it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.72it/s]
 12%|█▏        | 120/1000 [00:03<00:15, 57.71it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.65it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.68it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.65it/s]
 14%|█▍        | 144/1000 [00:03<00:15, 54.15it/s]
 15%|█▌        | 150/1000 [00:03<00:15, 55.26it/s]
 16%|█▌        | 156/1000 [00:03<00:15, 56.04it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 56.45it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 56.51it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 56.82it/s]
 18%|█▊        | 180/1000 [00:04<00:14, 57.00it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.09it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.13it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.16it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.03it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.96it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.11it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.09it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.35it/s]
 23%|██▎       | 234/1000 [00:05<00:13, 57.34it/s]
 24%|██▍       | 240/1000 [00:05<00:13, 57.24it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.30it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.40it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.50it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.54it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.38it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.55it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.66it/s]
 29%|██▉       | 294/1000 [00:06<00:12, 57.86it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.83it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.79it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.93it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.92it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.83it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.65it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.71it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.80it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.88it/s]
 35%|███▌      | 354/1000 [00:07<00:11, 57.60it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.62it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.72it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.82it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.75it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.52it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.63it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.77it/s]
 41%|████      | 408/1000 [00:08<00:10, 57.84it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.84it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.84it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.87it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.98it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.97it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.75it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.82it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.86it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.83it/s]
 47%|████▋     | 468/1000 [00:09<00:09, 57.80it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.95it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.86it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.84it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.82it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.74it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.82it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.73it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.80it/s]
 52%|█████▏    | 522/1000 [00:10<00:08, 57.83it/s]
 53%|█████▎    | 528/1000 [00:10<00:08, 57.91it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.80it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.92it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.95it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.84it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.87it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.95it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.82it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.77it/s]
 58%|█████▊    | 582/1000 [00:11<00:07, 57.89it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.87it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.79it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.91it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.81it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.81it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.95it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.93it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.01it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.80it/s]
 64%|██████▍   | 642/1000 [00:12<00:06, 57.75it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.92it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.85it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.95it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.99it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.91it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.96it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 696/1000 [00:13<00:05, 57.85it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.95it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.95it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.93it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.89it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.89it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.86it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.85it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.91it/s]
 76%|███████▌  | 756/1000 [00:14<00:04, 58.01it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.85it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.94it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.89it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.93it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.96it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.80it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.77it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.82it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.78it/s]
 82%|████████▏ | 816/1000 [00:15<00:03, 57.77it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.81it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.90it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.05it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.08it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.00it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.03it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.95it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.94it/s]
 87%|████████▋ | 870/1000 [00:16<00:02, 57.87it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.93it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.93it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.80it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.91it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.89it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.87it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.89it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.89it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.90it/s]
 93%|█████████▎| 930/1000 [00:17<00:01, 58.01it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 58.05it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.02it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.02it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.01it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.89it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.98it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.07it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.07it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.07it/s]
 99%|█████████▉| 990/1000 [00:18<00:00, 58.04it/s]
-2022-12-21,10:13:58 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:29, 111.01it/s]
  1%|          | 512/50176 [00:04<06:30, 127.17it/s]
  2%|▏         | 768/50176 [00:05<06:03, 135.91it/s]
  2%|▏         | 1024/50176 [00:07<05:52, 139.42it/s]
  3%|▎         | 1280/50176 [00:09<05:47, 140.83it/s]
  3%|▎         | 1536/50176 [00:11<05:59, 135.42it/s]
  4%|▎         | 1792/50176 [00:13<05:54, 136.47it/s]
  4%|▍         | 2048/50176 [00:15<05:46, 138.99it/s]
  5%|▍         | 2304/50176 [00:16<05:44, 139.09it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.45it/s]
  6%|▌         | 2816/50176 [00:20<05:40, 139.10it/s]
  6%|▌         | 3072/50176 [00:22<05:33, 141.12it/s]
  7%|▋         | 3328/50176 [00:24<05:30, 141.83it/s]
  7%|▋         | 3584/50176 [00:25<05:23, 144.05it/s]
  8%|▊         | 3840/50176 [00:27<05:33, 138.81it/s]
  8%|▊         | 4096/50176 [00:29<05:29, 139.96it/s]
  9%|▊         | 4352/50176 [00:31<05:23, 141.68it/s]
  9%|▉         | 4608/50176 [00:33<05:18, 143.22it/s]
 10%|▉         | 4864/50176 [00:34<05:11, 145.44it/s]
 10%|█         | 5120/50176 [00:36<05:07, 146.36it/s]
 11%|█         | 5376/50176 [00:38<05:04, 147.18it/s]
 11%|█         | 5632/50176 [00:39<05:00, 148.26it/s]
 12%|█▏        | 5888/50176 [00:41<04:59, 148.08it/s]
 12%|█▏        | 6144/50176 [00:43<05:24, 135.85it/s]
 13%|█▎        | 6400/50176 [00:45<05:14, 139.21it/s]
 13%|█▎        | 6656/50176 [00:47<05:08, 141.25it/s]
 14%|█▍        | 6912/50176 [00:49<05:04, 141.91it/s]
 14%|█▍        | 7168/50176 [00:50<05:00, 143.31it/s]
 15%|█▍        | 7424/50176 [00:52<04:53, 145.56it/s]
 15%|█▌        | 7680/50176 [00:54<04:53, 144.61it/s]
 16%|█▌        | 7936/50176 [00:56<04:45, 147.90it/s]
 16%|█▋        | 8192/50176 [00:57<04:45, 147.31it/s]
 17%|█▋        | 8448/50176 [00:59<04:41, 148.26it/s]
 17%|█▋        | 8704/50176 [01:01<04:38, 148.84it/s]
 18%|█▊        | 8960/50176 [01:03<04:45, 144.19it/s]
 18%|█▊        | 9216/50176 [01:04<04:37, 147.42it/s]
 19%|█▉        | 9472/50176 [01:06<04:36, 147.16it/s]
 19%|█▉        | 9728/50176 [01:08<04:39, 144.53it/s]
 20%|█▉        | 9984/50176 [01:09<04:31, 148.04it/s]
 20%|██        | 10240/50176 [01:11<04:26, 149.68it/s]
 21%|██        | 10496/50176 [01:13<04:24, 150.20it/s]
 21%|██▏       | 10752/50176 [01:15<04:29, 146.08it/s]
 22%|██▏       | 11008/50176 [01:16<04:25, 147.62it/s]
 22%|██▏       | 11264/50176 [01:18<04:21, 148.52it/s]
 23%|██▎       | 11520/50176 [01:20<04:16, 150.42it/s]
 23%|██▎       | 11776/50176 [01:21<04:16, 149.97it/s]
 24%|██▍       | 12032/50176 [01:23<04:14, 150.10it/s]
 24%|██▍       | 12288/50176 [01:25<04:11, 150.75it/s]
 25%|██▌       | 12544/50176 [01:27<04:08, 151.22it/s]
 26%|██▌       | 12800/50176 [01:28<04:06, 151.63it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.61it/s]
 27%|██▋       | 13312/50176 [01:32<04:05, 150.10it/s]
 27%|██▋       | 13568/50176 [01:34<04:12, 144.75it/s]
 28%|██▊       | 13824/50176 [01:35<04:16, 141.61it/s]
 28%|██▊       | 14080/50176 [01:37<04:13, 142.17it/s]
 29%|██▊       | 14336/50176 [01:39<04:16, 139.57it/s]
 29%|██▉       | 14592/50176 [01:41<04:06, 144.15it/s]
 30%|██▉       | 14848/50176 [01:43<04:18, 136.60it/s]
 30%|███       | 15104/50176 [01:45<04:18, 135.53it/s]
 31%|███       | 15360/50176 [01:47<04:10, 138.88it/s]
 31%|███       | 15616/50176 [01:48<04:05, 140.99it/s]
 32%|███▏      | 15872/50176 [01:50<04:09, 137.67it/s]
 32%|███▏      | 16128/50176 [01:52<04:04, 139.48it/s]
 33%|███▎      | 16384/50176 [01:54<04:01, 139.66it/s]
 33%|███▎      | 16640/50176 [01:56<04:08, 135.03it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 139.92it/s]
 34%|███▍      | 17152/50176 [01:59<03:50, 143.01it/s]
 35%|███▍      | 17408/50176 [02:01<03:45, 145.28it/s]
 35%|███▌      | 17664/50176 [02:03<03:41, 146.70it/s]
 36%|███▌      | 17920/50176 [02:04<03:36, 149.25it/s]
 36%|███▌      | 18176/50176 [02:06<03:36, 147.59it/s]
 37%|███▋      | 18432/50176 [02:08<03:34, 148.08it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.11it/s]
 38%|███▊      | 18944/50176 [02:12<03:40, 141.67it/s]
 38%|███▊      | 19200/50176 [02:14<03:50, 134.46it/s]
 39%|███▉      | 19456/50176 [02:16<03:53, 131.53it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.18it/s]
 40%|███▉      | 19968/50176 [02:19<03:44, 134.68it/s]
 40%|████      | 20224/50176 [02:21<03:32, 141.17it/s]
 41%|████      | 20480/50176 [02:23<03:26, 143.74it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 143.86it/s]
 42%|████▏     | 20992/50176 [02:26<03:18, 147.19it/s]
 42%|████▏     | 21248/50176 [02:28<03:22, 142.55it/s]
 43%|████▎     | 21504/50176 [02:30<03:15, 146.72it/s]
 43%|████▎     | 21760/50176 [02:31<03:11, 148.36it/s]
 44%|████▍     | 22016/50176 [02:33<03:05, 151.43it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.83it/s]
 45%|████▍     | 22528/50176 [02:36<03:02, 151.68it/s]
 45%|████▌     | 22784/50176 [02:38<03:08, 145.32it/s]
 46%|████▌     | 23040/50176 [02:40<03:08, 144.32it/s]
 46%|████▋     | 23296/50176 [02:42<03:11, 140.03it/s]
 47%|████▋     | 23552/50176 [02:44<03:14, 136.95it/s]
 47%|████▋     | 23808/50176 [02:46<03:07, 140.82it/s]
 48%|████▊     | 24064/50176 [02:48<03:03, 142.02it/s]
 48%|████▊     | 24320/50176 [02:49<03:01, 142.48it/s]
 49%|████▉     | 24576/50176 [02:51<02:56, 145.15it/s]
 49%|████▉     | 24832/50176 [02:53<02:50, 148.36it/s]
 50%|█████     | 25088/50176 [02:54<02:48, 148.85it/s]
 51%|█████     | 25344/50176 [02:56<02:46, 149.52it/s]
 51%|█████     | 25600/50176 [02:58<02:48, 145.62it/s]
 52%|█████▏    | 25856/50176 [02:59<02:41, 150.62it/s]
 52%|█████▏    | 26112/50176 [03:01<02:42, 148.50it/s]
 53%|█████▎    | 26368/50176 [03:03<02:45, 143.57it/s]
 53%|█████▎    | 26624/50176 [03:05<02:41, 145.71it/s]
 54%|█████▎    | 26880/50176 [03:07<02:40, 145.28it/s]
 54%|█████▍    | 27136/50176 [03:08<02:37, 146.48it/s]
 55%|█████▍    | 27392/50176 [03:10<02:35, 146.68it/s]
 55%|█████▌    | 27648/50176 [03:12<02:26, 153.43it/s]
 56%|█████▌    | 27904/50176 [03:14<02:36, 142.21it/s]
 56%|█████▌    | 28160/50176 [03:15<02:32, 144.08it/s]
 57%|█████▋    | 28416/50176 [03:17<02:30, 144.41it/s]
 57%|█████▋    | 28672/50176 [03:19<02:32, 140.63it/s]
 58%|█████▊    | 28928/50176 [03:21<02:28, 143.24it/s]
 58%|█████▊    | 29184/50176 [03:22<02:23, 146.38it/s]
 59%|█████▊    | 29440/50176 [03:25<02:30, 137.94it/s]
 59%|█████▉    | 29696/50176 [03:27<02:30, 136.30it/s]
 60%|█████▉    | 29952/50176 [03:28<02:29, 135.67it/s]
 60%|██████    | 30208/50176 [03:30<02:23, 138.81it/s]
 61%|██████    | 30464/50176 [03:32<02:15, 145.20it/s]
 61%|██████    | 30720/50176 [03:34<02:14, 144.23it/s]
 62%|██████▏   | 30976/50176 [03:35<02:10, 146.81it/s]
 62%|██████▏   | 31232/50176 [03:37<02:06, 149.17it/s]
 63%|██████▎   | 31488/50176 [03:39<02:03, 150.85it/s]
 63%|██████▎   | 31744/50176 [03:40<02:01, 151.27it/s]
 64%|██████▍   | 32000/50176 [03:42<02:01, 149.58it/s]
 64%|██████▍   | 32256/50176 [03:44<01:58, 150.66it/s]
 65%|██████▍   | 32512/50176 [03:46<02:01, 145.27it/s]
 65%|██████▌   | 32768/50176 [03:47<01:56, 149.53it/s]
 66%|██████▌   | 33024/50176 [03:49<01:56, 147.56it/s]
 66%|██████▋   | 33280/50176 [03:51<01:53, 149.03it/s]
 67%|██████▋   | 33536/50176 [03:52<01:54, 144.96it/s]
 67%|██████▋   | 33792/50176 [03:54<01:50, 148.23it/s]
 68%|██████▊   | 34048/50176 [03:56<01:45, 152.29it/s]
 68%|██████▊   | 34304/50176 [03:57<01:42, 154.56it/s]
 69%|██████▉   | 34560/50176 [03:59<01:45, 148.30it/s]
 69%|██████▉   | 34816/50176 [04:01<01:43, 148.75it/s]
 70%|██████▉   | 35072/50176 [04:03<01:42, 146.98it/s]
 70%|███████   | 35328/50176 [04:04<01:38, 150.08it/s]
 71%|███████   | 35584/50176 [04:06<01:38, 147.58it/s]
 71%|███████▏  | 35840/50176 [04:08<01:40, 143.19it/s]
 72%|███████▏  | 36096/50176 [04:10<01:36, 145.28it/s]
 72%|███████▏  | 36352/50176 [04:12<01:36, 143.47it/s]
 73%|███████▎  | 36608/50176 [04:13<01:33, 145.71it/s]
 73%|███████▎  | 36864/50176 [04:15<01:32, 143.92it/s]
 74%|███████▍  | 37120/50176 [04:17<01:29, 145.96it/s]
 74%|███████▍  | 37376/50176 [04:18<01:25, 150.31it/s]
 75%|███████▌  | 37632/50176 [04:20<01:22, 151.24it/s]
 76%|███████▌  | 37888/50176 [04:22<01:22, 149.26it/s]
 76%|███████▌  | 38144/50176 [04:23<01:20, 150.01it/s]
 77%|███████▋  | 38400/50176 [04:25<01:18, 149.17it/s]
 77%|███████▋  | 38656/50176 [04:27<01:17, 147.71it/s]
 78%|███████▊  | 38912/50176 [04:29<01:17, 145.87it/s]
 78%|███████▊  | 39168/50176 [04:31<01:17, 141.29it/s]
 79%|███████▊  | 39424/50176 [04:32<01:14, 144.07it/s]
 79%|███████▉  | 39680/50176 [04:34<01:12, 145.35it/s]
 80%|███████▉  | 39936/50176 [04:36<01:09, 147.38it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.43it/s]
 81%|████████  | 40448/50176 [04:39<01:07, 143.38it/s]
 81%|████████  | 40704/50176 [04:41<01:08, 138.24it/s]
 82%|████████▏ | 40960/50176 [04:43<01:04, 143.71it/s]
 82%|████████▏ | 41216/50176 [04:45<01:01, 146.50it/s]
 83%|████████▎ | 41472/50176 [04:46<00:58, 148.32it/s]
 83%|████████▎ | 41728/50176 [04:48<00:55, 150.98it/s]
 84%|████████▎ | 41984/50176 [04:50<00:54, 150.95it/s]
 84%|██████���█▍ | 42240/50176 [04:52<00:53, 147.90it/s]
 85%|████████▍ | 42496/50176 [04:53<00:51, 149.97it/s]
 85%|████████▌ | 42752/50176 [04:55<00:49, 148.85it/s]
 86%|████████▌ | 43008/50176 [04:57<00:47, 150.53it/s]
 86%|████████▌ | 43264/50176 [04:58<00:45, 150.44it/s]
 87%|████████▋ | 43520/50176 [05:00<00:44, 148.91it/s]
 87%|████████▋ | 43776/50176 [05:02<00:43, 146.58it/s]
 88%|████████▊ | 44032/50176 [05:04<00:41, 148.35it/s]
 88%|████████▊ | 44288/50176 [05:05<00:39, 149.93it/s]
 89%|████████▉ | 44544/50176 [05:07<00:38, 147.54it/s]
 89%|████████▉ | 44800/50176 [05:09<00:36, 145.69it/s]
 90%|████████▉ | 45056/50176 [05:11<00:34, 147.58it/s]
 90%|█████████ | 45312/50176 [05:13<00:34, 140.93it/s]
 91%|█████████ | 45568/50176 [05:14<00:31, 146.10it/s]
 91%|█████████▏| 45824/50176 [05:16<00:30, 144.84it/s]
 92%|█████████▏| 46080/50176 [05:18<00:28, 145.05it/s]
 92%|█████████▏| 46336/50176 [05:19<00:26, 145.57it/s]
 93%|█████████▎| 46592/50176 [05:21<00:24, 147.12it/s]
 93%|█████████▎| 46848/50176 [05:23<00:22, 145.96it/s]
 94%|█████████▍| 47104/50176 [05:25<00:21, 142.38it/s]
 94%|█████████▍| 47360/50176 [05:27<00:19, 143.98it/s]
 95%|█████████▍| 47616/50176 [05:28<00:17, 142.45it/s]
 95%|█████████▌| 47872/50176 [05:30<00:15, 145.01it/s]
 96%|█████████▌| 48128/50176 [05:32<00:13, 147.18it/s]
 96%|█████████▋| 48384/50176 [05:33<00:12, 148.60it/s]
 97%|█████████▋| 48640/50176 [05:35<00:10, 150.27it/s]
 97%|█████████▋| 48896/50176 [05:37<00:08, 150.44it/s]
 98%|█████████▊| 49152/50176 [05:39<00:06, 147.19it/s]
 98%|█████████▊| 49408/50176 [05:40<00:05, 145.56it/s]
 99%|█████████▉| 49664/50176 [05:42<00:03, 145.32it/s]
 99%|█████████▉| 49920/50176 [05:44<00:01, 137.68it/s]
-2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5796	imagenet-zeroshot-val-top5: 0.8498

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_3.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt).
-2022-12-21,10:13:41 | INFO | Model:
-2022-12-21,10:13:42 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:13:42 | INFO | Params:
-2022-12-21,10:13:42 | INFO |   batch_size: 256
-2022-12-21,10:13:42 | INFO |   beta1: 0.9
-2022-12-21,10:13:42 | INFO |   beta2: 0.98
-2022-12-21,10:13:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/checkpoints
-2022-12-21,10:13:42 | INFO |   copy_codebase: False
-2022-12-21,10:13:42 | INFO |   csv_caption_key: title
-2022-12-21,10:13:42 | INFO |   csv_img_key: filepath
-2022-12-21,10:13:42 | INFO |   csv_separator: 	
-2022-12-21,10:13:42 | INFO |   dataset_resampled: False
-2022-12-21,10:13:42 | INFO |   dataset_type: auto
-2022-12-21,10:13:42 | INFO |   ddp_static_graph: False
-2022-12-21,10:13:42 | INFO |   debug: False
-2022-12-21,10:13:42 | INFO |   device: cuda:0
-2022-12-21,10:13:42 | INFO |   dist_backend: nccl
-2022-12-21,10:13:42 | INFO |   dist_url: env://
-2022-12-21,10:13:42 | INFO |   distributed: False
-2022-12-21,10:13:42 | INFO |   epochs: 32
-2022-12-21,10:13:42 | INFO |   eps: 1e-06
-2022-12-21,10:13:42 | INFO |   force_quick_gelu: False
-2022-12-21,10:13:42 | INFO |   gather_with_grad: False
-2022-12-21,10:13:42 | INFO |   grad_checkpointing: False
-2022-12-21,10:13:42 | INFO |   horovod: False
-2022-12-21,10:13:42 | INFO |   image_mean: None
-2022-12-21,10:13:42 | INFO |   image_std: None
-2022-12-21,10:13:42 | INFO |   imagenet_v2: None
-2022-12-21,10:13:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:13:42 | INFO |   local_loss: False
-2022-12-21,10:13:42 | INFO |   local_rank: 0
-2022-12-21,10:13:42 | INFO |   lock_image: False
-2022-12-21,10:13:42 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:13:42 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:13:42 | INFO |   log_level: 20
-2022-12-21,10:13:42 | INFO |   log_local: False
-2022-12-21,10:13:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_3.pt/out.log
-2022-12-21,10:13:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:13:42 | INFO |   lr: 0.0005
-2022-12-21,10:13:42 | INFO |   model: ViT-B-16
-2022-12-21,10:13:42 | INFO |   name: epoch_3.pt
-2022-12-21,10:13:42 | INFO |   no_set_device_rank: False
-2022-12-21,10:13:42 | INFO |   norm_gradient_clip: None
-2022-12-21,10:13:42 | INFO |   precision: amp_bfloat16
-2022-12-21,10:13:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_3.pt
-2022-12-21,10:13:42 | INFO |   pretrained_image: False
-2022-12-21,10:13:42 | INFO |   rank: 0
-2022-12-21,10:13:42 | INFO |   report_to: 
-2022-12-21,10:13:42 | INFO |   resume: None
-2022-12-21,10:13:42 | INFO |   save_frequency: 1
-2022-12-21,10:13:42 | INFO |   save_most_recent: False
-2022-12-21,10:13:42 | INFO |   seed: 0
-2022-12-21,10:13:42 | INFO |   skip_scheduler: False
-2022-12-21,10:13:42 | INFO |   tensorboard: False
-2022-12-21,10:13:42 | INFO |   tensorboard_path: 
-2022-12-21,10:13:42 | INFO |   torchscript: False
-2022-12-21,10:13:42 | INFO |   trace: False
-2022-12-21,10:13:42 | INFO |   train_data: None
-2022-12-21,10:13:42 | INFO |   train_num_samples: None
-2022-12-21,10:13:42 | INFO |   use_bn_sync: False
-2022-12-21,10:13:42 | INFO |   val_data: None
-2022-12-21,10:13:42 | INFO |   val_frequency: 1
-2022-12-21,10:13:42 | INFO |   val_num_samples: None
-2022-12-21,10:13:42 | INFO |   wandb: False
-2022-12-21,10:13:42 | INFO |   wandb_notes: 
-2022-12-21,10:13:42 | INFO |   warmup: 10000
-2022-12-21,10:13:42 | INFO |   wd: 0.2
-2022-12-21,10:13:42 | INFO |   workers: 1
-2022-12-21,10:13:42 | INFO |   world_size: 1
-2022-12-21,10:13:42 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:13:42 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:13:42 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:24,  1.60it/s]
  1%|          | 6/1000 [00:00<01:34, 10.53it/s]
  1%|          | 12/1000 [00:00<00:48, 20.51it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.19it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.27it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.87it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.05it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.14it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.45it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.09it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.30it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.79it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.50it/s]
  8%|▊         | 78/1000 [00:01<00:16, 55.99it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.52it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.49it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.96it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.19it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.50it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.55it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.56it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.53it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.59it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.55it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.60it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.67it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.72it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.59it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.47it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.47it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.44it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.28it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.25it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.21it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.03it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.91it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.03it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.00it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.22it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.08it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.05it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.14it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.28it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.41it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.14it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.06it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.14it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.37it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.51it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.74it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.74it/s]
 31%|███       | 306/1000 [00:05<00:12, 57.73it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.88it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.86it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.76it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.59it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.66it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.75it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.83it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 53.98it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 55.03it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 55.87it/s]
 37%|███▋      | 372/1000 [00:07<00:11, 56.48it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 56.75it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 56.73it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 56.99it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.23it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.47it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.61it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.65it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.69it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.72it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.86it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.86it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.66it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.73it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.78it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.77it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.75it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.89it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.80it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.77it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.77it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.69it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.58it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.65it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.70it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.79it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.73it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.85it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.86it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.75it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.47it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.61it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.55it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.53it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.70it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.71it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.65it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.76it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.68it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.69it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.80it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.80it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.89it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.71it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.64it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.70it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.67it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.77it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.82it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.76it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.82it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.83it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.86it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.75it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.83it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.69it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.69it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.73it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.71it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.72it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.72it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.82it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.91it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.75it/s]
 77%|███████▋  | 768/1000 [00:13<00:04, 57.69it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.69it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.75it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.81it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.63it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.64it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.71it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.69it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.66it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.69it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.63it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.80it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.86it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.79it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.86it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.52it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.61it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.62it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.74it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 57.76it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.65it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.77it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.78it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.75it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.77it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.78it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.75it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.46it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 57.60it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.66it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.74it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.82it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.77it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.86it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.94it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.98it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.73it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.83it/s]
-2022-12-21,10:14:00 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<05:58, 139.42it/s]
  1%|          | 512/50176 [00:03<05:10, 159.99it/s]
  2%|▏         | 768/50176 [00:04<04:44, 173.40it/s]
  2%|▏         | 1024/50176 [00:06<04:59, 164.33it/s]
  3%|▎         | 1280/50176 [00:08<05:11, 157.09it/s]
  3%|▎         | 1536/50176 [00:10<05:36, 144.56it/s]
  4%|▎         | 1792/50176 [00:11<05:38, 143.13it/s]
  4%|▍         | 2048/50176 [00:13<05:35, 143.62it/s]
  5%|▍         | 2304/50176 [00:15<05:37, 141.84it/s]
  5%|▌         | 2560/50176 [00:17<05:34, 142.48it/s]
  6%|▌         | 2816/50176 [00:19<05:38, 139.95it/s]
  6%|▌         | 3072/50176 [00:20<05:31, 142.18it/s]
  7%|▋         | 3328/50176 [00:22<05:28, 142.52it/s]
  7%|▋         | 3584/50176 [00:24<05:21, 144.89it/s]
  8%|▊         | 3840/50176 [00:26<05:32, 139.47it/s]
  8%|▊         | 4096/50176 [00:28<05:28, 140.36it/s]
  9%|▊         | 4352/50176 [00:29<05:22, 142.04it/s]
  9%|▉         | 4608/50176 [00:31<05:17, 143.43it/s]
 10%|▉         | 4864/50176 [00:33<05:11, 145.68it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.29it/s]
 11%|█         | 5376/50176 [00:36<05:04, 147.16it/s]
 11%|█         | 5632/50176 [00:38<05:00, 148.16it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.18it/s]
 12%|█▏        | 6144/50176 [00:42<05:24, 135.62it/s]
 13%|█▎        | 6400/50176 [00:44<05:14, 139.40it/s]
 13%|█▎        | 6656/50176 [00:46<05:07, 141.61it/s]
 14%|█▍        | 6912/50176 [00:47<05:05, 141.58it/s]
 14%|█▍        | 7168/50176 [00:49<05:00, 142.94it/s]
 15%|█▍        | 7424/50176 [00:51<04:53, 145.79it/s]
 15%|█▌        | 7680/50176 [00:53<04:54, 144.44it/s]
 16%|█▌        | 7936/50176 [00:54<04:45, 148.01it/s]
 16%|█▋        | 8192/50176 [00:56<04:45, 147.11it/s]
 17%|█▋        | 8448/50176 [00:58<04:41, 148.41it/s]
 17%|█▋        | 8704/50176 [00:59<04:38, 149.11it/s]
 18%|█▊        | 8960/50176 [01:01<04:46, 143.67it/s]
 18%|█▊        | 9216/50176 [01:03<04:37, 147.72it/s]
 19%|█▉        | 9472/50176 [01:05<04:36, 147.29it/s]
 19%|█▉        | 9728/50176 [01:06<04:39, 144.52it/s]
 20%|█▉        | 9984/50176 [01:08<04:31, 147.95it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.64it/s]
 21%|██        | 10496/50176 [01:11<04:23, 150.34it/s]
 21%|██▏       | 10752/50176 [01:13<04:31, 145.30it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 147.97it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.40it/s]
 23%|██▎       | 11520/50176 [01:18<04:16, 150.71it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.73it/s]
 24%|██▍       | 12032/50176 [01:22<04:13, 150.41it/s]
 24%|██▍       | 12288/50176 [01:23<04:11, 150.73it/s]
 25%|██▌       | 12544/50176 [01:25<04:08, 151.40it/s]
 26%|██▌       | 12800/50176 [01:27<04:06, 151.56it/s]
 26%|██▌       | 13056/50176 [01:28<04:02, 152.76it/s]
 27%|██▋       | 13312/50176 [01:30<04:05, 150.03it/s]
 27%|██▋       | 13568/50176 [01:32<04:12, 144.84it/s]
 28%|██▊       | 13824/50176 [01:34<04:16, 141.53it/s]
 28%|██▊       | 14080/50176 [01:36<04:13, 142.23it/s]
 29%|██▊       | 14336/50176 [01:38<04:18, 138.91it/s]
 29%|██▉       | 14592/50176 [01:39<04:06, 144.44it/s]
 30%|██▉       | 14848/50176 [01:42<04:18, 136.68it/s]
 30%|███       | 15104/50176 [01:43<04:19, 135.13it/s]
 31%|███       | 15360/50176 [01:45<04:10, 139.09it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.21it/s]
 32%|███▏      | 15872/50176 [01:49<04:10, 137.20it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.89it/s]
 33%|███▎      | 16384/50176 [01:52<04:01, 139.87it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 135.03it/s]
 34%|███▎      | 16896/50176 [01:56<03:57, 140.04it/s]
 34%|███▍      | 17152/50176 [01:58<03:50, 143.09it/s]
 35%|███▍      | 17408/50176 [02:00<03:45, 145.03it/s]
 35%|███▌      | 17664/50176 [02:01<03:41, 146.92it/s]
 36%|███▌      | 17920/50176 [02:03<03:36, 149.08it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.72it/s]
 37%|███▋      | 18432/50176 [02:06<03:34, 147.91it/s]
 37%|███▋      | 18688/50176 [02:08<03:33, 147.29it/s]
 38%|███▊      | 18944/50176 [02:10<03:39, 142.26it/s]
 38%|███▊      | 19200/50176 [02:12<03:50, 134.25it/s]
 39%|███▉      | 19456/50176 [02:14<03:53, 131.48it/s]
 39%|███▉      | 19712/50176 [02:16<03:51, 131.70it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.71it/s]
 40%|████      | 20224/50176 [02:20<03:31, 141.31it/s]
 41%|████      | 20480/50176 [02:21<03:26, 143.90it/s]
 41%|████▏     | 20736/50176 [02:23<03:24, 144.07it/s]
 42%|████▏     | 20992/50176 [02:25<03:18, 147.29it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 142.08it/s]
 43%|████▎     | 21504/50176 [02:28<03:15, 146.85it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.43it/s]
 44%|████▍     | 22016/50176 [02:32<03:05, 151.66it/s]
 44%|████▍     | 22272/50176 [02:33<03:04, 150.92it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.61it/s]
 45%|████▌     | 22784/50176 [02:37<03:07, 145.85it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 143.90it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.52it/s]
 47%|████▋     | 23552/50176 [02:43<03:14, 136.92it/s]
 47%|████▋     | 23808/50176 [02:44<03:06, 141.02it/s]
 48%|████▊     | 24064/50176 [02:46<03:04, 141.87it/s]
 48%|████▊     | 24320/50176 [02:48<03:01, 142.44it/s]
 49%|████▉     | 24576/50176 [02:50<02:56, 145.21it/s]
 49%|████▉     | 24832/50176 [02:51<02:50, 148.70it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.94it/s]
 51%|█████     | 25344/50176 [02:55<02:45, 149.73it/s]
 51%|█████     | 25600/50176 [02:57<02:48, 146.27it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.54it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.35it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.07it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 145.41it/s]
 54%|█████▎    | 26880/50176 [03:05<02:40, 145.23it/s]
 54%|█████▍    | 27136/50176 [03:07<02:37, 146.12it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.62it/s]
 55%|█████▌    | 27648/50176 [03:10<02:26, 153.27it/s]
 56%|█████▌    | 27904/50176 [03:12<02:36, 142.67it/s]
 56%|█████▌    | 28160/50176 [03:14<02:33, 143.59it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.41it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.69it/s]
 58%|█████▊    | 28928/50176 [03:19<02:28, 143.23it/s]
 58%|█████▊    | 29184/50176 [03:21<02:23, 145.98it/s]
 59%|█████▊    | 29440/50176 [03:23<02:30, 138.01it/s]
 59%|█████▉    | 29696/50176 [03:25<02:30, 136.10it/s]
 60%|█████▉    | 29952/50176 [03:27<02:29, 135.36it/s]
 60%|██████    | 30208/50176 [03:29<02:23, 138.98it/s]
 61%|██████    | 30464/50176 [03:30<02:15, 145.40it/s]
 61%|██████    | 30720/50176 [03:32<02:14, 144.19it/s]
 62%|██████▏   | 30976/50176 [03:34<02:10, 146.99it/s]
 62%|██████▏   | 31232/50176 [03:36<02:06, 149.17it/s]
 63%|██████▎   | 31488/50176 [03:37<02:04, 150.41it/s]
 63%|██████▎   | 31744/50176 [03:39<02:01, 151.32it/s]
 64%|██████▍   | 32000/50176 [03:41<02:01, 149.74it/s]
 64%|██████▍   | 32256/50176 [03:42<01:58, 150.67it/s]
 65%|██████▍   | 32512/50176 [03:44<02:00, 146.01it/s]
 65%|██████▌   | 32768/50176 [03:46<01:56, 149.31it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 147.66it/s]
 66%|██████▋   | 33280/50176 [03:49<01:53, 149.01it/s]
 67%|██████▋   | 33536/50176 [03:51<01:54, 145.23it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 148.12it/s]
 68%|██████▊   | 34048/50176 [03:54<01:45, 152.25it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 154.48it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 148.58it/s]
 69%|██████▉   | 34816/50176 [04:00<01:43, 148.61it/s]
 70%|██████▉   | 35072/50176 [04:01<01:42, 146.80it/s]
 70%|███████   | 35328/50176 [04:03<01:38, 150.05it/s]
 71%|███████   | 35584/50176 [04:05<01:38, 147.85it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.46it/s]
 72%|███████▏  | 36096/50176 [04:08<01:36, 145.70it/s]
 72%|███████▏  | 36352/50176 [04:10<01:36, 143.47it/s]
 73%|███████▎  | 36608/50176 [04:12<01:32, 146.03it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.82it/s]
 74%|███████▍  | 37120/50176 [04:15<01:29, 146.06it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 150.23it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.54it/s]
 76%|███████▌  | 37888/50176 [04:20<01:22, 149.12it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 149.89it/s]
 77%|███████▋  | 38400/50176 [04:24<01:19, 149.04it/s]
 77%|███████▋  | 38656/50176 [04:26<01:17, 147.80it/s]
 78%|███████▊  | 38912/50176 [04:27<01:17, 145.79it/s]
 78%|███████▊  | 39168/50176 [04:29<01:17, 141.92it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.81it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.32it/s]
 80%|███████▉  | 39936/50176 [04:34<01:09, 146.60it/s]
 80%|████████  | 40192/50176 [04:36<01:06, 149.64it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 143.38it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.74it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.58it/s]
 82%|████████▏ | 41216/50176 [04:43<01:01, 146.52it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.06it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.02it/s]
 84%|████████▎ | 41984/50176 [04:48<00:54, 150.91it/s]
 84%|████████▍ | 42240/50176 [04:50<00:53, 147.97it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.88it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 148.92it/s]
 86%|████████▌ | 43008/50176 [04:55<00:47, 150.44it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.54it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.85it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.65it/s]
 88%|████████▊ | 44032/50176 [05:02<00:41, 148.30it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 149.98it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.51it/s]
 89%|████████▉ | 44800/50176 [05:07<00:36, 145.77it/s]
 90%|████████▉ | 45056/50176 [05:09<00:34, 147.53it/s]
 90%|█████████ | 45312/50176 [05:11<00:34, 141.22it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 145.97it/s]
 91%|█████████▏| 45824/50176 [05:15<00:30, 144.69it/s]
 92%|█████████▏| 46080/50176 [05:16<00:28, 144.97it/s]
 92%|█████████▏| 46336/50176 [05:18<00:26, 145.62it/s]
 93%|█████████▎| 46592/50176 [05:20<00:24, 147.03it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 146.04it/s]
 94%|█████████▍| 47104/50176 [05:23<00:21, 141.85it/s]
 94%|█████████▍| 47360/50176 [05:25<00:19, 144.03it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.29it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 144.49it/s]
 96%|█████████▌| 48128/50176 [05:30<00:13, 147.74it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.82it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.35it/s]
 97%|█████████▋| 48896/50176 [05:35<00:08, 150.62it/s]
 98%|█████████▊| 49152/50176 [05:37<00:06, 147.29it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.63it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.28it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 137.62it/s]
-2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.3597	imagenet-zeroshot-val-top5: 0.6317

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_30.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:13:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:13:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:13:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt).
-2022-12-21,10:13:41 | INFO | Model:
-2022-12-21,10:13:41 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:13:41 | INFO | Params:
-2022-12-21,10:13:41 | INFO |   batch_size: 256
-2022-12-21,10:13:41 | INFO |   beta1: 0.9
-2022-12-21,10:13:41 | INFO |   beta2: 0.98
-2022-12-21,10:13:41 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/checkpoints
-2022-12-21,10:13:41 | INFO |   copy_codebase: False
-2022-12-21,10:13:41 | INFO |   csv_caption_key: title
-2022-12-21,10:13:41 | INFO |   csv_img_key: filepath
-2022-12-21,10:13:41 | INFO |   csv_separator: 	
-2022-12-21,10:13:41 | INFO |   dataset_resampled: False
-2022-12-21,10:13:41 | INFO |   dataset_type: auto
-2022-12-21,10:13:41 | INFO |   ddp_static_graph: False
-2022-12-21,10:13:41 | INFO |   debug: False
-2022-12-21,10:13:41 | INFO |   device: cuda:0
-2022-12-21,10:13:41 | INFO |   dist_backend: nccl
-2022-12-21,10:13:41 | INFO |   dist_url: env://
-2022-12-21,10:13:41 | INFO |   distributed: False
-2022-12-21,10:13:41 | INFO |   epochs: 32
-2022-12-21,10:13:41 | INFO |   eps: 1e-06
-2022-12-21,10:13:41 | INFO |   force_quick_gelu: False
-2022-12-21,10:13:41 | INFO |   gather_with_grad: False
-2022-12-21,10:13:41 | INFO |   grad_checkpointing: False
-2022-12-21,10:13:41 | INFO |   horovod: False
-2022-12-21,10:13:41 | INFO |   image_mean: None
-2022-12-21,10:13:41 | INFO |   image_std: None
-2022-12-21,10:13:41 | INFO |   imagenet_v2: None
-2022-12-21,10:13:41 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:13:41 | INFO |   local_loss: False
-2022-12-21,10:13:41 | INFO |   local_rank: 0
-2022-12-21,10:13:41 | INFO |   lock_image: False
-2022-12-21,10:13:41 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:13:41 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:13:41 | INFO |   log_level: 20
-2022-12-21,10:13:41 | INFO |   log_local: False
-2022-12-21,10:13:41 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_30.pt/out.log
-2022-12-21,10:13:41 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:13:41 | INFO |   lr: 0.0005
-2022-12-21,10:13:41 | INFO |   model: ViT-B-16
-2022-12-21,10:13:41 | INFO |   name: epoch_30.pt
-2022-12-21,10:13:41 | INFO |   no_set_device_rank: False
-2022-12-21,10:13:41 | INFO |   norm_gradient_clip: None
-2022-12-21,10:13:41 | INFO |   precision: amp_bfloat16
-2022-12-21,10:13:41 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_30.pt
-2022-12-21,10:13:41 | INFO |   pretrained_image: False
-2022-12-21,10:13:41 | INFO |   rank: 0
-2022-12-21,10:13:41 | INFO |   report_to: 
-2022-12-21,10:13:41 | INFO |   resume: None
-2022-12-21,10:13:41 | INFO |   save_frequency: 1
-2022-12-21,10:13:41 | INFO |   save_most_recent: False
-2022-12-21,10:13:41 | INFO |   seed: 0
-2022-12-21,10:13:41 | INFO |   skip_scheduler: False
-2022-12-21,10:13:41 | INFO |   tensorboard: False
-2022-12-21,10:13:41 | INFO |   tensorboard_path: 
-2022-12-21,10:13:41 | INFO |   torchscript: False
-2022-12-21,10:13:41 | INFO |   trace: False
-2022-12-21,10:13:41 | INFO |   train_data: None
-2022-12-21,10:13:41 | INFO |   train_num_samples: None
-2022-12-21,10:13:41 | INFO |   use_bn_sync: False
-2022-12-21,10:13:41 | INFO |   val_data: None
-2022-12-21,10:13:41 | INFO |   val_frequency: 1
-2022-12-21,10:13:41 | INFO |   val_num_samples: None
-2022-12-21,10:13:41 | INFO |   wandb: False
-2022-12-21,10:13:41 | INFO |   wandb_notes: 
-2022-12-21,10:13:41 | INFO |   warmup: 10000
-2022-12-21,10:13:41 | INFO |   wd: 0.2
-2022-12-21,10:13:41 | INFO |   workers: 1
-2022-12-21,10:13:41 | INFO |   world_size: 1
-2022-12-21,10:13:41 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:13:41 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:13:41 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:24,  1.60it/s]
  1%|          | 6/1000 [00:00<01:33, 10.58it/s]
  1%|          | 12/1000 [00:00<00:48, 20.21it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.41it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.59it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.30it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.65it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.91it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.36it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.01it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.31it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.25it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.91it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.83it/s]
  9%|▉         | 90/1000 [00:02<00:17, 50.60it/s]
 10%|▉         | 96/1000 [00:02<00:17, 52.71it/s]
 10%|█         | 102/1000 [00:02<00:16, 54.22it/s]
 11%|█         | 108/1000 [00:02<00:16, 55.42it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 56.26it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 56.73it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 56.99it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.00it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.22it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.44it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.65it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.77it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.72it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.61it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.66it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.65it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.59it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.51it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.45it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.28it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.18it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.27it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.48it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.47it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.39it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.47it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.58it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.70it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.74it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.57it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.60it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.80it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.82it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.03it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.00it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.98it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.11it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.11it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.00it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.81it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.90it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.11it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.87it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.86it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.92it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.52it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.59it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.73it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.87it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.96it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.95it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.97it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.98it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.11it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.11it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.89it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.97it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.01it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.99it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.95it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.07it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.97it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.96it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.96it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.95it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.87it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.99it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.08it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.00it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.07it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.10it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.01it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.07it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.96it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.90it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.03it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.00it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.91it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.05it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.98it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.98it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.10it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.07it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.16it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.98it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.92it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.07it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.01it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.07it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.13it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.08it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.12it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.11it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.12it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.00it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.09it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.04it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.71it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.83it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.83it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.90it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.94it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.95it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.04it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.06it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.88it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.00it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.96it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.02it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.07it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.89it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.89it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.95it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.92it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.91it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.93it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.04it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.18it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.20it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.12it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.18it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.12it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.11it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.10it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.07it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.94it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.05it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.05it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.00it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.03it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.01it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.10it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.15it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.12it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.84it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.87it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.90it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.02it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.16it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.21it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.21it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.18it/s]
-2022-12-21,10:13:59 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:03, 137.32it/s]
  1%|          | 512/50176 [00:03<05:12, 158.76it/s]
  2%|▏         | 768/50176 [00:04<05:08, 159.96it/s]
  2%|▏         | 1024/50176 [00:06<05:17, 154.72it/s]
  3%|▎         | 1280/50176 [00:08<05:25, 150.30it/s]
  3%|▎         | 1536/50176 [00:10<05:45, 140.87it/s]
  4%|▎         | 1792/50176 [00:12<05:45, 139.84it/s]
  4%|▍         | 2048/50176 [00:14<05:39, 141.73it/s]
  5%|▍         | 2304/50176 [00:15<05:39, 140.94it/s]
  5%|▌         | 2560/50176 [00:17<05:36, 141.49it/s]
  6%|▌         | 2816/50176 [00:19<05:39, 139.53it/s]
  6%|▌         | 3072/50176 [00:21<05:32, 141.73it/s]
  7%|▋         | 3328/50176 [00:23<05:28, 142.48it/s]
  7%|▋         | 3584/50176 [00:24<05:22, 144.44it/s]
  8%|▊         | 3840/50176 [00:26<05:31, 139.67it/s]
  8%|▊         | 4096/50176 [00:28<05:29, 139.90it/s]
  9%|▊         | 4352/50176 [00:30<05:24, 141.32it/s]
  9%|▉         | 4608/50176 [00:32<05:17, 143.42it/s]
 10%|▉         | 4864/50176 [00:33<05:11, 145.51it/s]
 10%|█         | 5120/50176 [00:35<05:07, 146.39it/s]
 11%|█         | 5376/50176 [00:37<05:04, 147.29it/s]
 11%|█         | 5632/50176 [00:38<05:00, 148.36it/s]
 12%|█▏        | 5888/50176 [00:40<04:58, 148.13it/s]
 12%|█▏        | 6144/50176 [00:42<05:23, 135.98it/s]
 13%|█▎        | 6400/50176 [00:44<05:14, 139.18it/s]
 13%|█▎        | 6656/50176 [00:46<05:08, 141.13it/s]
 14%|█▍        | 6912/50176 [00:48<05:05, 141.67it/s]
 14%|█▍        | 7168/50176 [00:49<05:00, 143.29it/s]
 15%|█▍        | 7424/50176 [00:51<04:53, 145.67it/s]
 15%|█▌        | 7680/50176 [00:53<04:53, 144.62it/s]
 16%|█▌        | 7936/50176 [00:55<04:44, 148.29it/s]
 16%|█▋        | 8192/50176 [00:56<04:45, 147.01it/s]
 17%|█▋        | 8448/50176 [00:58<04:41, 148.46it/s]
 17%|█▋        | 8704/50176 [01:00<04:38, 148.78it/s]
 18%|█▊        | 8960/50176 [01:02<04:46, 143.87it/s]
 18%|█▊        | 9216/50176 [01:03<04:37, 147.51it/s]
 19%|█▉        | 9472/50176 [01:05<04:36, 147.35it/s]
 19%|█▉        | 9728/50176 [01:07<04:41, 143.82it/s]
 20%|█▉        | 9984/50176 [01:08<04:30, 148.36it/s]
 20%|██        | 10240/50176 [01:10<04:26, 149.78it/s]
 21%|██        | 10496/50176 [01:12<04:23, 150.39it/s]
 21%|██▏       | 10752/50176 [01:14<04:30, 145.69it/s]
 22%|██▏       | 11008/50176 [01:15<04:24, 147.93it/s]
 22%|██▏       | 11264/50176 [01:17<04:22, 148.38it/s]
 23%|██▎       | 11520/50176 [01:19<04:16, 150.54it/s]
 23%|██▎       | 11776/50176 [01:20<04:16, 149.87it/s]
 24%|██▍       | 12032/50176 [01:22<04:13, 150.43it/s]
 24%|██▍       | 12288/50176 [01:24<04:12, 149.86it/s]
 25%|██▌       | 12544/50176 [01:26<04:08, 151.60it/s]
 26%|██▌       | 12800/50176 [01:27<04:06, 151.55it/s]
 26%|██▌       | 13056/50176 [01:29<04:03, 152.76it/s]
 27%|██▋       | 13312/50176 [01:31<04:05, 150.05it/s]
 27%|██▋       | 13568/50176 [01:33<04:13, 144.49it/s]
 28%|██▊       | 13824/50176 [01:34<04:16, 141.94it/s]
 28%|██▊       | 14080/50176 [01:36<04:13, 142.34it/s]
 29%|██▊       | 14336/50176 [01:38<04:17, 139.18it/s]
 29%|██▉       | 14592/50176 [01:40<04:06, 144.65it/s]
 30%|██▉       | 14848/50176 [01:42<04:18, 136.66it/s]
 30%|███       | 15104/50176 [01:44<04:19, 135.35it/s]
 31%|███       | 15360/50176 [01:46<04:10, 138.91it/s]
 31%|███       | 15616/50176 [01:47<04:04, 141.33it/s]
 32%|███▏      | 15872/50176 [01:49<04:10, 137.03it/s]
 32%|███▏      | 16128/50176 [01:51<04:03, 139.70it/s]
 33%|███▎      | 16384/50176 [01:53<04:01, 139.72it/s]
 33%|███▎      | 16640/50176 [01:55<04:08, 135.02it/s]
 34%|███▎      | 16896/50176 [01:57<03:57, 139.95it/s]
 34%|███▍      | 17152/50176 [01:58<03:50, 143.12it/s]
 35%|███▍      | 17408/50176 [02:00<03:46, 144.96it/s]
 35%|███▌      | 17664/50176 [02:02<03:41, 146.90it/s]
 36%|███▌      | 17920/50176 [02:03<03:36, 149.13it/s]
 36%|███▌      | 18176/50176 [02:05<03:36, 147.99it/s]
 37%|███▋      | 18432/50176 [02:07<03:34, 148.06it/s]
 37%|███▋      | 18688/50176 [02:09<03:33, 147.41it/s]
 38%|███▊      | 18944/50176 [02:11<03:39, 142.24it/s]
 38%|███▊      | 19200/50176 [02:13<03:50, 134.25it/s]
 39%|███▉      | 19456/50176 [02:15<03:53, 131.43it/s]
 39%|███▉      | 19712/50176 [02:17<03:50, 132.01it/s]
 40%|███▉      | 19968/50176 [02:18<03:44, 134.79it/s]
 40%|████      | 20224/50176 [02:20<03:32, 141.25it/s]
 41%|████      | 20480/50176 [02:22<03:26, 143.73it/s]
 41%|████▏     | 20736/50176 [02:24<03:24, 143.89it/s]
 42%|████▏     | 20992/50176 [02:25<03:18, 147.21it/s]
 42%|████▏     | 21248/50176 [02:27<03:23, 142.27it/s]
 43%|████▎     | 21504/50176 [02:29<03:15, 146.73it/s]
 43%|████▎     | 21760/50176 [02:30<03:11, 148.48it/s]
 44%|████▍     | 22016/50176 [02:32<03:05, 151.52it/s]
 44%|████▍     | 22272/50176 [02:34<03:05, 150.75it/s]
 45%|████▍     | 22528/50176 [02:35<03:02, 151.55it/s]
 45%|████▌     | 22784/50176 [02:37<03:08, 145.62it/s]
 46%|████▌     | 23040/50176 [02:39<03:08, 144.00it/s]
 46%|████▋     | 23296/50176 [02:41<03:12, 139.44it/s]
 47%|████▋     | 23552/50176 [02:43<03:13, 137.42it/s]
 47%|████▋     | 23808/50176 [02:45<03:07, 140.72it/s]
 48%|████▊     | 24064/50176 [02:47<03:03, 141.99it/s]
 48%|████▊     | 24320/50176 [02:48<03:01, 142.55it/s]
 49%|████▉     | 24576/50176 [02:50<02:56, 145.15it/s]
 49%|████▉     | 24832/50176 [02:52<02:50, 148.42it/s]
 50%|█████     | 25088/50176 [02:53<02:48, 148.81it/s]
 51%|█████     | 25344/50176 [02:55<02:46, 149.53it/s]
 51%|█████     | 25600/50176 [02:57<02:48, 146.19it/s]
 52%|█████▏    | 25856/50176 [02:58<02:41, 150.50it/s]
 52%|█████▏    | 26112/50176 [03:00<02:42, 148.41it/s]
 53%|█████▎    | 26368/50176 [03:02<02:45, 144.00it/s]
 53%|█████▎    | 26624/50176 [03:04<02:41, 145.42it/s]
 54%|█████▎    | 26880/50176 [03:06<02:40, 145.16it/s]
 54%|█████▍    | 27136/50176 [03:07<02:38, 145.78it/s]
 55%|█████▍    | 27392/50176 [03:09<02:35, 146.85it/s]
 55%|█████▌    | 27648/50176 [03:11<02:26, 153.45it/s]
 56%|█████▌    | 27904/50176 [03:13<02:36, 142.51it/s]
 56%|█████▌    | 28160/50176 [03:14<02:32, 144.06it/s]
 57%|█████▋    | 28416/50176 [03:16<02:30, 144.41it/s]
 57%|█████▋    | 28672/50176 [03:18<02:32, 140.96it/s]
 58%|█████▊    | 28928/50176 [03:20<02:28, 143.16it/s]
 58%|█████▊    | 29184/50176 [03:22<02:23, 146.18it/s]
 59%|█████▊    | 29440/50176 [03:24<02:30, 138.09it/s]
 59%|█████▉    | 29696/50176 [03:26<02:31, 135.56it/s]
 60%|█████▉    | 29952/50176 [03:27<02:28, 135.87it/s]
 60%|██████    | 30208/50176 [03:29<02:23, 138.83it/s]
 61%|██████    | 30464/50176 [03:31<02:15, 145.34it/s]
 61%|██████    | 30720/50176 [03:33<02:14, 144.23it/s]
 62%|██████▏   | 30976/50176 [03:34<02:10, 146.90it/s]
 62%|██████▏   | 31232/50176 [03:36<02:07, 149.07it/s]
 63%|██████▎   | 31488/50176 [03:38<02:03, 150.74it/s]
 63%|██████▎   | 31744/50176 [03:39<02:02, 150.71it/s]
 64%|██████▍   | 32000/50176 [03:41<02:01, 149.81it/s]
 64%|██████▍   | 32256/50176 [03:43<01:59, 150.05it/s]
 65%|██████▍   | 32512/50176 [03:45<02:00, 146.14it/s]
 65%|██████▌   | 32768/50176 [03:46<01:56, 149.46it/s]
 66%|██████▌   | 33024/50176 [03:48<01:56, 147.67it/s]
 66%|██████▋   | 33280/50176 [03:50<01:53, 148.99it/s]
 67%|██████▋   | 33536/50176 [03:52<01:55, 144.53it/s]
 67%|██████▋   | 33792/50176 [03:53<01:50, 148.38it/s]
 68%|██████▊   | 34048/50176 [03:55<01:45, 152.49it/s]
 68%|██████▊   | 34304/50176 [03:56<01:42, 154.56it/s]
 69%|██████▉   | 34560/50176 [03:58<01:45, 147.71it/s]
 69%|██████▉   | 34816/50176 [04:00<01:43, 148.98it/s]
 70%|██████▉   | 35072/50176 [04:02<01:42, 147.13it/s]
 70%|███████   | 35328/50176 [04:03<01:38, 150.20it/s]
 71%|███████   | 35584/50176 [04:05<01:38, 147.65it/s]
 71%|███████▏  | 35840/50176 [04:07<01:40, 142.83it/s]
 72%|███████▏  | 36096/50176 [04:09<01:36, 145.47it/s]
 72%|███████▏  | 36352/50176 [04:11<01:36, 143.59it/s]
 73%|███████▎  | 36608/50176 [04:12<01:33, 145.78it/s]
 73%|███████▎  | 36864/50176 [04:14<01:32, 143.98it/s]
 74%|███████▍  | 37120/50176 [04:16<01:29, 146.09it/s]
 74%|███████▍  | 37376/50176 [04:17<01:25, 150.32it/s]
 75%|███████▌  | 37632/50176 [04:19<01:22, 151.24it/s]
 76%|███████▌  | 37888/50176 [04:21<01:22, 149.27it/s]
 76%|███████▌  | 38144/50176 [04:22<01:20, 150.06it/s]
 77%|███████▋  | 38400/50176 [04:24<01:18, 149.14it/s]
 77%|███████▋  | 38656/50176 [04:26<01:17, 147.77it/s]
 78%|███████▊  | 38912/50176 [04:28<01:17, 145.84it/s]
 78%|███████▊  | 39168/50176 [04:30<01:17, 141.86it/s]
 79%|███████▊  | 39424/50176 [04:31<01:14, 143.84it/s]
 79%|███████▉  | 39680/50176 [04:33<01:12, 145.28it/s]
 80%|███████▉  | 39936/50176 [04:35<01:09, 147.17it/s]
 80%|████████  | 40192/50176 [04:37<01:06, 149.46it/s]
 81%|████████  | 40448/50176 [04:38<01:07, 144.00it/s]
 81%|████████  | 40704/50176 [04:40<01:08, 138.32it/s]
 82%|████████▏ | 40960/50176 [04:42<01:04, 143.36it/s]
 82%|████████▏ | 41216/50176 [04:44<01:01, 145.76it/s]
 83%|████████▎ | 41472/50176 [04:45<00:58, 148.48it/s]
 83%|████████▎ | 41728/50176 [04:47<00:55, 151.06it/s]
 84%|████████▎ | 41984/50176 [04:49<00:54, 150.95it/s]
 84%|██████���█▍ | 42240/50176 [04:51<00:53, 147.43it/s]
 85%|████████▍ | 42496/50176 [04:52<00:51, 149.87it/s]
 85%|████████▌ | 42752/50176 [04:54<00:49, 149.09it/s]
 86%|████████▌ | 43008/50176 [04:56<00:47, 150.65it/s]
 86%|████████▌ | 43264/50176 [04:57<00:45, 150.55it/s]
 87%|████████▋ | 43520/50176 [04:59<00:44, 148.92it/s]
 87%|████████▋ | 43776/50176 [05:01<00:43, 146.61it/s]
 88%|████████▊ | 44032/50176 [05:03<00:41, 148.34it/s]
 88%|████████▊ | 44288/50176 [05:04<00:39, 150.03it/s]
 89%|████████▉ | 44544/50176 [05:06<00:38, 147.53it/s]
 89%|████████▉ | 44800/50176 [05:08<00:36, 145.72it/s]
 90%|████████▉ | 45056/50176 [05:10<00:34, 147.58it/s]
 90%|█████████ | 45312/50176 [05:12<00:34, 141.15it/s]
 91%|█████████ | 45568/50176 [05:13<00:31, 146.05it/s]
 91%|█████████▏| 45824/50176 [05:15<00:30, 144.99it/s]
 92%|█████████▏| 46080/50176 [05:17<00:28, 144.91it/s]
 92%|█████████▏| 46336/50176 [05:18<00:26, 145.55it/s]
 93%|█████████▎| 46592/50176 [05:20<00:24, 147.10it/s]
 93%|█████████▎| 46848/50176 [05:22<00:22, 145.95it/s]
 94%|█████████▍| 47104/50176 [05:24<00:21, 142.33it/s]
 94%|█████████▍| 47360/50176 [05:26<00:19, 144.12it/s]
 95%|█████████▍| 47616/50176 [05:27<00:17, 142.46it/s]
 95%|█████████▌| 47872/50176 [05:29<00:15, 145.20it/s]
 96%|█████████▌| 48128/50176 [05:31<00:13, 146.96it/s]
 96%|█████████▋| 48384/50176 [05:32<00:12, 148.57it/s]
 97%|█████████▋| 48640/50176 [05:34<00:10, 150.13it/s]
 97%|█████████▋| 48896/50176 [05:36<00:08, 150.44it/s]
 98%|█████████▊| 49152/50176 [05:38<00:06, 147.13it/s]
 98%|█████████▊| 49408/50176 [05:39<00:05, 145.51it/s]
 99%|█████████▉| 49664/50176 [05:41<00:03, 145.31it/s]
 99%|█████████▉| 49920/50176 [05:43<00:01, 137.56it/s]
-2022-12-21,10:19:44 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:19:44 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5460	imagenet-zeroshot-val-top5: 0.8225

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_31.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt).
-2022-12-21,10:19:58 | INFO | Model:
-2022-12-21,10:19:58 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:19:58 | INFO | Params:
-2022-12-21,10:19:58 | INFO |   batch_size: 256
-2022-12-21,10:19:58 | INFO |   beta1: 0.9
-2022-12-21,10:19:58 | INFO |   beta2: 0.98
-2022-12-21,10:19:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/checkpoints
-2022-12-21,10:19:58 | INFO |   copy_codebase: False
-2022-12-21,10:19:58 | INFO |   csv_caption_key: title
-2022-12-21,10:19:58 | INFO |   csv_img_key: filepath
-2022-12-21,10:19:58 | INFO |   csv_separator: 	
-2022-12-21,10:19:58 | INFO |   dataset_resampled: False
-2022-12-21,10:19:58 | INFO |   dataset_type: auto
-2022-12-21,10:19:58 | INFO |   ddp_static_graph: False
-2022-12-21,10:19:58 | INFO |   debug: False
-2022-12-21,10:19:58 | INFO |   device: cuda:0
-2022-12-21,10:19:58 | INFO |   dist_backend: nccl
-2022-12-21,10:19:58 | INFO |   dist_url: env://
-2022-12-21,10:19:58 | INFO |   distributed: False
-2022-12-21,10:19:58 | INFO |   epochs: 32
-2022-12-21,10:19:58 | INFO |   eps: 1e-06
-2022-12-21,10:19:58 | INFO |   force_quick_gelu: False
-2022-12-21,10:19:58 | INFO |   gather_with_grad: False
-2022-12-21,10:19:58 | INFO |   grad_checkpointing: False
-2022-12-21,10:19:58 | INFO |   horovod: False
-2022-12-21,10:19:58 | INFO |   image_mean: None
-2022-12-21,10:19:58 | INFO |   image_std: None
-2022-12-21,10:19:58 | INFO |   imagenet_v2: None
-2022-12-21,10:19:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:19:58 | INFO |   local_loss: False
-2022-12-21,10:19:58 | INFO |   local_rank: 0
-2022-12-21,10:19:58 | INFO |   lock_image: False
-2022-12-21,10:19:58 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:19:58 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:19:58 | INFO |   log_level: 20
-2022-12-21,10:19:58 | INFO |   log_local: False
-2022-12-21,10:19:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_31.pt/out.log
-2022-12-21,10:19:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:19:58 | INFO |   lr: 0.0005
-2022-12-21,10:19:58 | INFO |   model: ViT-B-16
-2022-12-21,10:19:58 | INFO |   name: epoch_31.pt
-2022-12-21,10:19:58 | INFO |   no_set_device_rank: False
-2022-12-21,10:19:58 | INFO |   norm_gradient_clip: None
-2022-12-21,10:19:58 | INFO |   precision: amp_bfloat16
-2022-12-21,10:19:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_31.pt
-2022-12-21,10:19:58 | INFO |   pretrained_image: False
-2022-12-21,10:19:58 | INFO |   rank: 0
-2022-12-21,10:19:58 | INFO |   report_to: 
-2022-12-21,10:19:58 | INFO |   resume: None
-2022-12-21,10:19:58 | INFO |   save_frequency: 1
-2022-12-21,10:19:58 | INFO |   save_most_recent: False
-2022-12-21,10:19:58 | INFO |   seed: 0
-2022-12-21,10:19:58 | INFO |   skip_scheduler: False
-2022-12-21,10:19:58 | INFO |   tensorboard: False
-2022-12-21,10:19:58 | INFO |   tensorboard_path: 
-2022-12-21,10:19:58 | INFO |   torchscript: False
-2022-12-21,10:19:58 | INFO |   trace: False
-2022-12-21,10:19:58 | INFO |   train_data: None
-2022-12-21,10:19:58 | INFO |   train_num_samples: None
-2022-12-21,10:19:58 | INFO |   use_bn_sync: False
-2022-12-21,10:19:58 | INFO |   val_data: None
-2022-12-21,10:19:58 | INFO |   val_frequency: 1
-2022-12-21,10:19:58 | INFO |   val_num_samples: None
-2022-12-21,10:19:58 | INFO |   wandb: False
-2022-12-21,10:19:58 | INFO |   wandb_notes: 
-2022-12-21,10:19:58 | INFO |   warmup: 10000
-2022-12-21,10:19:58 | INFO |   wd: 0.2
-2022-12-21,10:19:58 | INFO |   workers: 1
-2022-12-21,10:19:58 | INFO |   world_size: 1
-2022-12-21,10:19:58 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:19:58 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:19:58 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:14,  1.36it/s]
  1%|          | 6/1000 [00:00<01:48,  9.18it/s]
  1%|          | 12/1000 [00:00<00:54, 18.11it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.25it/s]
  2%|▏         | 24/1000 [00:01<00:29, 33.16it/s]
  3%|▎         | 30/1000 [00:01<00:25, 38.73it/s]
  4%|▎         | 36/1000 [00:01<00:22, 43.41it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.09it/s]
  5%|▍         | 48/1000 [00:01<00:19, 49.89it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.95it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.44it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.57it/s]
  7%|▋         | 72/1000 [00:02<00:16, 55.32it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.87it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.38it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.70it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.06it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.22it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.44it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.56it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.56it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.49it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.54it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.51it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.59it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.72it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.55it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.43it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.45it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.41it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.32it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.29it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.22it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.00it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.88it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.00it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.95it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.16it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.16it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.79it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 56.95it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.13it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.28it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.35it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.21it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.27it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.46it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.57it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.77it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.74it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.71it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.83it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.79it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.71it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.52it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.61it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.70it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.78it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.58it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.61it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.64it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.72it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.62it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.31it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.38it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.49it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.64it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.71it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.72it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.69it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.72it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.84it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.82it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.62it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.70it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.75it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.74it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.71it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.85it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.78it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.75it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.74it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.65it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.70it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.61it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.70it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.70it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.79it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.70it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.81it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.84it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.75it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.76it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.81it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.68it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.61it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.75it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.75it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.67it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.79it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.71it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.71it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.83it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.81it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.88it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.70it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.65it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.80it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.77it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.85it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.87it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.80it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.84it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.84it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.87it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.76it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.84it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.82it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.81it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.81it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.73it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.74it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.75it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.75it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.85it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.94it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.79it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.88it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.85it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.88it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.91it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.71it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.72it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.76it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.74it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.72it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.73it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.84it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.97it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.00it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.90it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.93it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.88it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.90it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.82it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.89it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.85it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.73it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.84it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.83it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.80it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.84it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.84it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.83it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.92it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.88it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.87it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.90it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.95it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.89it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.96it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.03it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.05it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.05it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.00it/s]
-2022-12-21,10:20:16 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:26, 111.68it/s]
  1%|          | 512/50176 [00:04<06:37, 125.05it/s]
  2%|▏         | 768/50176 [00:05<06:16, 131.39it/s]
  2%|▏         | 1024/50176 [00:07<06:03, 135.32it/s]
  3%|▎         | 1280/50176 [00:09<05:58, 136.47it/s]
  3%|▎         | 1536/50176 [00:11<06:09, 131.48it/s]
  4%|▎         | 1792/50176 [00:13<06:07, 131.79it/s]
  4%|▍         | 2048/50176 [00:15<05:50, 137.15it/s]
  5%|▍         | 2304/50176 [00:17<05:46, 138.20it/s]
  5%|▌         | 2560/50176 [00:18<05:40, 139.74it/s]
  6%|▌         | 2816/50176 [00:20<05:40, 139.05it/s]
  6%|▌         | 3072/50176 [00:22<05:29, 142.97it/s]
  7%|▋         | 3328/50176 [00:24<05:25, 143.78it/s]
  7%|▋         | 3584/50176 [00:26<05:22, 144.38it/s]
  8%|▊         | 3840/50176 [00:28<05:33, 138.75it/s]
  8%|▊         | 4096/50176 [00:29<05:31, 138.89it/s]
  9%|▊         | 4352/50176 [00:31<05:25, 140.75it/s]
  9%|▉         | 4608/50176 [00:33<05:21, 141.58it/s]
 10%|▉         | 4864/50176 [00:35<05:16, 143.24it/s]
 10%|█         | 5120/50176 [00:36<05:12, 143.98it/s]
 11%|█         | 5376/50176 [00:38<05:09, 144.77it/s]
 11%|█         | 5632/50176 [00:40<05:05, 145.58it/s]
 12%|█▏        | 5888/50176 [00:42<05:02, 146.52it/s]
 12%|█▏        | 6144/50176 [00:44<05:27, 134.30it/s]
 13%|█▎        | 6400/50176 [00:46<05:16, 138.29it/s]
 13%|█▎        | 6656/50176 [00:47<05:09, 140.40it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.49it/s]
 14%|█▍        | 7168/50176 [00:51<05:02, 142.28it/s]
 15%|█▍        | 7424/50176 [00:53<04:55, 144.76it/s]
 15%|█▌        | 7680/50176 [00:54<04:54, 144.33it/s]
 16%|█▌        | 7936/50176 [00:56<04:45, 147.83it/s]
 16%|█▋        | 8192/50176 [00:58<04:57, 140.99it/s]
 17%|█▋        | 8448/50176 [01:00<04:49, 144.30it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.23it/s]
 18%|█▊        | 8960/50176 [01:03<04:51, 141.62it/s]
 18%|█▊        | 9216/50176 [01:05<04:41, 145.60it/s]
 19%|█▉        | 9472/50176 [01:07<04:38, 145.90it/s]
 19%|█▉        | 9728/50176 [01:09<04:42, 143.33it/s]
 20%|█▉        | 9984/50176 [01:10<04:35, 146.03it/s]
 20%|██        | 10240/50176 [01:12<04:29, 148.21it/s]
 21%|██        | 10496/50176 [01:14<04:25, 149.22it/s]
 21%|██▏       | 10752/50176 [01:16<04:32, 144.59it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 146.97it/s]
 22%|██▏       | 11264/50176 [01:19<04:22, 148.19it/s]
 23%|██▎       | 11520/50176 [01:21<04:16, 150.60it/s]
 23%|██▎       | 11776/50176 [01:22<04:16, 149.61it/s]
 24%|██▍       | 12032/50176 [01:24<04:13, 150.46it/s]
 24%|██▍       | 12288/50176 [01:26<04:10, 151.49it/s]
 25%|██▌       | 12544/50176 [01:27<04:10, 150.06it/s]
 26%|██▌       | 12800/50176 [01:29<04:08, 150.71it/s]
 26%|██▌       | 13056/50176 [01:31<04:03, 152.22it/s]
 27%|██▋       | 13312/50176 [01:33<04:06, 149.63it/s]
 27%|██▋       | 13568/50176 [01:35<04:18, 141.70it/s]
 28%|██▊       | 13824/50176 [01:36<04:21, 138.91it/s]
 28%|██▊       | 14080/50176 [01:38<04:14, 141.67it/s]
 29%|██▊       | 14336/50176 [01:40<04:19, 138.24it/s]
 29%|██▉       | 14592/50176 [01:42<04:07, 143.84it/s]
 30%|██▉       | 14848/50176 [01:44<04:20, 135.83it/s]
 30%|███       | 15104/50176 [01:46<04:19, 135.40it/s]
 31%|███       | 15360/50176 [01:48<04:11, 138.18it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.75it/s]
 32%|███▏      | 15872/50176 [01:51<04:10, 136.97it/s]
 32%|███▏      | 16128/50176 [01:53<04:04, 139.35it/s]
 33%|███▎      | 16384/50176 [01:55<04:01, 139.75it/s]
 33%|███▎      | 16640/50176 [01:57<04:08, 135.02it/s]
 34%|███▎      | 16896/50176 [01:59<03:57, 140.12it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.90it/s]
 35%|███▍      | 17408/50176 [02:02<03:48, 143.64it/s]
 35%|███▌      | 17664/50176 [02:04<03:44, 144.70it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.29it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.65it/s]
 37%|███▋      | 18432/50176 [02:09<03:35, 147.31it/s]
 37%|███▋      | 18688/50176 [02:11<03:34, 147.02it/s]
 38%|███▊      | 18944/50176 [02:13<03:39, 142.21it/s]
 38%|███▊      | 19200/50176 [02:15<03:52, 133.47it/s]
 39%|███▉      | 19456/50176 [02:17<03:53, 131.36it/s]
 39%|███▉      | 19712/50176 [02:19<03:49, 132.86it/s]
 40%|███▉      | 19968/50176 [02:21<03:44, 134.84it/s]
 40%|████      | 20224/50176 [02:22<03:33, 140.32it/s]
 41%|████      | 20480/50176 [02:24<03:25, 144.75it/s]
 41%|████▏     | 20736/50176 [02:26<03:23, 144.59it/s]
 42%|████▏     | 20992/50176 [02:27<03:18, 147.39it/s]
 42%|████▏     | 21248/50176 [02:29<03:23, 142.19it/s]
 43%|████▎     | 21504/50176 [02:31<03:16, 145.62it/s]
 43%|████▎     | 21760/50176 [02:33<03:12, 147.65it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 151.24it/s]
 44%|████▍     | 22272/50176 [02:36<03:05, 150.53it/s]
 45%|████▍     | 22528/50176 [02:38<03:16, 140.91it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.96it/s]
 46%|████▌     | 23040/50176 [02:43<03:45, 120.38it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.46it/s]
 47%|████▋     | 23552/50176 [02:48<03:58, 111.59it/s]
 47%|████▋     | 23808/50176 [02:50<03:52, 113.63it/s]
 48%|████▊     | 24064/50176 [02:52<03:49, 113.74it/s]
 48%|████▊     | 24320/50176 [02:55<03:52, 111.18it/s]
 49%|████▉     | 24576/50176 [02:57<03:44, 114.21it/s]
 49%|████▉     | 24832/50176 [02:59<03:33, 118.82it/s]
 50%|█████     | 25088/50176 [03:01<03:30, 118.90it/s]
 51%|█████     | 25344/50176 [03:03<03:26, 120.48it/s]
 51%|█████     | 25600/50176 [03:05<03:29, 117.18it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.35it/s]
 52%|█████▏    | 26112/50176 [03:10<03:27, 115.98it/s]
 53%|█████▎    | 26368/50176 [03:12<03:29, 113.37it/s]
 53%|█████▎    | 26624/50176 [03:14<03:24, 115.27it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.33it/s]
 54%|█████▍    | 27136/50176 [03:19<03:23, 113.16it/s]
 55%|█████▍    | 27392/50176 [03:21<03:18, 114.91it/s]
 55%|█████▌    | 27648/50176 [03:23<03:03, 122.56it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.53it/s]
 56%|█████▌    | 28160/50176 [03:28<03:16, 111.77it/s]
 57%|█████▋    | 28416/50176 [03:30<03:14, 112.13it/s]
 57%|█████▋    | 28672/50176 [03:32<03:14, 110.66it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.80it/s]
 58%|█████▊    | 29184/50176 [03:37<03:06, 112.85it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.41it/s]
 59%|█████▉    | 29696/50176 [03:42<03:17, 103.83it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.50it/s]
 60%|██████    | 30208/50176 [03:47<03:05, 107.58it/s]
 61%|██████    | 30464/50176 [03:49<02:53, 113.62it/s]
 61%|██████    | 30720/50176 [03:51<02:51, 113.68it/s]
 62%|██████▏   | 30976/50176 [03:53<02:46, 115.38it/s]
 62%|██████▏   | 31232/50176 [03:55<02:44, 115.48it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.88it/s]
 63%|██████▎   | 31744/50176 [03:59<02:34, 119.05it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.31it/s]
 64%|██████▍   | 32256/50176 [04:04<02:31, 118.31it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.57it/s]
 65%|██████▌   | 32768/50176 [04:08<02:30, 115.54it/s]
 66%|██████▌   | 33024/50176 [04:11<02:29, 114.62it/s]
 66%|██████▋   | 33280/50176 [04:13<02:26, 115.32it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.78it/s]
 67%|██████▋   | 33792/50176 [04:18<02:26, 111.82it/s]
 68%|██████▊   | 34048/50176 [04:20<02:18, 116.12it/s]
 68%|██████▊   | 34304/50176 [04:22<02:14, 117.65it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.68it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.97it/s]
 70%|██████▉   | 35072/50176 [04:29<02:13, 112.89it/s]
 70%|███████   | 35328/50176 [04:31<02:11, 113.17it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.90it/s]
 71%|███████▏  | 35840/50176 [04:36<02:08, 111.41it/s]
 72%|███████▏  | 36096/50176 [04:38<02:03, 113.58it/s]
 72%|███████▏  | 36352/50176 [04:40<02:03, 112.09it/s]
 73%|███████▎  | 36608/50176 [04:43<02:03, 109.90it/s]
 73%|███████▎  | 36864/50176 [04:45<02:04, 106.93it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 109.05it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 115.03it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.65it/s]
 76%|███████▌  | 37888/50176 [04:54<01:46, 115.80it/s]
 76%|███████▌  | 38144/50176 [04:56<01:42, 117.32it/s]
 77%|███████▋  | 38400/50176 [04:58<01:44, 113.22it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 113.80it/s]
 78%|███████▊  | 38912/50176 [05:03<01:39, 112.93it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.56it/s]
 79%|███████▊  | 39424/50176 [05:07<01:34, 113.23it/s]
 79%|███████▉  | 39680/50176 [05:10<01:33, 112.55it/s]
 80%|███████▉  | 39936/50176 [05:12<01:28, 115.91it/s]
 80%|████████  | 40192/50176 [05:14<01:26, 114.89it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.52it/s]
 81%|████████  | 40704/50176 [05:19<01:28, 106.79it/s]
 82%|████████▏ | 40960/50176 [05:21<01:22, 111.46it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 110.77it/s]
 83%|████████▎ | 41472/50176 [05:26<01:20, 108.13it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 109.05it/s]
 84%|████████▎ | 41984/50176 [05:31<01:14, 109.71it/s]
 84%|██████���█▍ | 42240/50176 [05:33<01:11, 110.90it/s]
 85%|████████▍ | 42496/50176 [05:35<01:08, 111.72it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 112.97it/s]
 86%|████████▌ | 43008/50176 [05:40<01:03, 112.66it/s]
 86%|████████▌ | 43264/50176 [05:42<01:01, 113.01it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.72it/s]
 87%|████████▋ | 43776/50176 [05:47<00:58, 110.31it/s]
 88%|████████▊ | 44032/50176 [05:49<00:56, 109.00it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.18it/s]
 89%|████████▉ | 44544/50176 [05:54<00:52, 106.58it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.14it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.36it/s]
 90%|█████████ | 45312/50176 [06:01<00:45, 106.88it/s]
 91%|█████████ | 45568/50176 [06:03<00:41, 111.18it/s]
 91%|█████████▏| 45824/50176 [06:05<00:39, 108.84it/s]
 92%|█████████▏| 46080/50176 [06:08<00:38, 105.96it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.28it/s]
 93%|█████████▎| 46592/50176 [06:13<00:33, 108.52it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 106.04it/s]
 94%|█████████▍| 47104/50176 [06:18<00:30, 101.90it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.85it/s]
 95%|█████████▍| 47616/50176 [06:23<00:24, 102.71it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 103.06it/s]
 96%|█████████▌| 48128/50176 [06:28<00:19, 104.54it/s]
 96%|█████████▋| 48384/50176 [06:30<00:16, 106.89it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.22it/s]
 97%|█████████▋| 48896/50176 [06:35<00:11, 106.82it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 104.04it/s]
 98%|█████████▊| 49408/50176 [06:40<00:07, 103.74it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.74it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.54it/s] 
-2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5883	imagenet-zeroshot-val-top5: 0.8555

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_32.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt).
-2022-12-21,10:19:58 | INFO | Model:
-2022-12-21,10:19:58 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:19:58 | INFO | Params:
-2022-12-21,10:19:58 | INFO |   batch_size: 256
-2022-12-21,10:19:58 | INFO |   beta1: 0.9
-2022-12-21,10:19:58 | INFO |   beta2: 0.98
-2022-12-21,10:19:58 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/checkpoints
-2022-12-21,10:19:58 | INFO |   copy_codebase: False
-2022-12-21,10:19:58 | INFO |   csv_caption_key: title
-2022-12-21,10:19:58 | INFO |   csv_img_key: filepath
-2022-12-21,10:19:58 | INFO |   csv_separator: 	
-2022-12-21,10:19:58 | INFO |   dataset_resampled: False
-2022-12-21,10:19:58 | INFO |   dataset_type: auto
-2022-12-21,10:19:58 | INFO |   ddp_static_graph: False
-2022-12-21,10:19:58 | INFO |   debug: False
-2022-12-21,10:19:58 | INFO |   device: cuda:0
-2022-12-21,10:19:58 | INFO |   dist_backend: nccl
-2022-12-21,10:19:58 | INFO |   dist_url: env://
-2022-12-21,10:19:58 | INFO |   distributed: False
-2022-12-21,10:19:58 | INFO |   epochs: 32
-2022-12-21,10:19:58 | INFO |   eps: 1e-06
-2022-12-21,10:19:58 | INFO |   force_quick_gelu: False
-2022-12-21,10:19:58 | INFO |   gather_with_grad: False
-2022-12-21,10:19:58 | INFO |   grad_checkpointing: False
-2022-12-21,10:19:58 | INFO |   horovod: False
-2022-12-21,10:19:58 | INFO |   image_mean: None
-2022-12-21,10:19:58 | INFO |   image_std: None
-2022-12-21,10:19:58 | INFO |   imagenet_v2: None
-2022-12-21,10:19:58 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:19:58 | INFO |   local_loss: False
-2022-12-21,10:19:58 | INFO |   local_rank: 0
-2022-12-21,10:19:58 | INFO |   lock_image: False
-2022-12-21,10:19:58 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:19:58 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:19:58 | INFO |   log_level: 20
-2022-12-21,10:19:58 | INFO |   log_local: False
-2022-12-21,10:19:58 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_32.pt/out.log
-2022-12-21,10:19:58 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:19:58 | INFO |   lr: 0.0005
-2022-12-21,10:19:58 | INFO |   model: ViT-B-16
-2022-12-21,10:19:58 | INFO |   name: epoch_32.pt
-2022-12-21,10:19:58 | INFO |   no_set_device_rank: False
-2022-12-21,10:19:58 | INFO |   norm_gradient_clip: None
-2022-12-21,10:19:58 | INFO |   precision: amp_bfloat16
-2022-12-21,10:19:58 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_32.pt
-2022-12-21,10:19:58 | INFO |   pretrained_image: False
-2022-12-21,10:19:58 | INFO |   rank: 0
-2022-12-21,10:19:58 | INFO |   report_to: 
-2022-12-21,10:19:58 | INFO |   resume: None
-2022-12-21,10:19:58 | INFO |   save_frequency: 1
-2022-12-21,10:19:58 | INFO |   save_most_recent: False
-2022-12-21,10:19:58 | INFO |   seed: 0
-2022-12-21,10:19:58 | INFO |   skip_scheduler: False
-2022-12-21,10:19:58 | INFO |   tensorboard: False
-2022-12-21,10:19:58 | INFO |   tensorboard_path: 
-2022-12-21,10:19:58 | INFO |   torchscript: False
-2022-12-21,10:19:58 | INFO |   trace: False
-2022-12-21,10:19:58 | INFO |   train_data: None
-2022-12-21,10:19:58 | INFO |   train_num_samples: None
-2022-12-21,10:19:58 | INFO |   use_bn_sync: False
-2022-12-21,10:19:58 | INFO |   val_data: None
-2022-12-21,10:19:58 | INFO |   val_frequency: 1
-2022-12-21,10:19:58 | INFO |   val_num_samples: None
-2022-12-21,10:19:58 | INFO |   wandb: False
-2022-12-21,10:19:58 | INFO |   wandb_notes: 
-2022-12-21,10:19:58 | INFO |   warmup: 10000
-2022-12-21,10:19:58 | INFO |   wd: 0.2
-2022-12-21,10:19:58 | INFO |   workers: 1
-2022-12-21,10:19:58 | INFO |   world_size: 1
-2022-12-21,10:19:58 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:19:58 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:19:58 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:20,  1.25it/s]
  1%|          | 6/1000 [00:00<01:55,  8.61it/s]
  1%|          | 12/1000 [00:01<00:57, 17.15it/s]
  2%|▏         | 18/1000 [00:01<00:39, 24.83it/s]
  2%|▏         | 24/1000 [00:01<00:30, 31.64it/s]
  3%|▎         | 30/1000 [00:01<00:25, 37.33it/s]
  4%|▎         | 36/1000 [00:01<00:23, 41.91it/s]
  4%|▍         | 42/1000 [00:01<00:20, 45.88it/s]
  5%|▍         | 48/1000 [00:01<00:19, 48.94it/s]
  5%|▌         | 54/1000 [00:01<00:18, 51.21it/s]
  6%|▌         | 60/1000 [00:01<00:17, 52.88it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.13it/s]
  7%|▋         | 72/1000 [00:02<00:16, 54.98it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.58it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.19it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.55it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.97it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.13it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.43it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.56it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.54it/s]
 13%|█▎        | 126/1000 [00:03<00:15, 57.44it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.43it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.40it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.29it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.43it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.50it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.39it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.25it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.27it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.24it/s]
 19%|█▊        | 186/1000 [00:04<00:14, 57.16it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.15it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.08it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.86it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.76it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.83it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 56.85it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 56.90it/s]
 24%|██▍       | 240/1000 [00:05<00:15, 48.16it/s]
 25%|██▍       | 246/1000 [00:05<00:14, 50.52it/s]
 25%|██▌       | 252/1000 [00:05<00:14, 52.32it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 53.78it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 54.80it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 55.31it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 55.87it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 56.42it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 56.79it/s]
 29%|██▉       | 294/1000 [00:06<00:12, 57.14it/s]
 30%|███       | 300/1000 [00:06<00:12, 57.24it/s]
 31%|███       | 306/1000 [00:06<00:12, 56.97it/s]
 31%|███       | 312/1000 [00:06<00:12, 57.28it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.39it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.39it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.24it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.37it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.50it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.61it/s]
 35%|███▌      | 354/1000 [00:07<00:11, 57.43it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.44it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.55it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.65it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.54it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.21it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.25it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.36it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.52it/s]
 41%|████      | 408/1000 [00:08<00:10, 57.59it/s]
 41%|████▏     | 414/1000 [00:08<00:10, 57.54it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.57it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.59it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.70it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.62it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.40it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.48it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.54it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.53it/s]
 47%|████▋     | 468/1000 [00:09<00:09, 57.49it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.66it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.58it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.56it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.54it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.43it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.48it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.40it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.46it/s]
 52%|█████▏    | 522/1000 [00:10<00:08, 57.50it/s]
 53%|█████▎    | 528/1000 [00:10<00:08, 57.58it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.52it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.64it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.65it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.56it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.57it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.63it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.51it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.45it/s]
 58%|█████▊    | 582/1000 [00:11<00:07, 57.59it/s]
 59%|█████▉    | 588/1000 [00:11<00:07, 57.57it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.49it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.60it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.50it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.50it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.60it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.60it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.69it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.50it/s]
 64%|██████▍   | 642/1000 [00:12<00:06, 57.47it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.62it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.59it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.68it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.70it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.64it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.69it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.70it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.73it/s]
 70%|██████▉   | 696/1000 [00:13<00:05, 57.61it/s]
 70%|███████   | 702/1000 [00:13<00:05, 57.70it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.66it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.64it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.65it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.59it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.60it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.62it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.61it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.68it/s]
 76%|███████▌  | 756/1000 [00:14<00:04, 57.76it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.60it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.72it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.67it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.71it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.74it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.52it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.51it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.54it/s]
 81%|████████  | 810/1000 [00:15<00:03, 57.51it/s]
 82%|████████▏ | 816/1000 [00:15<00:03, 57.50it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.53it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.62it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.76it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.79it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.69it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.73it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.67it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.68it/s]
 87%|████████▋ | 870/1000 [00:16<00:02, 57.64it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.70it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.34it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.32it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.50it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.52it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.54it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.60it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.62it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.61it/s]
 93%|█████████▎| 930/1000 [00:17<00:01, 57.72it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.77it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.76it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.77it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.80it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.73it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.77it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.85it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.87it/s]
 98%|█████████▊| 984/1000 [00:18<00:00, 57.85it/s]
 99%|█████████▉| 990/1000 [00:18<00:00, 57.76it/s]
-2022-12-21,10:20:16 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<07:01, 118.48it/s]
  1%|          | 512/50176 [00:04<06:33, 126.27it/s]
  2%|▏         | 768/50176 [00:05<06:08, 133.90it/s]
  2%|▏         | 1024/50176 [00:07<06:01, 136.09it/s]
  3%|▎         | 1280/50176 [00:09<05:56, 137.34it/s]
  3%|▎         | 1536/50176 [00:11<06:07, 132.48it/s]
  4%|▎         | 1792/50176 [00:13<06:01, 133.98it/s]
  4%|▍         | 2048/50176 [00:15<05:51, 136.92it/s]
  5%|▍         | 2304/50176 [00:17<05:47, 137.95it/s]
  5%|▌         | 2560/50176 [00:18<05:40, 139.75it/s]
  6%|▌         | 2816/50176 [00:20<05:39, 139.60it/s]
  6%|▌         | 3072/50176 [00:22<05:30, 142.69it/s]
  7%|▋         | 3328/50176 [00:24<05:26, 143.63it/s]
  7%|▋         | 3584/50176 [00:25<05:23, 144.03it/s]
  8%|▊         | 3840/50176 [00:27<05:33, 138.79it/s]
  8%|▊         | 4096/50176 [00:29<05:30, 139.24it/s]
  9%|▊         | 4352/50176 [00:31<05:25, 140.91it/s]
  9%|▉         | 4608/50176 [00:33<05:22, 141.12it/s]
 10%|▉         | 4864/50176 [00:35<05:21, 141.11it/s]
 10%|█         | 5120/50176 [00:36<05:10, 144.88it/s]
 11%|█         | 5376/50176 [00:38<05:07, 145.49it/s]
 11%|█         | 5632/50176 [00:40<05:04, 146.22it/s]
 12%|█▏        | 5888/50176 [00:41<05:01, 146.80it/s]
 12%|█▏        | 6144/50176 [00:44<05:27, 134.27it/s]
 13%|█▎        | 6400/50176 [00:45<05:16, 138.35it/s]
 13%|█▎        | 6656/50176 [00:47<05:08, 140.84it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.54it/s]
 14%|█▍        | 7168/50176 [00:51<05:02, 142.39it/s]
 15%|█▍        | 7424/50176 [00:53<04:56, 144.28it/s]
 15%|█▌        | 7680/50176 [00:54<04:53, 144.56it/s]
 16%|█▌        | 7936/50176 [00:56<04:45, 147.85it/s]
 16%|█▋        | 8192/50176 [00:58<04:57, 140.90it/s]
 17%|█▋        | 8448/50176 [01:00<04:49, 144.31it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.39it/s]
 18%|█▊        | 8960/50176 [01:03<04:51, 141.60it/s]
 18%|█▊        | 9216/50176 [01:05<04:41, 145.73it/s]
 19%|█▉        | 9472/50176 [01:07<04:39, 145.68it/s]
 19%|█▉        | 9728/50176 [01:09<04:41, 143.48it/s]
 20%|█▉        | 9984/50176 [01:10<04:35, 145.95it/s]
 20%|██        | 10240/50176 [01:12<04:29, 148.30it/s]
 21%|██        | 10496/50176 [01:14<04:26, 149.11it/s]
 21%|██▏       | 10752/50176 [01:15<04:33, 144.14it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 147.05it/s]
 22%|██▏       | 11264/50176 [01:19<04:22, 148.44it/s]
 23%|██▎       | 11520/50176 [01:20<04:17, 149.93it/s]
 23%|██▎       | 11776/50176 [01:22<04:15, 150.29it/s]
 24%|██▍       | 12032/50176 [01:24<04:13, 150.42it/s]
 24%|██▍       | 12288/50176 [01:26<04:09, 151.63it/s]
 25%|██▌       | 12544/50176 [01:27<04:11, 149.90it/s]
 26%|██▌       | 12800/50176 [01:29<04:09, 149.85it/s]
 26%|██▌       | 13056/50176 [01:31<04:03, 152.54it/s]
 27%|██▋       | 13312/50176 [01:32<04:05, 150.45it/s]
 27%|██▋       | 13568/50176 [01:34<04:18, 141.66it/s]
 28%|██▊       | 13824/50176 [01:36<04:20, 139.56it/s]
 28%|██▊       | 14080/50176 [01:38<04:15, 141.47it/s]
 29%|██▊       | 14336/50176 [01:40<04:18, 138.76it/s]
 29%|██▉       | 14592/50176 [01:42<04:07, 143.53it/s]
 30%|██▉       | 14848/50176 [01:44<04:19, 136.08it/s]
 30%|███       | 15104/50176 [01:46<04:19, 135.12it/s]
 31%|███       | 15360/50176 [01:47<04:12, 137.97it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.56it/s]
 32%|███▏      | 15872/50176 [01:51<04:11, 136.62it/s]
 32%|███▏      | 16128/50176 [01:53<04:04, 139.53it/s]
 33%|███▎      | 16384/50176 [01:55<04:01, 139.81it/s]
 33%|███▎      | 16640/50176 [01:57<04:08, 134.93it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 140.19it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.73it/s]
 35%|███▍      | 17408/50176 [02:02<03:48, 143.24it/s]
 35%|███▌      | 17664/50176 [02:04<03:44, 144.71it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.42it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.64it/s]
 37%|███▋      | 18432/50176 [02:09<03:35, 147.27it/s]
 37%|███▋      | 18688/50176 [02:11<03:34, 146.94it/s]
 38%|███▊      | 18944/50176 [02:12<03:39, 142.28it/s]
 38%|███▊      | 19200/50176 [02:15<03:51, 133.89it/s]
 39%|███▉      | 19456/50176 [02:17<03:52, 131.94it/s]
 39%|███▉      | 19712/50176 [02:19<03:50, 131.98it/s]
 40%|███▉      | 19968/50176 [02:20<03:43, 135.43it/s]
 40%|████      | 20224/50176 [02:22<03:32, 140.99it/s]
 41%|████      | 20480/50176 [02:24<03:25, 144.30it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 144.04it/s]
 42%|████▏     | 20992/50176 [02:27<03:17, 147.47it/s]
 42%|████▏     | 21248/50176 [02:29<03:24, 141.57it/s]
 43%|████▎     | 21504/50176 [02:31<03:15, 146.98it/s]
 43%|████▎     | 21760/50176 [02:32<03:12, 147.34it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 151.24it/s]
 44%|████▍     | 22272/50176 [02:36<03:05, 150.27it/s]
 45%|████▍     | 22528/50176 [02:38<03:16, 141.05it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.88it/s]
 46%|████▌     | 23040/50176 [02:43<03:45, 120.52it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.41it/s]
 47%|████▋     | 23552/50176 [02:48<03:59, 111.07it/s]
 47%|████▋     | 23808/50176 [02:50<03:52, 113.31it/s]
 48%|████▊     | 24064/50176 [02:52<03:48, 114.03it/s]
 48%|████▊     | 24320/50176 [02:54<03:52, 111.35it/s]
 49%|████▉     | 24576/50176 [02:57<03:44, 114.10it/s]
 49%|████▉     | 24832/50176 [02:58<03:33, 118.98it/s]
 50%|█████     | 25088/50176 [03:01<03:31, 118.75it/s]
 51%|█████     | 25344/50176 [03:03<03:26, 120.30it/s]
 51%|█████     | 25600/50176 [03:05<03:30, 117.00it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.23it/s]
 52%|█████▏    | 26112/50176 [03:09<03:27, 116.16it/s]
 53%|█████▎    | 26368/50176 [03:12<03:29, 113.49it/s]
 53%|█████▎    | 26624/50176 [03:14<03:24, 115.38it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.25it/s]
 54%|█████▍    | 27136/50176 [03:19<03:23, 113.33it/s]
 55%|█████▍    | 27392/50176 [03:21<03:18, 114.80it/s]
 55%|█████▌    | 27648/50176 [03:22<03:03, 122.57it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.55it/s]
 56%|█████▌    | 28160/50176 [03:27<03:17, 111.75it/s]
 57%|█████▋    | 28416/50176 [03:30<03:14, 112.02it/s]
 57%|█████▋    | 28672/50176 [03:32<03:13, 110.85it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.74it/s]
 58%|█████▊    | 29184/50176 [03:37<03:05, 112.90it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.34it/s]
 59%|█████▉    | 29696/50176 [03:42<03:16, 104.00it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.34it/s]
 60%|██████    | 30208/50176 [03:46<03:05, 107.85it/s]
 61%|██████    | 30464/50176 [03:48<02:53, 113.36it/s]
 61%|██████    | 30720/50176 [03:51<02:50, 113.82it/s]
 62%|██████▏   | 30976/50176 [03:53<02:46, 115.14it/s]
 62%|██████▏   | 31232/50176 [03:55<02:44, 115.25it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.96it/s]
 63%|██████▎   | 31744/50176 [03:59<02:34, 119.29it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.30it/s]
 64%|██████▍   | 32256/50176 [04:04<02:31, 118.42it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.53it/s]
 65%|██████▌   | 32768/50176 [04:08<02:31, 115.18it/s]
 66%|██████▌   | 33024/50176 [04:10<02:29, 114.67it/s]
 66%|██████▋   | 33280/50176 [04:13<02:26, 115.44it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.66it/s]
 67%|██████▋   | 33792/50176 [04:17<02:26, 111.96it/s]
 68%|██████▊   | 34048/50176 [04:19<02:18, 116.14it/s]
 68%|██████▊   | 34304/50176 [04:22<02:14, 117.87it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.49it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.97it/s]
 70%|██████▉   | 35072/50176 [04:29<02:13, 112.85it/s]
 70%|███████   | 35328/50176 [04:31<02:11, 113.25it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.86it/s]
 71%|███████▏  | 35840/50176 [04:35<02:08, 111.51it/s]
 72%|███████▏  | 36096/50176 [04:38<02:04, 113.42it/s]
 72%|███████▏  | 36352/50176 [04:40<02:03, 112.34it/s]
 73%|███████▎  | 36608/50176 [04:42<02:03, 109.98it/s]
 73%|███████▎  | 36864/50176 [04:45<02:04, 106.99it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 108.98it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 114.69it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.46it/s]
 76%|███████▌  | 37888/50176 [04:53<01:45, 116.19it/s]
 76%|███████▌  | 38144/50176 [04:56<01:42, 117.26it/s]
 77%|███████▋  | 38400/50176 [04:58<01:44, 113.20it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 113.87it/s]
 78%|███████▊  | 38912/50176 [05:03<01:39, 112.94it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.56it/s]
 79%|███████▊  | 39424/50176 [05:07<01:34, 113.38it/s]
 79%|███████▉  | 39680/50176 [05:10<01:33, 112.38it/s]
 80%|███████▉  | 39936/50176 [05:12<01:28, 116.18it/s]
 80%|████████  | 40192/50176 [05:14<01:26, 115.18it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.50it/s]
 81%|████████  | 40704/50176 [05:19<01:28, 106.75it/s]
 82%|████████▏ | 40960/50176 [05:21<01:22, 111.19it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 110.96it/s]
 83%|████████▎ | 41472/50176 [05:26<01:20, 108.36it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 108.85it/s]
 84%|████████▎ | 41984/50176 [05:30<01:14, 109.93it/s]
 84%|██████���█▍ | 42240/50176 [05:33<01:11, 110.51it/s]
 85%|████████▍ | 42496/50176 [05:35<01:08, 111.79it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 112.90it/s]
 86%|████████▌ | 43008/50176 [05:39<01:03, 112.77it/s]
 86%|████████▌ | 43264/50176 [05:42<01:01, 112.97it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.59it/s]
 87%|████████▋ | 43776/50176 [05:46<00:58, 110.23it/s]
 88%|████████▊ | 44032/50176 [05:49<00:56, 108.98it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.04it/s]
 89%|████████▉ | 44544/50176 [05:54<00:52, 106.62it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.16it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.60it/s]
 90%|█████████ | 45312/50176 [06:01<00:45, 106.53it/s]
 91%|█████████ | 45568/50176 [06:03<00:41, 111.44it/s]
 91%|█████████▏| 45824/50176 [06:05<00:40, 108.78it/s]
 92%|█████████▏| 46080/50176 [06:08<00:38, 106.08it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.14it/s]
 93%|█████████▎| 46592/50176 [06:13<00:33, 108.57it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 105.91it/s]
 94%|█████████▍| 47104/50176 [06:18<00:30, 102.14it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.75it/s]
 95%|█████████▍| 47616/50176 [06:23<00:24, 102.77it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 102.98it/s]
 96%|█████████▌| 48128/50176 [06:28<00:19, 104.45it/s]
 96%|█████████▋| 48384/50176 [06:30<00:16, 106.75it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.41it/s]
 97%|█████████▋| 48896/50176 [06:35<00:12, 106.62it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 104.03it/s]
 98%|█████████▊| 49408/50176 [06:40<00:07, 103.79it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.87it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.55it/s] 
-2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5907	imagenet-zeroshot-val-top5: 0.8584

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_33.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt).
-2022-12-21,10:19:59 | INFO | Model:
-2022-12-21,10:19:59 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:19:59 | INFO | Params:
-2022-12-21,10:19:59 | INFO |   batch_size: 256
-2022-12-21,10:19:59 | INFO |   beta1: 0.9
-2022-12-21,10:19:59 | INFO |   beta2: 0.98
-2022-12-21,10:19:59 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/checkpoints
-2022-12-21,10:19:59 | INFO |   copy_codebase: False
-2022-12-21,10:19:59 | INFO |   csv_caption_key: title
-2022-12-21,10:19:59 | INFO |   csv_img_key: filepath
-2022-12-21,10:19:59 | INFO |   csv_separator: 	
-2022-12-21,10:19:59 | INFO |   dataset_resampled: False
-2022-12-21,10:19:59 | INFO |   dataset_type: auto
-2022-12-21,10:19:59 | INFO |   ddp_static_graph: False
-2022-12-21,10:19:59 | INFO |   debug: False
-2022-12-21,10:19:59 | INFO |   device: cuda:0
-2022-12-21,10:19:59 | INFO |   dist_backend: nccl
-2022-12-21,10:19:59 | INFO |   dist_url: env://
-2022-12-21,10:19:59 | INFO |   distributed: False
-2022-12-21,10:19:59 | INFO |   epochs: 32
-2022-12-21,10:19:59 | INFO |   eps: 1e-06
-2022-12-21,10:19:59 | INFO |   force_quick_gelu: False
-2022-12-21,10:19:59 | INFO |   gather_with_grad: False
-2022-12-21,10:19:59 | INFO |   grad_checkpointing: False
-2022-12-21,10:19:59 | INFO |   horovod: False
-2022-12-21,10:19:59 | INFO |   image_mean: None
-2022-12-21,10:19:59 | INFO |   image_std: None
-2022-12-21,10:19:59 | INFO |   imagenet_v2: None
-2022-12-21,10:19:59 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:19:59 | INFO |   local_loss: False
-2022-12-21,10:19:59 | INFO |   local_rank: 0
-2022-12-21,10:19:59 | INFO |   lock_image: False
-2022-12-21,10:19:59 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:19:59 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:19:59 | INFO |   log_level: 20
-2022-12-21,10:19:59 | INFO |   log_local: False
-2022-12-21,10:19:59 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_33.pt/out.log
-2022-12-21,10:19:59 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:19:59 | INFO |   lr: 0.0005
-2022-12-21,10:19:59 | INFO |   model: ViT-B-16
-2022-12-21,10:19:59 | INFO |   name: epoch_33.pt
-2022-12-21,10:19:59 | INFO |   no_set_device_rank: False
-2022-12-21,10:19:59 | INFO |   norm_gradient_clip: None
-2022-12-21,10:19:59 | INFO |   precision: amp_bfloat16
-2022-12-21,10:19:59 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_33.pt
-2022-12-21,10:19:59 | INFO |   pretrained_image: False
-2022-12-21,10:19:59 | INFO |   rank: 0
-2022-12-21,10:19:59 | INFO |   report_to: 
-2022-12-21,10:19:59 | INFO |   resume: None
-2022-12-21,10:19:59 | INFO |   save_frequency: 1
-2022-12-21,10:19:59 | INFO |   save_most_recent: False
-2022-12-21,10:19:59 | INFO |   seed: 0
-2022-12-21,10:19:59 | INFO |   skip_scheduler: False
-2022-12-21,10:19:59 | INFO |   tensorboard: False
-2022-12-21,10:19:59 | INFO |   tensorboard_path: 
-2022-12-21,10:19:59 | INFO |   torchscript: False
-2022-12-21,10:19:59 | INFO |   trace: False
-2022-12-21,10:19:59 | INFO |   train_data: None
-2022-12-21,10:19:59 | INFO |   train_num_samples: None
-2022-12-21,10:19:59 | INFO |   use_bn_sync: False
-2022-12-21,10:19:59 | INFO |   val_data: None
-2022-12-21,10:19:59 | INFO |   val_frequency: 1
-2022-12-21,10:19:59 | INFO |   val_num_samples: None
-2022-12-21,10:19:59 | INFO |   wandb: False
-2022-12-21,10:19:59 | INFO |   wandb_notes: 
-2022-12-21,10:19:59 | INFO |   warmup: 10000
-2022-12-21,10:19:59 | INFO |   wd: 0.2
-2022-12-21,10:19:59 | INFO |   workers: 1
-2022-12-21,10:19:59 | INFO |   world_size: 1
-2022-12-21,10:19:59 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:19:59 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:19:59 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:33,  1.58it/s]
  1%|          | 6/1000 [00:00<01:35, 10.43it/s]
  1%|          | 12/1000 [00:00<00:48, 20.25it/s]
  2%|▏         | 18/1000 [00:00<00:33, 28.95it/s]
  2%|▏         | 24/1000 [00:01<00:27, 36.03it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.65it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.90it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.04it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.42it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.92it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.21it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.02it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.74it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.20it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.71it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.88it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.31it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.50it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.90it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.79it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.83it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.76it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.82it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.81it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.85it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.76it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.64it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.64it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.61it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.55it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.49it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.45it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.26it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.07it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.18it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.18it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.39it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.40it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.30it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.37it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.48it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.59it/s]
 26%|██▋       | 264/1000 [00:05<00:14, 49.84it/s]
 27%|██▋       | 270/1000 [00:05<00:14, 51.81it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 53.42it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 54.74it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 55.67it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.48it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.88it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.18it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.50it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.64it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.67it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.57it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.70it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.84it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.93it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.75it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.76it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.86it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.94it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.62it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.34it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.45it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.60it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.77it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.87it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.88it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.90it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.92it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.04it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.04it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.83it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.94it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.94it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.90it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.01it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.92it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.89it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.89it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.82it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.86it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.79it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.86it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.88it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.98it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.97it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.88it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.91it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.98it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.87it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.82it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.93it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.84it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.96it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.89it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.85it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.98it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.97it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.04it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.85it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.80it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.94it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.90it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.96it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.00it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.99it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.99it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.01it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.90it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.01it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.00it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.97it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.98it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.91it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.91it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.94it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.93it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.02it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.05it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.91it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.03it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.99it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.04it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.08it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.88it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.78it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.84it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.83it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.84it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.86it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.95it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.11it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.15it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.06it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.08it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.01it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.02it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.99it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.04it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.04it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.91it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.00it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.01it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.99it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.96it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.10it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.08it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.10it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.13it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.08it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.17it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.01it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.86it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.87it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.95it/s]
-2022-12-21,10:20:17 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:20, 131.33it/s]
  1%|          | 512/50176 [00:03<05:47, 143.05it/s]
  2%|▏         | 768/50176 [00:05<05:45, 143.02it/s]
  2%|▏         | 1024/50176 [00:07<05:46, 141.77it/s]
  3%|▎         | 1280/50176 [00:09<05:47, 140.59it/s]
  3%|▎         | 1536/50176 [00:11<06:03, 133.89it/s]
  4%|▎         | 1792/50176 [00:13<05:57, 135.53it/s]
  4%|▍         | 2048/50176 [00:14<05:48, 138.01it/s]
  5%|▍         | 2304/50176 [00:16<05:46, 138.14it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.11it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 140.04it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 142.94it/s]
  7%|▋         | 3328/50176 [00:23<05:25, 143.80it/s]
  7%|▋         | 3584/50176 [00:25<05:23, 144.19it/s]
  8%|▊         | 3840/50176 [00:27<05:33, 138.95it/s]
  8%|▊         | 4096/50176 [00:29<05:31, 138.91it/s]
  9%|▊         | 4352/50176 [00:31<05:25, 140.96it/s]
  9%|▉         | 4608/50176 [00:32<05:24, 140.63it/s]
 10%|▉         | 4864/50176 [00:34<05:16, 143.08it/s]
 10%|█         | 5120/50176 [00:36<05:11, 144.65it/s]
 11%|█         | 5376/50176 [00:38<05:08, 145.20it/s]
 11%|█         | 5632/50176 [00:39<05:05, 145.93it/s]
 12%|█▏        | 5888/50176 [00:41<05:01, 146.72it/s]
 12%|█▏        | 6144/50176 [00:43<05:27, 134.25it/s]
 13%|█▎        | 6400/50176 [00:45<05:16, 138.33it/s]
 13%|█▎        | 6656/50176 [00:47<05:09, 140.70it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.57it/s]
 14%|█▍        | 7168/50176 [00:50<05:02, 142.19it/s]
 15%|█▍        | 7424/50176 [00:52<04:55, 144.72it/s]
 15%|█▌        | 7680/50176 [00:54<04:54, 144.36it/s]
 16%|█▌        | 7936/50176 [00:55<04:45, 147.90it/s]
 16%|█▋        | 8192/50176 [00:58<04:58, 140.77it/s]
 17%|█▋        | 8448/50176 [00:59<04:48, 144.40it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.23it/s]
 18%|█▊        | 8960/50176 [01:03<04:50, 141.71it/s]
 18%|█▊        | 9216/50176 [01:04<04:41, 145.61it/s]
 19%|█▉        | 9472/50176 [01:06<04:40, 145.36it/s]
 19%|█▉        | 9728/50176 [01:08<04:41, 143.52it/s]
 20%|█▉        | 9984/50176 [01:10<04:34, 146.16it/s]
 20%|██        | 10240/50176 [01:11<04:29, 148.24it/s]
 21%|██        | 10496/50176 [01:13<04:25, 149.23it/s]
 21%|██▏       | 10752/50176 [01:15<04:33, 144.14it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 147.17it/s]
 22%|██▏       | 11264/50176 [01:18<04:22, 148.32it/s]
 23%|██▎       | 11520/50176 [01:20<04:16, 150.76it/s]
 23%|██▎       | 11776/50176 [01:22<04:16, 149.99it/s]
 24%|██▍       | 12032/50176 [01:23<04:13, 150.41it/s]
 24%|██▍       | 12288/50176 [01:25<04:11, 150.73it/s]
 25%|██▌       | 12544/50176 [01:27<04:10, 150.30it/s]
 26%|██▌       | 12800/50176 [01:29<04:07, 150.74it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.35it/s]
 27%|██▋       | 13312/50176 [01:32<04:06, 149.84it/s]
 27%|██▋       | 13568/50176 [01:34<04:18, 141.78it/s]
 28%|██▊       | 13824/50176 [01:36<04:20, 139.40it/s]
 28%|██▊       | 14080/50176 [01:38<04:14, 141.61it/s]
 29%|██▊       | 14336/50176 [01:40<04:18, 138.66it/s]
 29%|██▉       | 14592/50176 [01:41<04:08, 143.48it/s]
 30%|██▉       | 14848/50176 [01:43<04:20, 135.69it/s]
 30%|███       | 15104/50176 [01:45<04:19, 135.23it/s]
 31%|███       | 15360/50176 [01:47<04:12, 137.97it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.76it/s]
 32%|███▏      | 15872/50176 [01:51<04:11, 136.25it/s]
 32%|███▏      | 16128/50176 [01:52<04:03, 139.65it/s]
 33%|███▎      | 16384/50176 [01:54<04:02, 139.27it/s]
 33%|███▎      | 16640/50176 [01:56<04:07, 135.40it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 140.23it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.81it/s]
 35%|███▍      | 17408/50176 [02:01<03:48, 143.70it/s]
 35%|███▌      | 17664/50176 [02:03<03:44, 144.78it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.41it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.75it/s]
 37%|███▋      | 18432/50176 [02:08<03:35, 147.46it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.03it/s]
 38%|███▊      | 18944/50176 [02:12<03:40, 141.66it/s]
 38%|███▊      | 19200/50176 [02:14<03:51, 133.69it/s]
 39%|███▉      | 19456/50176 [02:16<03:52, 131.95it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.20it/s]
 40%|███▉      | 19968/50176 [02:20<03:43, 135.24it/s]
 40%|████      | 20224/50176 [02:22<03:32, 141.07it/s]
 41%|████      | 20480/50176 [02:23<03:25, 144.22it/s]
 41%|████▏     | 20736/50176 [02:25<03:24, 144.31it/s]
 42%|████▏     | 20992/50176 [02:27<03:18, 147.38it/s]
 42%|████▏     | 21248/50176 [02:29<03:24, 141.62it/s]
 43%|████▎     | 21504/50176 [02:30<03:15, 146.84it/s]
 43%|████▎     | 21760/50176 [02:32<03:12, 147.39it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 151.09it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.43it/s]
 45%|████▍     | 22528/50176 [02:37<03:16, 140.95it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.74it/s]
 46%|████▌     | 23040/50176 [02:42<03:45, 120.38it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.50it/s]
 47%|████▋     | 23552/50176 [02:47<03:59, 111.23it/s]
 47%|████▋     | 23808/50176 [02:49<03:52, 113.53it/s]
 48%|████▊     | 24064/50176 [02:52<03:49, 113.89it/s]
 48%|████▊     | 24320/50176 [02:54<03:52, 111.36it/s]
 49%|████▉     | 24576/50176 [02:56<03:44, 114.25it/s]
 49%|████▉     | 24832/50176 [02:58<03:32, 119.23it/s]
 50%|█████     | 25088/50176 [03:00<03:31, 118.81it/s]
 51%|█████     | 25344/50176 [03:02<03:26, 120.43it/s]
 51%|█████     | 25600/50176 [03:05<03:30, 116.78it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.48it/s]
 52%|█████▏    | 26112/50176 [03:09<03:27, 115.94it/s]
 53%|█████▎    | 26368/50176 [03:11<03:29, 113.41it/s]
 53%|█████▎    | 26624/50176 [03:13<03:24, 115.23it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.32it/s]
 54%|█████▍    | 27136/50176 [03:18<03:23, 113.12it/s]
 55%|█████▍    | 27392/50176 [03:20<03:18, 114.81it/s]
 55%|█████▌    | 27648/50176 [03:22<03:03, 122.73it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.48it/s]
 56%|█████▌    | 28160/50176 [03:27<03:16, 111.86it/s]
 57%|█████▋    | 28416/50176 [03:29<03:14, 112.14it/s]
 57%|█████▋    | 28672/50176 [03:32<03:14, 110.70it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.81it/s]
 58%|█████▊    | 29184/50176 [03:36<03:06, 112.84it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.36it/s]
 59%|█████▉    | 29696/50176 [03:41<03:17, 103.86it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.52it/s]
 60%|██████    | 30208/50176 [03:46<03:05, 107.73it/s]
 61%|██████    | 30464/50176 [03:48<02:53, 113.47it/s]
 61%|██████    | 30720/50176 [03:50<02:51, 113.73it/s]
 62%|██████▏   | 30976/50176 [03:52<02:46, 115.22it/s]
 62%|██████▏   | 31232/50176 [03:55<02:44, 115.35it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.70it/s]
 63%|██████▎   | 31744/50176 [03:59<02:35, 118.88it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.33it/s]
 64%|██████▍   | 32256/50176 [04:03<02:31, 118.33it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.69it/s]
 65%|██████▌   | 32768/50176 [04:08<02:30, 115.43it/s]
 66%|██████▌   | 33024/50176 [04:10<02:29, 114.79it/s]
 66%|██████▋   | 33280/50176 [04:12<02:26, 115.38it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.74it/s]
 67%|██████▋   | 33792/50176 [04:17<02:26, 111.80it/s]
 68%|██████▊   | 34048/50176 [04:19<02:18, 116.05it/s]
 68%|██████▊   | 34304/50176 [04:21<02:14, 117.84it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.81it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.94it/s]
 70%|██████▉   | 35072/50176 [04:28<02:13, 112.90it/s]
 70%|███████   | 35328/50176 [04:30<02:11, 113.21it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.88it/s]
 71%|███████▏  | 35840/50176 [04:35<02:08, 111.38it/s]
 72%|███████▏  | 36096/50176 [04:37<02:04, 113.39it/s]
 72%|███████▏  | 36352/50176 [04:40<02:03, 111.93it/s]
 73%|███████▎  | 36608/50176 [04:42<02:03, 109.74it/s]
 73%|███████▎  | 36864/50176 [04:45<02:04, 106.88it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 109.10it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 114.65it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.67it/s]
 76%|███████▌  | 37888/50176 [04:53<01:46, 115.88it/s]
 76%|███████▌  | 38144/50176 [04:55<01:42, 117.52it/s]
 77%|███████▋  | 38400/50176 [04:58<01:43, 113.43it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 114.01it/s]
 78%|███████▊  | 38912/50176 [05:02<01:39, 112.64it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.71it/s]
 79%|███████▊  | 39424/50176 [05:07<01:35, 113.17it/s]
 79%|███████▉  | 39680/50176 [05:09<01:33, 112.36it/s]
 80%|███████▉  | 39936/50176 [05:11<01:28, 116.10it/s]
 80%|████████  | 40192/50176 [05:13<01:26, 115.47it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.42it/s]
 81%|████████  | 40704/50176 [05:18<01:28, 106.78it/s]
 82%|████████▏ | 40960/50176 [05:21<01:22, 111.33it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 111.09it/s]
 83%|████████▎ | 41472/50176 [05:25<01:20, 108.18it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 109.02it/s]
 84%|████████▎ | 41984/50176 [05:30<01:14, 109.76it/s]
 84%|██████���█▍ | 42240/50176 [05:32<01:11, 110.69it/s]
 85%|████████▍ | 42496/50176 [05:34<01:08, 111.71it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 113.00it/s]
 86%|████████▌ | 43008/50176 [05:39<01:03, 112.73it/s]
 86%|████████▌ | 43264/50176 [05:41<01:01, 113.04it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.53it/s]
 87%|████████▋ | 43776/50176 [05:46<00:58, 110.31it/s]
 88%|████████▊ | 44032/50176 [05:48<00:56, 108.85it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.17it/s]
 89%|████████▉ | 44544/50176 [05:53<00:52, 106.54it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.17it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.48it/s]
 90%|█████████ | 45312/50176 [06:00<00:45, 106.74it/s]
 91%|█████████ | 45568/50176 [06:02<00:41, 111.37it/s]
 91%|█████████▏| 45824/50176 [06:05<00:39, 108.84it/s]
 92%|█████████▏| 46080/50176 [06:07<00:38, 106.09it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.19it/s]
 93%|█████████▎| 46592/50176 [06:12<00:33, 108.60it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 105.93it/s]
 94%|█████████▍| 47104/50176 [06:17<00:30, 102.03it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.79it/s]
 95%|█████████▍| 47616/50176 [06:22<00:24, 102.69it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 103.10it/s]
 96%|█████████▌| 48128/50176 [06:27<00:19, 104.39it/s]
 96%|█████████▋| 48384/50176 [06:29<00:16, 106.83it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.40it/s]
 97%|█████████▋| 48896/50176 [06:34<00:12, 106.64it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 103.95it/s]
 98%|█████████▊| 49408/50176 [06:39<00:07, 103.88it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.77it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.62it/s] 
-2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5897	imagenet-zeroshot-val-top5: 0.8586

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_34.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:19:50 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:19:50 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:19:51 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt).
-2022-12-21,10:19:59 | INFO | Model:
-2022-12-21,10:19:59 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:19:59 | INFO | Params:
-2022-12-21,10:19:59 | INFO |   batch_size: 256
-2022-12-21,10:19:59 | INFO |   beta1: 0.9
-2022-12-21,10:19:59 | INFO |   beta2: 0.98
-2022-12-21,10:19:59 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/checkpoints
-2022-12-21,10:19:59 | INFO |   copy_codebase: False
-2022-12-21,10:19:59 | INFO |   csv_caption_key: title
-2022-12-21,10:19:59 | INFO |   csv_img_key: filepath
-2022-12-21,10:19:59 | INFO |   csv_separator: 	
-2022-12-21,10:19:59 | INFO |   dataset_resampled: False
-2022-12-21,10:19:59 | INFO |   dataset_type: auto
-2022-12-21,10:19:59 | INFO |   ddp_static_graph: False
-2022-12-21,10:19:59 | INFO |   debug: False
-2022-12-21,10:19:59 | INFO |   device: cuda:0
-2022-12-21,10:19:59 | INFO |   dist_backend: nccl
-2022-12-21,10:19:59 | INFO |   dist_url: env://
-2022-12-21,10:19:59 | INFO |   distributed: False
-2022-12-21,10:19:59 | INFO |   epochs: 32
-2022-12-21,10:19:59 | INFO |   eps: 1e-06
-2022-12-21,10:19:59 | INFO |   force_quick_gelu: False
-2022-12-21,10:19:59 | INFO |   gather_with_grad: False
-2022-12-21,10:19:59 | INFO |   grad_checkpointing: False
-2022-12-21,10:19:59 | INFO |   horovod: False
-2022-12-21,10:19:59 | INFO |   image_mean: None
-2022-12-21,10:19:59 | INFO |   image_std: None
-2022-12-21,10:19:59 | INFO |   imagenet_v2: None
-2022-12-21,10:19:59 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:19:59 | INFO |   local_loss: False
-2022-12-21,10:19:59 | INFO |   local_rank: 0
-2022-12-21,10:19:59 | INFO |   lock_image: False
-2022-12-21,10:19:59 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:19:59 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:19:59 | INFO |   log_level: 20
-2022-12-21,10:19:59 | INFO |   log_local: False
-2022-12-21,10:19:59 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_34.pt/out.log
-2022-12-21,10:19:59 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:19:59 | INFO |   lr: 0.0005
-2022-12-21,10:19:59 | INFO |   model: ViT-B-16
-2022-12-21,10:19:59 | INFO |   name: epoch_34.pt
-2022-12-21,10:19:59 | INFO |   no_set_device_rank: False
-2022-12-21,10:19:59 | INFO |   norm_gradient_clip: None
-2022-12-21,10:19:59 | INFO |   precision: amp_bfloat16
-2022-12-21,10:19:59 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_34.pt
-2022-12-21,10:19:59 | INFO |   pretrained_image: False
-2022-12-21,10:19:59 | INFO |   rank: 0
-2022-12-21,10:19:59 | INFO |   report_to: 
-2022-12-21,10:19:59 | INFO |   resume: None
-2022-12-21,10:19:59 | INFO |   save_frequency: 1
-2022-12-21,10:19:59 | INFO |   save_most_recent: False
-2022-12-21,10:19:59 | INFO |   seed: 0
-2022-12-21,10:19:59 | INFO |   skip_scheduler: False
-2022-12-21,10:19:59 | INFO |   tensorboard: False
-2022-12-21,10:19:59 | INFO |   tensorboard_path: 
-2022-12-21,10:19:59 | INFO |   torchscript: False
-2022-12-21,10:19:59 | INFO |   trace: False
-2022-12-21,10:19:59 | INFO |   train_data: None
-2022-12-21,10:19:59 | INFO |   train_num_samples: None
-2022-12-21,10:19:59 | INFO |   use_bn_sync: False
-2022-12-21,10:19:59 | INFO |   val_data: None
-2022-12-21,10:19:59 | INFO |   val_frequency: 1
-2022-12-21,10:19:59 | INFO |   val_num_samples: None
-2022-12-21,10:19:59 | INFO |   wandb: False
-2022-12-21,10:19:59 | INFO |   wandb_notes: 
-2022-12-21,10:19:59 | INFO |   warmup: 10000
-2022-12-21,10:19:59 | INFO |   wd: 0.2
-2022-12-21,10:19:59 | INFO |   workers: 1
-2022-12-21,10:19:59 | INFO |   world_size: 1
-2022-12-21,10:19:59 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:19:59 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:19:59 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:18,  1.47it/s]
  1%|          | 6/1000 [00:00<01:40,  9.90it/s]
  1%|          | 12/1000 [00:00<00:51, 19.30it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.89it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.01it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.84it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.26it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.61it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.92it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.76it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.11it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.07it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.75it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.23it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.74it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.04it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.38it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.56it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.80it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.93it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.89it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.81it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.86it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.82it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.65it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.78it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.83it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.73it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.60it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.60it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.55it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.32it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.37it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.37it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.18it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.21it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.95it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.01it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.13it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.12it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.25it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.39it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.54it/s]
 26%|██▋       | 264/1000 [00:05<00:14, 51.13it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 52.78it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 54.15it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 55.29it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 56.11it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.82it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.16it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.39it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.69it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.79it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.68it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.34it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.53it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.73it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.59it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.68it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.81it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.92it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.84it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 56.47it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 56.84it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.17it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.17it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.40it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.54it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.68it/s]
 43%|████▎     | 426/1000 [00:08<00:10, 57.20it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.51it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.67it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.57it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.46it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.64it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.73it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.75it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.64it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.67it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.74it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.80it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.76it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.85it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.78it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.86it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.89it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.98it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.93it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.05it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.05it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.94it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.97it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.03it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.67it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.65it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.81it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.81it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.75it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.86it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.78it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.79it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.93it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.02it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.85it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.79it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.93it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.87it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.93it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.00it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.94it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.98it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.00it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.02it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.92it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.00it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.99it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.98it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.95it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.98it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.97it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.05it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.13it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.97it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.07it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.02it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.06it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.09it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.89it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.86it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.92it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.88it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.90it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.98it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.10it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.14it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.05it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.09it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.03it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.03it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.96it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.01it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.01it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.89it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.99it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.00it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.96it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.09it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.06it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.07it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.11it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.06it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.11it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.88it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.68it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.44it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.56it/s]
-2022-12-21,10:20:17 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:01<06:04, 137.11it/s]
  1%|          | 512/50176 [00:03<05:43, 144.42it/s]
  2%|▏         | 768/50176 [00:05<05:44, 143.41it/s]
  2%|▏         | 1024/50176 [00:07<05:42, 143.34it/s]
  3%|▎         | 1280/50176 [00:08<05:43, 142.15it/s]
  3%|▎         | 1536/50176 [00:11<06:00, 135.06it/s]
  4%|▎         | 1792/50176 [00:12<05:56, 135.61it/s]
  4%|▍         | 2048/50176 [00:14<05:50, 137.23it/s]
  5%|▍         | 2304/50176 [00:16<05:46, 138.19it/s]
  5%|▌         | 2560/50176 [00:18<05:39, 140.31it/s]
  6%|▌         | 2816/50176 [00:20<05:38, 139.74it/s]
  6%|▌         | 3072/50176 [00:21<05:29, 143.12it/s]
  7%|▋         | 3328/50176 [00:23<05:25, 143.91it/s]
  7%|▋         | 3584/50176 [00:25<05:22, 144.34it/s]
  8%|▊         | 3840/50176 [00:27<05:34, 138.67it/s]
  8%|▊         | 4096/50176 [00:29<05:31, 138.98it/s]
  9%|▊         | 4352/50176 [00:30<05:24, 141.15it/s]
  9%|▉         | 4608/50176 [00:32<05:22, 141.32it/s]
 10%|▉         | 4864/50176 [00:34<05:16, 143.03it/s]
 10%|█         | 5120/50176 [00:36<05:13, 143.85it/s]
 11%|█         | 5376/50176 [00:38<05:09, 144.68it/s]
 11%|█         | 5632/50176 [00:39<05:06, 145.48it/s]
 12%|█▏        | 5888/50176 [00:41<05:02, 146.44it/s]
 12%|█▏        | 6144/50176 [00:43<05:28, 134.13it/s]
 13%|█▎        | 6400/50176 [00:45<05:16, 138.19it/s]
 13%|█▎        | 6656/50176 [00:47<05:09, 140.49it/s]
 14%|█▍        | 6912/50176 [00:49<05:07, 140.52it/s]
 14%|█▍        | 7168/50176 [00:50<05:03, 141.69it/s]
 15%|█▍        | 7424/50176 [00:52<04:54, 145.02it/s]
 15%|█▌        | 7680/50176 [00:54<04:55, 144.03it/s]
 16%|█▌        | 7936/50176 [00:55<04:45, 148.00it/s]
 16%|█▋        | 8192/50176 [00:57<04:57, 141.10it/s]
 17%|█▋        | 8448/50176 [00:59<04:49, 144.39it/s]
 17%|█▋        | 8704/50176 [01:01<04:45, 145.29it/s]
 18%|█▊        | 8960/50176 [01:03<04:52, 141.11it/s]
 18%|█▊        | 9216/50176 [01:04<04:40, 145.82it/s]
 19%|█▉        | 9472/50176 [01:06<04:38, 146.10it/s]
 19%|█▉        | 9728/50176 [01:08<04:42, 143.42it/s]
 20%|█▉        | 9984/50176 [01:10<04:35, 146.12it/s]
 20%|██        | 10240/50176 [01:11<04:29, 148.22it/s]
 21%|██        | 10496/50176 [01:13<04:27, 148.59it/s]
 21%|██▏       | 10752/50176 [01:15<04:32, 144.80it/s]
 22%|██▏       | 11008/50176 [01:17<04:26, 147.15it/s]
 22%|██▏       | 11264/50176 [01:18<04:23, 147.60it/s]
 23%|██▎       | 11520/50176 [01:20<04:16, 150.92it/s]
 23%|██▎       | 11776/50176 [01:22<04:15, 150.27it/s]
 24%|██▍       | 12032/50176 [01:23<04:13, 150.48it/s]
 24%|██▍       | 12288/50176 [01:25<04:10, 151.46it/s]
 25%|██▌       | 12544/50176 [01:27<04:10, 150.10it/s]
 26%|██▌       | 12800/50176 [01:28<04:07, 150.76it/s]
 26%|██▌       | 13056/50176 [01:30<04:03, 152.20it/s]
 27%|██▋       | 13312/50176 [01:32<04:05, 150.06it/s]
 27%|██▋       | 13568/50176 [01:34<04:18, 141.45it/s]
 28%|██▊       | 13824/50176 [01:36<04:21, 138.98it/s]
 28%|██▊       | 14080/50176 [01:38<04:14, 141.61it/s]
 29%|██▊       | 14336/50176 [01:40<04:19, 138.15it/s]
 29%|██▉       | 14592/50176 [01:41<04:07, 143.83it/s]
 30%|██▉       | 14848/50176 [01:43<04:19, 135.99it/s]
 30%|███       | 15104/50176 [01:45<04:19, 135.10it/s]
 31%|███       | 15360/50176 [01:47<04:11, 138.18it/s]
 31%|███       | 15616/50176 [01:49<04:07, 139.79it/s]
 32%|███▏      | 15872/50176 [01:51<04:10, 136.99it/s]
 32%|███▏      | 16128/50176 [01:52<04:04, 139.40it/s]
 33%|███▎      | 16384/50176 [01:54<04:01, 139.72it/s]
 33%|███▎      | 16640/50176 [01:56<04:08, 134.95it/s]
 34%|███▎      | 16896/50176 [01:58<03:57, 140.11it/s]
 34%|███▍      | 17152/50176 [02:00<03:51, 142.88it/s]
 35%|███▍      | 17408/50176 [02:01<03:48, 143.63it/s]
 35%|███▌      | 17664/50176 [02:03<03:44, 144.69it/s]
 36%|███▌      | 17920/50176 [02:05<03:38, 147.33it/s]
 36%|███▌      | 18176/50176 [02:07<03:38, 146.25it/s]
 37%|███▋      | 18432/50176 [02:08<03:35, 147.50it/s]
 37%|███▋      | 18688/50176 [02:10<03:34, 147.13it/s]
 38%|███▊      | 18944/50176 [02:12<03:39, 142.17it/s]
 38%|███▊      | 19200/50176 [02:14<03:50, 134.16it/s]
 39%|███▉      | 19456/50176 [02:16<03:52, 131.97it/s]
 39%|███▉      | 19712/50176 [02:18<03:50, 132.23it/s]
 40%|███▉      | 19968/50176 [02:20<03:44, 134.79it/s]
 40%|████      | 20224/50176 [02:22<03:32, 140.84it/s]
 41%|████      | 20480/50176 [02:23<03:25, 144.48it/s]
 41%|████▏     | 20736/50176 [02:25<03:23, 144.41it/s]
 42%|████▏     | 20992/50176 [02:27<03:19, 146.58it/s]
 42%|████▏     | 21248/50176 [02:29<03:23, 142.30it/s]
 43%|████▎     | 21504/50176 [02:30<03:15, 146.91it/s]
 43%|████▎     | 21760/50176 [02:32<03:12, 147.28it/s]
 44%|████▍     | 22016/50176 [02:34<03:06, 150.89it/s]
 44%|████▍     | 22272/50176 [02:35<03:05, 150.30it/s]
 45%|████▍     | 22528/50176 [02:37<03:16, 140.77it/s]
 45%|████▌     | 22784/50176 [02:40<03:34, 127.92it/s]
 46%|████▌     | 23040/50176 [02:42<03:45, 120.35it/s]
 46%|████▋     | 23296/50176 [02:45<03:52, 115.49it/s]
 47%|████▋     | 23552/50176 [02:47<03:58, 111.58it/s]
 47%|████▋     | 23808/50176 [02:49<03:51, 113.67it/s]
 48%|████▊     | 24064/50176 [02:51<03:49, 113.78it/s]
 48%|████▊     | 24320/50176 [02:54<03:52, 111.42it/s]
 49%|████▉     | 24576/50176 [02:56<03:44, 114.05it/s]
 49%|████▉     | 24832/50176 [02:58<03:33, 118.66it/s]
 50%|█████     | 25088/50176 [03:00<03:31, 118.84it/s]
 51%|█████     | 25344/50176 [03:02<03:26, 120.48it/s]
 51%|█████     | 25600/50176 [03:05<03:29, 117.03it/s]
 52%|█████▏    | 25856/50176 [03:07<03:25, 118.22it/s]
 52%|█████▏    | 26112/50176 [03:09<03:27, 116.00it/s]
 53%|█████▎    | 26368/50176 [03:11<03:30, 113.30it/s]
 53%|█████▎    | 26624/50176 [03:13<03:24, 115.28it/s]
 54%|█████▎    | 26880/50176 [03:16<03:25, 113.40it/s]
 54%|█████▍    | 27136/50176 [03:18<03:23, 113.23it/s]
 55%|█████▍    | 27392/50176 [03:20<03:18, 114.78it/s]
 55%|█████▌    | 27648/50176 [03:22<03:03, 122.69it/s]
 56%|█████▌    | 27904/50176 [03:25<03:14, 114.53it/s]
 56%|█████▌    | 28160/50176 [03:27<03:16, 111.77it/s]
 57%|█████▋    | 28416/50176 [03:29<03:14, 112.04it/s]
 57%|█████▋    | 28672/50176 [03:32<03:14, 110.69it/s]
 58%|█████▊    | 28928/50176 [03:34<03:10, 111.80it/s]
 58%|█████▊    | 29184/50176 [03:36<03:06, 112.74it/s]
 59%|█████▊    | 29440/50176 [03:39<03:14, 106.47it/s]
 59%|█████▉    | 29696/50176 [03:41<03:17, 103.89it/s]
 60%|█████▉    | 29952/50176 [03:44<03:11, 105.49it/s]
 60%|██████    | 30208/50176 [03:46<03:05, 107.61it/s]
 61%|██████    | 30464/50176 [03:48<02:53, 113.51it/s]
 61%|██████    | 30720/50176 [03:50<02:51, 113.70it/s]
 62%|██████▏   | 30976/50176 [03:52<02:46, 115.38it/s]
 62%|██████▏   | 31232/50176 [03:55<02:43, 115.52it/s]
 63%|██████▎   | 31488/50176 [03:57<02:38, 117.80it/s]
 63%|██████▎   | 31744/50176 [03:59<02:34, 119.09it/s]
 64%|██████▍   | 32000/50176 [04:01<02:32, 119.34it/s]
 64%|██████▍   | 32256/50176 [04:03<02:31, 118.30it/s]
 65%|██████▍   | 32512/50176 [04:06<02:39, 110.51it/s]
 65%|██████▌   | 32768/50176 [04:08<02:30, 115.50it/s]
 66%|██████▌   | 33024/50176 [04:10<02:29, 114.70it/s]
 66%|██████▋   | 33280/50176 [04:12<02:26, 115.35it/s]
 67%|██████▋   | 33536/50176 [04:15<02:30, 110.83it/s]
 67%|██████▋   | 33792/50176 [04:17<02:26, 111.73it/s]
 68%|██████▊   | 34048/50176 [04:19<02:18, 116.12it/s]
 68%|██████▊   | 34304/50176 [04:21<02:14, 117.66it/s]
 69%|██████▉   | 34560/50176 [04:24<02:17, 113.66it/s]
 69%|██████▉   | 34816/50176 [04:26<02:17, 111.92it/s]
 70%|██████▉   | 35072/50176 [04:28<02:13, 112.95it/s]
 70%|███████   | 35328/50176 [04:30<02:11, 113.12it/s]
 71%|███████   | 35584/50176 [04:33<02:08, 113.87it/s]
 71%|███████▏  | 35840/50176 [04:35<02:08, 111.46it/s]
 72%|███████▏  | 36096/50176 [04:37<02:03, 113.57it/s]
 72%|███████▏  | 36352/50176 [04:39<02:03, 112.11it/s]
 73%|███████▎  | 36608/50176 [04:42<02:03, 109.88it/s]
 73%|███████▎  | 36864/50176 [04:44<02:04, 106.92it/s]
 74%|███████▍  | 37120/50176 [04:47<01:59, 109.02it/s]
 74%|███████▍  | 37376/50176 [04:49<01:51, 114.97it/s]
 75%|███████▌  | 37632/50176 [04:51<01:48, 115.69it/s]
 76%|███████▌  | 37888/50176 [04:53<01:46, 115.87it/s]
 76%|███████▌  | 38144/50176 [04:55<01:42, 117.35it/s]
 77%|███████▋  | 38400/50176 [04:58<01:43, 113.26it/s]
 77%|███████▋  | 38656/50176 [05:00<01:41, 113.85it/s]
 78%|███████▊  | 38912/50176 [05:02<01:39, 112.91it/s]
 78%|███████▊  | 39168/50176 [05:05<01:40, 109.59it/s]
 79%|███████▊  | 39424/50176 [05:07<01:34, 113.22it/s]
 79%|███████▉  | 39680/50176 [05:09<01:33, 112.58it/s]
 80%|███████▉  | 39936/50176 [05:11<01:28, 115.88it/s]
 80%|████████  | 40192/50176 [05:13<01:26, 114.97it/s]
 81%|████████  | 40448/50176 [05:16<01:27, 111.44it/s]
 81%|████████  | 40704/50176 [05:18<01:28, 106.80it/s]
 82%|████████▏ | 40960/50176 [05:20<01:22, 111.37it/s]
 82%|████████▏ | 41216/50176 [05:23<01:20, 111.06it/s]
 83%|████████▎ | 41472/50176 [05:25<01:20, 108.37it/s]
 83%|████████▎ | 41728/50176 [05:28<01:17, 108.88it/s]
 84%|████████▎ | 41984/50176 [05:30<01:14, 109.68it/s]
 84%|██████���█▍ | 42240/50176 [05:32<01:11, 110.81it/s]
 85%|████████▍ | 42496/50176 [05:34<01:08, 111.74it/s]
 85%|████████▌ | 42752/50176 [05:37<01:05, 113.01it/s]
 86%|████████▌ | 43008/50176 [05:39<01:03, 112.60it/s]
 86%|████████▌ | 43264/50176 [05:41<01:01, 112.96it/s]
 87%|████████▋ | 43520/50176 [05:44<00:59, 111.59it/s]
 87%|████████▋ | 43776/50176 [05:46<00:58, 110.34it/s]
 88%|████████▊ | 44032/50176 [05:48<00:56, 109.06it/s]
 88%|████████▊ | 44288/50176 [05:51<00:53, 110.04it/s]
 89%|████████▉ | 44544/50176 [05:53<00:52, 106.61it/s]
 89%|████████▉ | 44800/50176 [05:56<00:50, 107.14it/s]
 90%|████████▉ | 45056/50176 [05:58<00:46, 109.34it/s]
 90%|█████████ | 45312/50176 [06:00<00:45, 106.88it/s]
 91%|█████████ | 45568/50176 [06:02<00:41, 111.26it/s]
 91%|█████████▏| 45824/50176 [06:05<00:40, 108.80it/s]
 92%|█████████▏| 46080/50176 [06:07<00:38, 105.99it/s]
 92%|█████████▏| 46336/50176 [06:10<00:35, 107.27it/s]
 93%|█████████▎| 46592/50176 [06:12<00:33, 108.53it/s]
 93%|█████████▎| 46848/50176 [06:15<00:31, 106.11it/s]
 94%|█████████▍| 47104/50176 [06:17<00:30, 101.94it/s]
 94%|█████████▍| 47360/50176 [06:20<00:27, 102.80it/s]
 95%|█████████▍| 47616/50176 [06:22<00:24, 102.74it/s]
 95%|█████████▌| 47872/50176 [06:25<00:22, 103.01it/s]
 96%|█████████▌| 48128/50176 [06:27<00:19, 104.53it/s]
 96%|█████████▋| 48384/50176 [06:29<00:16, 106.84it/s]
 97%|█████████▋| 48640/50176 [06:32<00:14, 106.26it/s]
 97%|█████████▋| 48896/50176 [06:34<00:11, 106.77it/s]
 98%|█████████▊| 49152/50176 [06:37<00:09, 103.94it/s]
 98%|█████████▊| 49408/50176 [06:39<00:07, 103.83it/s]
 99%|█████████▉| 49664/50176 [06:42<00:04, 102.78it/s]
 99%|█████████▉| 49920/50176 [06:45<00:02, 97.54it/s] 
-2022-12-21,10:27:03 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:27:03 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5929	imagenet-zeroshot-val-top5: 0.8605

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_35.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt).
-2022-12-21,10:27:26 | INFO | Model:
-2022-12-21,10:27:26 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:27:26 | INFO | Params:
-2022-12-21,10:27:26 | INFO |   batch_size: 256
-2022-12-21,10:27:26 | INFO |   beta1: 0.9
-2022-12-21,10:27:26 | INFO |   beta2: 0.98
-2022-12-21,10:27:26 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/checkpoints
-2022-12-21,10:27:26 | INFO |   copy_codebase: False
-2022-12-21,10:27:26 | INFO |   csv_caption_key: title
-2022-12-21,10:27:26 | INFO |   csv_img_key: filepath
-2022-12-21,10:27:26 | INFO |   csv_separator: 	
-2022-12-21,10:27:26 | INFO |   dataset_resampled: False
-2022-12-21,10:27:26 | INFO |   dataset_type: auto
-2022-12-21,10:27:26 | INFO |   ddp_static_graph: False
-2022-12-21,10:27:26 | INFO |   debug: False
-2022-12-21,10:27:26 | INFO |   device: cuda:0
-2022-12-21,10:27:26 | INFO |   dist_backend: nccl
-2022-12-21,10:27:26 | INFO |   dist_url: env://
-2022-12-21,10:27:26 | INFO |   distributed: False
-2022-12-21,10:27:26 | INFO |   epochs: 32
-2022-12-21,10:27:26 | INFO |   eps: 1e-06
-2022-12-21,10:27:26 | INFO |   force_quick_gelu: False
-2022-12-21,10:27:26 | INFO |   gather_with_grad: False
-2022-12-21,10:27:26 | INFO |   grad_checkpointing: False
-2022-12-21,10:27:26 | INFO |   horovod: False
-2022-12-21,10:27:26 | INFO |   image_mean: None
-2022-12-21,10:27:26 | INFO |   image_std: None
-2022-12-21,10:27:26 | INFO |   imagenet_v2: None
-2022-12-21,10:27:26 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:27:26 | INFO |   local_loss: False
-2022-12-21,10:27:26 | INFO |   local_rank: 0
-2022-12-21,10:27:26 | INFO |   lock_image: False
-2022-12-21,10:27:26 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:27:26 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:27:26 | INFO |   log_level: 20
-2022-12-21,10:27:26 | INFO |   log_local: False
-2022-12-21,10:27:26 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_35.pt/out.log
-2022-12-21,10:27:26 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:27:26 | INFO |   lr: 0.0005
-2022-12-21,10:27:26 | INFO |   model: ViT-B-16
-2022-12-21,10:27:26 | INFO |   name: epoch_35.pt
-2022-12-21,10:27:26 | INFO |   no_set_device_rank: False
-2022-12-21,10:27:26 | INFO |   norm_gradient_clip: None
-2022-12-21,10:27:26 | INFO |   precision: amp_bfloat16
-2022-12-21,10:27:26 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_35.pt
-2022-12-21,10:27:26 | INFO |   pretrained_image: False
-2022-12-21,10:27:26 | INFO |   rank: 0
-2022-12-21,10:27:26 | INFO |   report_to: 
-2022-12-21,10:27:26 | INFO |   resume: None
-2022-12-21,10:27:26 | INFO |   save_frequency: 1
-2022-12-21,10:27:26 | INFO |   save_most_recent: False
-2022-12-21,10:27:26 | INFO |   seed: 0
-2022-12-21,10:27:26 | INFO |   skip_scheduler: False
-2022-12-21,10:27:26 | INFO |   tensorboard: False
-2022-12-21,10:27:26 | INFO |   tensorboard_path: 
-2022-12-21,10:27:26 | INFO |   torchscript: False
-2022-12-21,10:27:26 | INFO |   trace: False
-2022-12-21,10:27:26 | INFO |   train_data: None
-2022-12-21,10:27:26 | INFO |   train_num_samples: None
-2022-12-21,10:27:26 | INFO |   use_bn_sync: False
-2022-12-21,10:27:26 | INFO |   val_data: None
-2022-12-21,10:27:26 | INFO |   val_frequency: 1
-2022-12-21,10:27:26 | INFO |   val_num_samples: None
-2022-12-21,10:27:26 | INFO |   wandb: False
-2022-12-21,10:27:26 | INFO |   wandb_notes: 
-2022-12-21,10:27:26 | INFO |   warmup: 10000
-2022-12-21,10:27:26 | INFO |   wd: 0.2
-2022-12-21,10:27:26 | INFO |   workers: 1
-2022-12-21,10:27:26 | INFO |   world_size: 1
-2022-12-21,10:27:26 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:27:26 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:27:26 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<09:35,  1.73it/s]
  1%|          | 6/1000 [00:00<01:28, 11.30it/s]
  1%|          | 12/1000 [00:00<00:45, 21.61it/s]
  2%|▏         | 18/1000 [00:00<00:32, 30.07it/s]
  2%|▏         | 24/1000 [00:00<00:26, 37.16it/s]
  3%|▎         | 30/1000 [00:01<00:22, 42.68it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.62it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.59it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.86it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.50it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.60it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.49it/s]
  7%|▋         | 72/1000 [00:01<00:16, 56.00it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.39it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.85it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.93it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.33it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.54it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.84it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.98it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.94it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.68it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.78it/s]
 14%|█▍        | 138/1000 [00:02<00:14, 57.79it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.87it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.97it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.97it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.85it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.71it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.72it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.67it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.60it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.58it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.52it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.35it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.36it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.33it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.55it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.04it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.09it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.25it/s]
 25%|██▌       | 252/1000 [00:04<00:13, 57.43it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.59it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.58it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.43it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.49it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.72it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.84it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.03it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.00it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.98it/s]
 31%|███       | 312/1000 [00:05<00:11, 58.15it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.15it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.06it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.88it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.97it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.07it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.18it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.95it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.95it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.03it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.13it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.00it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.67it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.86it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.01it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.08it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.08it/s]
 43%|████▎     | 426/1000 [00:07<00:09, 58.12it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.20it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.97it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.09it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.05it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.03it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.18it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.09it/s]
 49%|████▊     | 486/1000 [00:08<00:08, 58.07it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.02it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.95it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.02it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.91it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.03it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.01it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.14it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.16it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.07it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.02it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.95it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.11it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.02it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.15it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.06it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.07it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.19it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.18it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.27it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.08it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.03it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.15it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.10it/s]
 66%|██████▌   | 660/1000 [00:11<00:05, 58.20it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.25it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.17it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.22it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.23it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.24it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.15it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.23it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.23it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.20it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.22it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.14it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.11it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.20it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.29it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.13it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.22it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.17it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.21it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.24it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.02it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.02it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.08it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.03it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.07it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.17it/s]
 83%|████████▎ | 834/1000 [00:14<00:02, 58.31it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.36it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.27it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.30it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.24it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.25it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.18it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.24it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.22it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.10it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.22it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.24it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.18it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.18it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.18it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.16it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.24it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.28it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.27it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.28it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.31it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.25it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.27it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.35it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.38it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.33it/s]
-2022-12-21,10:27:44 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:35, 78.55it/s]
  1%|          | 512/50176 [00:05<09:13, 89.65it/s]
  2%|▏         | 768/50176 [00:07<08:09, 100.87it/s]
  2%|▏         | 1024/50176 [00:10<07:57, 102.85it/s]
  3%|▎         | 1280/50176 [00:12<07:52, 103.49it/s]
  3%|▎         | 1536/50176 [00:15<08:18, 97.49it/s] 
  4%|▎         | 1792/50176 [00:18<08:05, 99.76it/s]
  4%|▍         | 2048/50176 [00:20<07:50, 102.20it/s]
  5%|▍         | 2304/50176 [00:23<07:50, 101.67it/s]
  5%|▌         | 2560/50176 [00:25<07:41, 103.09it/s]
  6%|▌         | 2816/50176 [00:28<07:47, 101.33it/s]
  6%|▌         | 3072/50176 [00:30<07:32, 104.18it/s]
  7%|▋         | 3328/50176 [00:32<07:19, 106.55it/s]
  7%|▋         | 3584/50176 [00:35<07:11, 108.02it/s]
  8%|▊         | 3840/50176 [00:37<07:27, 103.48it/s]
  8%|▊         | 4096/50176 [00:40<07:31, 102.13it/s]
  9%|▊         | 4352/50176 [00:42<07:25, 102.89it/s]
  9%|▉         | 4608/50176 [00:44<07:08, 106.44it/s]
 10%|▉         | 4864/50176 [00:47<07:03, 107.12it/s]
 10%|█         | 5120/50176 [00:49<06:52, 109.23it/s]
 11%|█         | 5376/50176 [00:51<06:53, 108.25it/s]
 11%|█         | 5632/50176 [00:54<06:53, 107.79it/s]
 12%|█▏        | 5888/50176 [00:56<06:54, 106.75it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.68it/s]
 13%|█▎        | 6400/50176 [01:01<06:51, 106.46it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.15it/s]
 14%|█▍        | 6912/50176 [01:06<06:40, 107.93it/s]
 14%|█▍        | 7168/50176 [01:08<06:21, 112.69it/s]
 15%|█▍        | 7424/50176 [01:10<06:14, 114.13it/s]
 15%|█▌        | 7680/50176 [01:12<06:05, 116.13it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.50it/s]
 16%|█▋        | 8192/50176 [01:17<06:06, 114.69it/s]
 17%|█▋        | 8448/50176 [01:19<05:58, 116.32it/s]
 17%|█▋        | 8704/50176 [01:21<06:04, 113.75it/s]
 18%|█▊        | 8960/50176 [01:24<06:04, 112.99it/s]
 18%|█▊        | 9216/50176 [01:26<05:57, 114.70it/s]
 19%|█▉        | 9472/50176 [01:28<05:56, 114.27it/s]
 19%|█▉        | 9728/50176 [01:30<06:01, 111.98it/s]
 20%|█▉        | 9984/50176 [01:33<05:58, 112.26it/s]
 20%|██        | 10240/50176 [01:35<05:42, 116.54it/s]
 21%|██        | 10496/50176 [01:37<05:41, 116.25it/s]
 21%|██▏       | 10752/50176 [01:39<05:54, 111.29it/s]
 22%|██▏       | 11008/50176 [01:42<05:53, 110.71it/s]
 22%|██▏       | 11264/50176 [01:44<05:40, 114.22it/s]
 23%|██▎       | 11520/50176 [01:46<05:34, 115.58it/s]
 23%|██▎       | 11776/50176 [01:48<05:28, 117.07it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.46it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.51it/s]
 25%|██▌       | 12544/50176 [01:55<05:22, 116.78it/s]
 26%|██▌       | 12800/50176 [01:57<05:13, 119.16it/s]
 26%|██▌       | 13056/50176 [01:59<05:10, 119.58it/s]
 27%|██▋       | 13312/50176 [02:01<05:10, 118.68it/s]
 27%|██▋       | 13568/50176 [02:04<05:28, 111.59it/s]
 28%|██▊       | 13824/50176 [02:06<05:29, 110.19it/s]
 28%|██▊       | 14080/50176 [02:08<05:29, 109.69it/s]
 29%|██▊       | 14336/50176 [02:11<05:32, 107.70it/s]
 29%|██▉       | 14592/50176 [02:13<05:18, 111.77it/s]
 30%|██▉       | 14848/50176 [02:15<05:29, 107.33it/s]
 30%|███       | 15104/50176 [02:18<05:28, 106.83it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.79it/s]
 31%|███       | 15616/50176 [02:22<05:12, 110.45it/s]
 32%|███▏      | 15872/50176 [02:25<05:17, 108.02it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.55it/s]
 33%|███▎      | 16384/50176 [02:29<05:06, 110.09it/s]
 33%|███▎      | 16640/50176 [02:32<05:20, 104.60it/s]
 34%|███▎      | 16896/50176 [02:34<05:10, 107.05it/s]
 34%|███▍      | 17152/50176 [02:37<05:04, 108.57it/s]
 35%|███▍      | 17408/50176 [02:39<04:56, 110.70it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.42it/s]
 36%|███▌      | 17920/50176 [02:43<04:39, 115.36it/s]
 36%|███▌      | 18176/50176 [02:45<04:38, 114.92it/s]
 37%|███▋      | 18432/50176 [02:48<04:37, 114.38it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.00it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.63it/s]
 38%|███▊      | 19200/50176 [02:55<04:43, 109.32it/s]
 39%|███▉      | 19456/50176 [02:57<04:49, 106.26it/s]
 39%|███▉      | 19712/50176 [03:00<04:51, 104.48it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.10it/s]
 40%|████      | 20224/50176 [03:04<04:32, 109.97it/s]
 41%|████      | 20480/50176 [03:06<04:20, 114.18it/s]
 41%|████▏     | 20736/50176 [03:09<04:17, 114.46it/s]
 42%|████▏     | 20992/50176 [03:11<04:04, 119.44it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.87it/s]
 43%|████▎     | 21504/50176 [03:15<04:10, 114.61it/s]
 43%|████▎     | 21760/50176 [03:18<04:08, 114.57it/s]
 44%|████▍     | 22016/50176 [03:19<03:55, 119.44it/s]
 44%|████▍     | 22272/50176 [03:22<03:55, 118.58it/s]
 45%|████▍     | 22528/50176 [03:24<03:54, 118.11it/s]
 45%|████▌     | 22784/50176 [03:26<04:02, 112.92it/s]
 46%|████▌     | 23040/50176 [03:29<04:02, 111.80it/s]
 46%|████▋     | 23296/50176 [03:31<04:07, 108.39it/s]
 47%|████▋     | 23552/50176 [03:34<04:09, 106.76it/s]
 47%|████▋     | 23808/50176 [03:36<03:55, 111.75it/s]
 48%|████▊     | 24064/50176 [03:38<03:49, 113.56it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.68it/s]
 49%|████▉     | 24576/50176 [03:43<03:47, 112.77it/s]
 49%|████▉     | 24832/50176 [03:45<03:42, 113.98it/s]
 50%|█████     | 25088/50176 [03:47<03:35, 116.45it/s]
 51%|█████     | 25344/50176 [03:49<03:30, 118.21it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 114.22it/s]
 52%|█████▏    | 25856/50176 [03:53<03:29, 116.22it/s]
 52%|█████▏    | 26112/50176 [03:56<03:28, 115.59it/s]
 53%|█████▎    | 26368/50176 [03:58<03:31, 112.30it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 115.13it/s]
 54%|█████▎    | 26880/50176 [04:03<03:24, 113.95it/s]
 54%|█████▍    | 27136/50176 [04:05<03:21, 114.55it/s]
 55%|█████▍    | 27392/50176 [04:07<03:15, 116.31it/s]
 55%|█████▌    | 27648/50176 [04:09<03:03, 122.66it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.30it/s]
 56%|█████▌    | 28160/50176 [04:14<03:20, 110.02it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.46it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.77it/s]
 58%|█████▊    | 28928/50176 [04:21<03:08, 112.52it/s]
 58%|█████▊    | 29184/50176 [04:23<03:09, 110.77it/s]
 59%|█████▊    | 29440/50176 [04:26<03:15, 105.88it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.22it/s]
 60%|█████▉    | 29952/50176 [04:31<03:11, 105.47it/s]
 60%|██████    | 30208/50176 [04:33<03:06, 107.24it/s]
 61%|██████    | 30464/50176 [04:35<02:56, 111.95it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.40it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.38it/s]
 62%|██████▏   | 31232/50176 [04:42<02:45, 114.57it/s]
 63%|██████▎   | 31488/50176 [04:44<02:41, 116.03it/s]
 63%|██████▎   | 31744/50176 [04:46<02:37, 117.26it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.73it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 117.00it/s]
 65%|██████▍   | 32512/50176 [04:53<02:38, 111.22it/s]
 65%|██████▌   | 32768/50176 [04:55<02:32, 114.52it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.73it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.89it/s]
 67%|██████▋   | 33536/50176 [05:02<02:30, 110.91it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.36it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.55it/s]
 68%|██████▊   | 34304/50176 [05:08<02:16, 116.58it/s]
 69%|██████▉   | 34560/50176 [05:11<02:19, 111.74it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.77it/s]
 70%|██████▉   | 35072/50176 [05:16<02:18, 108.82it/s]
 70%|███████   | 35328/50176 [05:18<02:13, 110.93it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 109.77it/s]
 71%|███████▏  | 35840/50176 [05:23<02:12, 108.25it/s]
 72%|███████▏  | 36096/50176 [05:25<02:07, 110.12it/s]
 72%|███████▏  | 36352/50176 [05:28<02:09, 107.10it/s]
 73%|███████▎  | 36608/50176 [05:30<02:05, 108.20it/s]
 73%|███████▎  | 36864/50176 [05:32<02:03, 107.41it/s]
 74%|███████▍  | 37120/50176 [05:35<02:02, 106.28it/s]
 74%|███████▍  | 37376/50176 [05:37<01:53, 112.32it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.43it/s]
 76%|███████▌  | 37888/50176 [05:42<01:51, 110.14it/s]
 76%|███████▌  | 38144/50176 [05:44<01:46, 112.83it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 110.95it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.37it/s]
 78%|███████▊  | 38912/50176 [05:51<01:42, 110.09it/s]
 78%|███████▊  | 39168/50176 [05:53<01:41, 108.87it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.91it/s]
 79%|███████▉  | 39680/50176 [05:58<01:37, 107.87it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.36it/s]
 80%|████████  | 40192/50176 [06:03<01:34, 105.92it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.16it/s]
 81%|████████  | 40704/50176 [06:08<01:33, 101.81it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.60it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.45it/s]
 83%|████████▎ | 41472/50176 [06:15<01:22, 105.97it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.55it/s]
 84%|████████▎ | 41984/50176 [06:20<01:15, 108.51it/s]
 84%|███████���▍ | 42240/50176 [06:22<01:13, 108.42it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 110.06it/s]
 85%|████████▌ | 42752/50176 [06:27<01:07, 109.47it/s]
 86%|████████▌ | 43008/50176 [06:29<01:05, 110.08it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.56it/s]
 87%|████████▋ | 43520/50176 [06:34<01:02, 105.98it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.07it/s]
 88%|████████▊ | 44032/50176 [06:39<00:58, 105.31it/s]
 88%|████████▊ | 44288/50176 [06:41<00:54, 107.69it/s]
 89%|████████▉ | 44544/50176 [06:44<00:54, 104.17it/s]
 89%|████████▉ | 44800/50176 [06:46<00:51, 104.89it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.03it/s]
 90%|█████████ | 45312/50176 [06:51<00:45, 106.30it/s]
 91%|█████████ | 45568/50176 [06:53<00:41, 110.14it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.33it/s]
 92%|█████████▏| 46080/50176 [06:58<00:39, 104.48it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.42it/s]
 93%|█████████▎| 46592/50176 [07:03<00:33, 108.34it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.16it/s]
 94%|█████████▍| 47104/50176 [07:08<00:29, 102.41it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.35it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.79it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.08it/s]
 96%|█████████▌| 48128/50176 [07:18<00:19, 105.02it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.21it/s]
 97%|█████████▋| 48640/50176 [07:23<00:14, 105.29it/s]
 97%|█████████▋| 48896/50176 [07:25<00:12, 106.23it/s]
 98%|█████████▊| 49152/50176 [07:28<00:09, 104.03it/s]
 98%|█████████▊| 49408/50176 [07:30<00:07, 103.43it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.77it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.98it/s] 
-2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.5998	imagenet-zeroshot-val-top5: 0.8633

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_36.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt).
-2022-12-21,10:27:27 | INFO | Model:
-2022-12-21,10:27:27 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:27:27 | INFO | Params:
-2022-12-21,10:27:27 | INFO |   batch_size: 256
-2022-12-21,10:27:27 | INFO |   beta1: 0.9
-2022-12-21,10:27:27 | INFO |   beta2: 0.98
-2022-12-21,10:27:27 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/checkpoints
-2022-12-21,10:27:27 | INFO |   copy_codebase: False
-2022-12-21,10:27:27 | INFO |   csv_caption_key: title
-2022-12-21,10:27:27 | INFO |   csv_img_key: filepath
-2022-12-21,10:27:27 | INFO |   csv_separator: 	
-2022-12-21,10:27:27 | INFO |   dataset_resampled: False
-2022-12-21,10:27:27 | INFO |   dataset_type: auto
-2022-12-21,10:27:27 | INFO |   ddp_static_graph: False
-2022-12-21,10:27:27 | INFO |   debug: False
-2022-12-21,10:27:27 | INFO |   device: cuda:0
-2022-12-21,10:27:27 | INFO |   dist_backend: nccl
-2022-12-21,10:27:27 | INFO |   dist_url: env://
-2022-12-21,10:27:27 | INFO |   distributed: False
-2022-12-21,10:27:27 | INFO |   epochs: 32
-2022-12-21,10:27:27 | INFO |   eps: 1e-06
-2022-12-21,10:27:27 | INFO |   force_quick_gelu: False
-2022-12-21,10:27:27 | INFO |   gather_with_grad: False
-2022-12-21,10:27:27 | INFO |   grad_checkpointing: False
-2022-12-21,10:27:27 | INFO |   horovod: False
-2022-12-21,10:27:27 | INFO |   image_mean: None
-2022-12-21,10:27:27 | INFO |   image_std: None
-2022-12-21,10:27:27 | INFO |   imagenet_v2: None
-2022-12-21,10:27:27 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:27:27 | INFO |   local_loss: False
-2022-12-21,10:27:27 | INFO |   local_rank: 0
-2022-12-21,10:27:27 | INFO |   lock_image: False
-2022-12-21,10:27:27 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:27:27 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:27:27 | INFO |   log_level: 20
-2022-12-21,10:27:27 | INFO |   log_local: False
-2022-12-21,10:27:27 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_36.pt/out.log
-2022-12-21,10:27:27 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:27:27 | INFO |   lr: 0.0005
-2022-12-21,10:27:27 | INFO |   model: ViT-B-16
-2022-12-21,10:27:27 | INFO |   name: epoch_36.pt
-2022-12-21,10:27:27 | INFO |   no_set_device_rank: False
-2022-12-21,10:27:27 | INFO |   norm_gradient_clip: None
-2022-12-21,10:27:27 | INFO |   precision: amp_bfloat16
-2022-12-21,10:27:27 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_36.pt
-2022-12-21,10:27:27 | INFO |   pretrained_image: False
-2022-12-21,10:27:27 | INFO |   rank: 0
-2022-12-21,10:27:27 | INFO |   report_to: 
-2022-12-21,10:27:27 | INFO |   resume: None
-2022-12-21,10:27:27 | INFO |   save_frequency: 1
-2022-12-21,10:27:27 | INFO |   save_most_recent: False
-2022-12-21,10:27:27 | INFO |   seed: 0
-2022-12-21,10:27:27 | INFO |   skip_scheduler: False
-2022-12-21,10:27:27 | INFO |   tensorboard: False
-2022-12-21,10:27:27 | INFO |   tensorboard_path: 
-2022-12-21,10:27:27 | INFO |   torchscript: False
-2022-12-21,10:27:27 | INFO |   trace: False
-2022-12-21,10:27:27 | INFO |   train_data: None
-2022-12-21,10:27:27 | INFO |   train_num_samples: None
-2022-12-21,10:27:27 | INFO |   use_bn_sync: False
-2022-12-21,10:27:27 | INFO |   val_data: None
-2022-12-21,10:27:27 | INFO |   val_frequency: 1
-2022-12-21,10:27:27 | INFO |   val_num_samples: None
-2022-12-21,10:27:27 | INFO |   wandb: False
-2022-12-21,10:27:27 | INFO |   wandb_notes: 
-2022-12-21,10:27:27 | INFO |   warmup: 10000
-2022-12-21,10:27:27 | INFO |   wd: 0.2
-2022-12-21,10:27:27 | INFO |   workers: 1
-2022-12-21,10:27:27 | INFO |   world_size: 1
-2022-12-21,10:27:27 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:27:27 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:27:27 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:30,  1.45it/s]
  1%|          | 6/1000 [00:00<01:42,  9.70it/s]
  1%|          | 12/1000 [00:00<00:51, 19.04it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.42it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.63it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.49it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.01it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.39it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.94it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.80it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.13it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.14it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.80it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.13it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.66it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.96it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.35it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.47it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.74it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.88it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.78it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.82it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.79it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.81it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.88it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.90it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.78it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.65it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.66it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.62it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.55it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.52it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.47it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.27it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.17it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.27it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.25it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.46it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.45it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.35it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.43it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.52it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.63it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 55.09it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 55.68it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 56.25it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 56.80it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.14it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.55it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.66it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.70it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.85it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.88it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.84it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.70it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.79it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.91it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.01it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.80it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.81it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.88it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.97it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.86it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.54it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.61it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.72it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.84it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.92it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.92it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.94it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.97it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.08it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.08it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.86it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.97it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.96it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.93it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.07it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.98it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.96it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.96it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.87it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.92it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.82it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.92it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.00it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.95it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.05it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.05it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.95it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.95it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.00it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.90it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.84it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.98it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.95it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.87it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.00it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.93it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.93it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.05it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.04it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.11it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.92it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.86it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.02it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.97it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.05it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.10it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.03it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.09it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.10it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.13it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.02it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.10it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.07it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.03it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.04it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.96it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.02it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.01it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.07it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.14it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.97it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.08it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.96it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.00it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.05it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.88it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.88it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.95it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.92it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.92it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.01it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.14it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.17it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.09it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.14it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.08it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.08it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.97it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.02it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.92it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.02it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.02it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.99it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.02it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.99it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.08it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.11it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.11it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.12it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.15it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.09it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.17it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.25it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.87it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.97it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.00it/s]
-2022-12-21,10:27:45 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<08:54, 93.33it/s]
  1%|          | 512/50176 [00:05<08:29, 97.48it/s]
  2%|▏         | 768/50176 [00:07<07:47, 105.61it/s]
  2%|▏         | 1024/50176 [00:09<07:43, 106.10it/s]
  3%|▎         | 1280/50176 [00:12<07:44, 105.19it/s]
  3%|▎         | 1536/50176 [00:15<08:12, 98.73it/s] 
  4%|▎         | 1792/50176 [00:17<08:00, 100.72it/s]
  4%|▍         | 2048/50176 [00:20<07:47, 103.00it/s]
  5%|▍         | 2304/50176 [00:22<07:48, 102.11it/s]
  5%|▌         | 2560/50176 [00:25<07:41, 103.28it/s]
  6%|▌         | 2816/50176 [00:27<07:46, 101.50it/s]
  6%|▌         | 3072/50176 [00:29<07:31, 104.39it/s]
  7%|▋         | 3328/50176 [00:32<07:19, 106.55it/s]
  7%|▋         | 3584/50176 [00:34<07:10, 108.18it/s]
  8%|▊         | 3840/50176 [00:37<07:26, 103.67it/s]
  8%|▊         | 4096/50176 [00:39<07:31, 102.01it/s]
  9%|▊         | 4352/50176 [00:42<07:24, 103.03it/s]
  9%|▉         | 4608/50176 [00:44<07:07, 106.54it/s]
 10%|▉         | 4864/50176 [00:46<07:03, 107.11it/s]
 10%|█         | 5120/50176 [00:49<06:52, 109.16it/s]
 11%|█         | 5376/50176 [00:51<06:54, 108.05it/s]
 11%|█         | 5632/50176 [00:53<06:52, 107.88it/s]
 12%|█▏        | 5888/50176 [00:56<06:54, 106.76it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.59it/s]
 13%|█▎        | 6400/50176 [01:01<06:51, 106.45it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.13it/s]
 14%|█▍        | 6912/50176 [01:05<06:40, 107.92it/s]
 14%|█▍        | 7168/50176 [01:07<06:21, 112.71it/s]
 15%|█▍        | 7424/50176 [01:10<06:13, 114.36it/s]
 15%|█▌        | 7680/50176 [01:12<06:06, 115.87it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.48it/s]
 16%|█▋        | 8192/50176 [01:16<06:05, 114.79it/s]
 17%|█▋        | 8448/50176 [01:18<05:58, 116.24it/s]
 17%|█▋        | 8704/50176 [01:21<06:04, 113.74it/s]
 18%|█▊        | 8960/50176 [01:23<06:04, 112.98it/s]
 18%|█▊        | 9216/50176 [01:25<05:56, 114.87it/s]
 19%|█▉        | 9472/50176 [01:27<05:57, 113.78it/s]
 19%|█▉        | 9728/50176 [01:30<06:00, 112.25it/s]
 20%|█▉        | 9984/50176 [01:32<05:58, 112.25it/s]
 20%|██        | 10240/50176 [01:34<05:42, 116.70it/s]
 21%|██        | 10496/50176 [01:36<05:41, 116.20it/s]
 21%|██▏       | 10752/50176 [01:39<05:54, 111.32it/s]
 22%|██▏       | 11008/50176 [01:41<05:53, 110.69it/s]
 22%|██▏       | 11264/50176 [01:43<05:40, 114.27it/s]
 23%|██▎       | 11520/50176 [01:45<05:34, 115.48it/s]
 23%|██▎       | 11776/50176 [01:47<05:27, 117.15it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.40it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.50it/s]
 25%|██▌       | 12544/50176 [01:54<05:22, 116.65it/s]
 26%|██▌       | 12800/50176 [01:56<05:13, 119.17it/s]
 26%|██▌       | 13056/50176 [01:58<05:10, 119.48it/s]
 27%|██▋       | 13312/50176 [02:00<05:10, 118.79it/s]
 27%|██▋       | 13568/50176 [02:03<05:27, 111.80it/s]
 28%|██▊       | 13824/50176 [02:05<05:29, 110.26it/s]
 28%|██▊       | 14080/50176 [02:08<05:28, 109.76it/s]
 29%|██▊       | 14336/50176 [02:10<05:32, 107.68it/s]
 29%|██▉       | 14592/50176 [02:12<05:19, 111.46it/s]
 30%|██▉       | 14848/50176 [02:15<05:28, 107.50it/s]
 30%|███       | 15104/50176 [02:17<05:27, 106.94it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.62it/s]
 31%|███       | 15616/50176 [02:22<05:13, 110.31it/s]
 32%|███▏      | 15872/50176 [02:24<05:17, 108.20it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.68it/s]
 33%|███▎      | 16384/50176 [02:29<05:07, 109.94it/s]
 33%|███▎      | 16640/50176 [02:32<05:20, 104.48it/s]
 34%|███▎      | 16896/50176 [02:34<05:10, 107.10it/s]
 34%|███▍      | 17152/50176 [02:36<05:04, 108.47it/s]
 35%|███▍      | 17408/50176 [02:38<04:56, 110.60it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.53it/s]
 36%|███▌      | 17920/50176 [02:43<04:40, 115.08it/s]
 36%|███▌      | 18176/50176 [02:45<04:38, 115.08it/s]
 37%|███▋      | 18432/50176 [02:47<04:37, 114.30it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.03it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.71it/s]
 38%|███▊      | 19200/50176 [02:54<04:42, 109.51it/s]
 39%|███▉      | 19456/50176 [02:57<04:48, 106.32it/s]
 39%|███▉      | 19712/50176 [02:59<04:51, 104.43it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.06it/s]
 40%|████      | 20224/50176 [03:04<04:31, 110.17it/s]
 41%|████      | 20480/50176 [03:06<04:20, 114.13it/s]
 41%|████▏     | 20736/50176 [03:08<04:17, 114.46it/s]
 42%|████▏     | 20992/50176 [03:10<04:04, 119.22it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.98it/s]
 43%|████▎     | 21504/50176 [03:15<04:09, 114.70it/s]
 43%|████▎     | 21760/50176 [03:17<04:07, 114.61it/s]
 44%|████▍     | 22016/50176 [03:19<03:56, 119.28it/s]
 44%|████▍     | 22272/50176 [03:21<03:55, 118.56it/s]
 45%|████▍     | 22528/50176 [03:23<03:54, 118.13it/s]
 45%|████▌     | 22784/50176 [03:26<04:02, 112.78it/s]
 46%|████▌     | 23040/50176 [03:28<04:02, 111.72it/s]
 46%|████▋     | 23296/50176 [03:31<04:07, 108.57it/s]
 47%|████▋     | 23552/50176 [03:33<04:09, 106.85it/s]
 47%|████▋     | 23808/50176 [03:35<03:55, 111.87it/s]
 48%|████▊     | 24064/50176 [03:37<03:49, 113.58it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.70it/s]
 49%|████▉     | 24576/50176 [03:42<03:47, 112.73it/s]
 49%|████▉     | 24832/50176 [03:44<03:42, 114.00it/s]
 50%|█████     | 25088/50176 [03:46<03:35, 116.42it/s]
 51%|█████     | 25344/50176 [03:48<03:30, 118.23it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 113.98it/s]
 52%|█████▏    | 25856/50176 [03:53<03:28, 116.46it/s]
 52%|█████▏    | 26112/50176 [03:55<03:27, 115.81it/s]
 53%|█████▎    | 26368/50176 [03:58<03:32, 112.27it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 115.10it/s]
 54%|█████▎    | 26880/50176 [04:02<03:24, 113.95it/s]
 54%|█████▍    | 27136/50176 [04:04<03:21, 114.56it/s]
 55%|█████▍    | 27392/50176 [04:06<03:15, 116.36it/s]
 55%|█████▌    | 27648/50176 [04:08<03:03, 122.62it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.12it/s]
 56%|█████▌    | 28160/50176 [04:13<03:20, 109.92it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.41it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.68it/s]
 58%|█████▊    | 28928/50176 [04:20<03:08, 112.66it/s]
 58%|█████▊    | 29184/50176 [04:22<03:09, 110.88it/s]
 59%|█████▊    | 29440/50176 [04:25<03:15, 106.00it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.16it/s]
 60%|█████▉    | 29952/50176 [04:30<03:11, 105.40it/s]
 60%|██████    | 30208/50176 [04:32<03:06, 107.24it/s]
 61%|██████    | 30464/50176 [04:34<02:56, 112.00it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.18it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.44it/s]
 62%|██████▏   | 31232/50176 [04:41<02:45, 114.71it/s]
 63%|██████▎   | 31488/50176 [04:43<02:41, 116.00it/s]
 63%|██████▎   | 31744/50176 [04:45<02:37, 117.07it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.80it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 116.95it/s]
 65%|██████▍   | 32512/50176 [04:52<02:38, 111.31it/s]
 65%|██████▌   | 32768/50176 [04:54<02:32, 114.36it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.83it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.95it/s]
 67%|██████▋   | 33536/50176 [05:01<02:29, 110.94it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.22it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.50it/s]
 68%|██████▊   | 34304/50176 [05:08<02:15, 116.94it/s]
 69%|██████▉   | 34560/50176 [05:10<02:19, 111.71it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.83it/s]
 70%|██████▉   | 35072/50176 [05:15<02:19, 108.59it/s]
 70%|███████   | 35328/50176 [05:17<02:13, 110.93it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 109.78it/s]
 71%|███████▏  | 35840/50176 [05:22<02:12, 108.28it/s]
 72%|███████▏  | 36096/50176 [05:25<02:08, 109.94it/s]
 72%|███████▏  | 36352/50176 [05:27<02:09, 107.08it/s]
 73%|███████▎  | 36608/50176 [05:29<02:05, 108.22it/s]
 73%|███████▎  | 36864/50176 [05:32<02:03, 107.41it/s]
 74%|███████▍  | 37120/50176 [05:34<02:02, 106.41it/s]
 74%|███████▍  | 37376/50176 [05:36<01:54, 112.20it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.48it/s]
 76%|███████▌  | 37888/50176 [05:41<01:51, 110.06it/s]
 76%|███████▌  | 38144/50176 [05:43<01:46, 112.90it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 111.01it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.28it/s]
 78%|███████▊  | 38912/50176 [05:50<01:42, 110.18it/s]
 78%|███████▊  | 39168/50176 [05:53<01:40, 109.06it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.86it/s]
 79%|███████▉  | 39680/50176 [05:57<01:37, 107.82it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.39it/s]
 80%|████████  | 40192/50176 [06:02<01:34, 105.85it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.16it/s]
 81%|████████  | 40704/50176 [06:07<01:33, 101.81it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.70it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.46it/s]
 83%|████████▎ | 41472/50176 [06:14<01:22, 106.05it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.52it/s]
 84%|████████▎ | 41984/50176 [06:19<01:15, 108.58it/s]
 84%|████████▍ | 42240/50176 [06:21<01:13, 108.48it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 109.94it/s]
 85%|████████▌ | 42752/50176 [06:26<01:07, 109.41it/s]
 86%|████████▌ | 43008/50176 [06:28<01:05, 110.04it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.61it/s]
 87%|████████▋ | 43520/50176 [06:33<01:02, 105.94it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.06it/s]
 88%|████████▊ | 44032/50176 [06:38<00:58, 105.30it/s]
 88%|████████▊ | 44288/50176 [06:40<00:54, 107.78it/s]
 89%|████████▉ | 44544/50176 [06:43<00:54, 104.14it/s]
 89%|████████▉ | 44800/50176 [06:45<00:51, 104.93it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.12it/s]
 90%|█████████ | 45312/50176 [06:50<00:45, 106.14it/s]
 91%|█████████ | 45568/50176 [06:52<00:41, 110.04it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.33it/s]
 92%|█████████▏| 46080/50176 [06:57<00:39, 104.38it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.42it/s]
 93%|█████████▎| 46592/50176 [07:02<00:33, 108.23it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.12it/s]
 94%|█████████▍| 47104/50176 [07:07<00:29, 102.51it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.38it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.86it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.10it/s]
 96%|█████████▌| 48128/50176 [07:17<00:19, 105.05it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.27it/s]
 97%|█████████▋| 48640/50176 [07:22<00:14, 105.17it/s]
 97%|█████████▋| 48896/50176 [07:24<00:12, 106.19it/s]
 98%|█████████▊| 49152/50176 [07:27<00:09, 103.82it/s]
 98%|█████████▊| 49408/50176 [07:29<00:07, 103.57it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.77it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.85it/s] 
-2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6048	imagenet-zeroshot-val-top5: 0.8640

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_37.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt).
-2022-12-21,10:27:26 | INFO | Model:
-2022-12-21,10:27:26 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:27:26 | INFO | Params:
-2022-12-21,10:27:26 | INFO |   batch_size: 256
-2022-12-21,10:27:26 | INFO |   beta1: 0.9
-2022-12-21,10:27:26 | INFO |   beta2: 0.98
-2022-12-21,10:27:26 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/checkpoints
-2022-12-21,10:27:26 | INFO |   copy_codebase: False
-2022-12-21,10:27:26 | INFO |   csv_caption_key: title
-2022-12-21,10:27:26 | INFO |   csv_img_key: filepath
-2022-12-21,10:27:26 | INFO |   csv_separator: 	
-2022-12-21,10:27:26 | INFO |   dataset_resampled: False
-2022-12-21,10:27:26 | INFO |   dataset_type: auto
-2022-12-21,10:27:26 | INFO |   ddp_static_graph: False
-2022-12-21,10:27:26 | INFO |   debug: False
-2022-12-21,10:27:26 | INFO |   device: cuda:0
-2022-12-21,10:27:26 | INFO |   dist_backend: nccl
-2022-12-21,10:27:26 | INFO |   dist_url: env://
-2022-12-21,10:27:26 | INFO |   distributed: False
-2022-12-21,10:27:26 | INFO |   epochs: 32
-2022-12-21,10:27:26 | INFO |   eps: 1e-06
-2022-12-21,10:27:26 | INFO |   force_quick_gelu: False
-2022-12-21,10:27:26 | INFO |   gather_with_grad: False
-2022-12-21,10:27:26 | INFO |   grad_checkpointing: False
-2022-12-21,10:27:26 | INFO |   horovod: False
-2022-12-21,10:27:26 | INFO |   image_mean: None
-2022-12-21,10:27:26 | INFO |   image_std: None
-2022-12-21,10:27:26 | INFO |   imagenet_v2: None
-2022-12-21,10:27:26 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:27:26 | INFO |   local_loss: False
-2022-12-21,10:27:26 | INFO |   local_rank: 0
-2022-12-21,10:27:26 | INFO |   lock_image: False
-2022-12-21,10:27:26 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:27:26 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:27:26 | INFO |   log_level: 20
-2022-12-21,10:27:26 | INFO |   log_local: False
-2022-12-21,10:27:26 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_37.pt/out.log
-2022-12-21,10:27:26 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:27:26 | INFO |   lr: 0.0005
-2022-12-21,10:27:26 | INFO |   model: ViT-B-16
-2022-12-21,10:27:26 | INFO |   name: epoch_37.pt
-2022-12-21,10:27:26 | INFO |   no_set_device_rank: False
-2022-12-21,10:27:26 | INFO |   norm_gradient_clip: None
-2022-12-21,10:27:26 | INFO |   precision: amp_bfloat16
-2022-12-21,10:27:26 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_37.pt
-2022-12-21,10:27:26 | INFO |   pretrained_image: False
-2022-12-21,10:27:26 | INFO |   rank: 0
-2022-12-21,10:27:26 | INFO |   report_to: 
-2022-12-21,10:27:26 | INFO |   resume: None
-2022-12-21,10:27:26 | INFO |   save_frequency: 1
-2022-12-21,10:27:26 | INFO |   save_most_recent: False
-2022-12-21,10:27:26 | INFO |   seed: 0
-2022-12-21,10:27:26 | INFO |   skip_scheduler: False
-2022-12-21,10:27:26 | INFO |   tensorboard: False
-2022-12-21,10:27:26 | INFO |   tensorboard_path: 
-2022-12-21,10:27:26 | INFO |   torchscript: False
-2022-12-21,10:27:26 | INFO |   trace: False
-2022-12-21,10:27:26 | INFO |   train_data: None
-2022-12-21,10:27:26 | INFO |   train_num_samples: None
-2022-12-21,10:27:26 | INFO |   use_bn_sync: False
-2022-12-21,10:27:26 | INFO |   val_data: None
-2022-12-21,10:27:26 | INFO |   val_frequency: 1
-2022-12-21,10:27:26 | INFO |   val_num_samples: None
-2022-12-21,10:27:26 | INFO |   wandb: False
-2022-12-21,10:27:26 | INFO |   wandb_notes: 
-2022-12-21,10:27:26 | INFO |   warmup: 10000
-2022-12-21,10:27:26 | INFO |   wd: 0.2
-2022-12-21,10:27:26 | INFO |   workers: 1
-2022-12-21,10:27:26 | INFO |   world_size: 1
-2022-12-21,10:27:26 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:27:26 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:27:26 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<12:01,  1.38it/s]
  1%|          | 6/1000 [00:00<01:46,  9.32it/s]
  1%|          | 11/1000 [00:00<00:58, 17.00it/s]
  2%|▏         | 17/1000 [00:01<00:38, 25.47it/s]
  2%|▏         | 23/1000 [00:01<00:29, 32.99it/s]
  3%|▎         | 29/1000 [00:01<00:24, 39.18it/s]
  4%|▎         | 35/1000 [00:01<00:21, 44.07it/s]
  4%|▍         | 41/1000 [00:01<00:20, 47.75it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.25it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.43it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.89it/s]
  6%|▋         | 65/1000 [00:01<00:16, 55.05it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.82it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.43it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.97it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.40it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.63it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.84it/s]
 11%|█         | 107/1000 [00:02<00:15, 58.13it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.26it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.27it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 58.23it/s]
 13%|█▎        | 131/1000 [00:03<00:14, 58.29it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 58.21it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 58.19it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.25it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.27it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 58.16it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.93it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.94it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.80it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.76it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.71it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.60it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.46it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.52it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.54it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.74it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.83it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.65it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 57.67it/s]
 25%|██▌       | 251/1000 [00:05<00:12, 57.74it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.91it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.95it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.72it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.82it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.95it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 58.03it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 58.26it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 58.28it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.29it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.41it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.39it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.35it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 58.14it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.22it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.26it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.38it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 58.15it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.27it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.21it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.34it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.14it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.91it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.88it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.99it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.15it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.24it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.29it/s]
 42%|████▏     | 419/1000 [00:07<00:09, 58.32it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.23it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.39it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.40it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.20it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.25it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.32it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.28it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.27it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.37it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.26it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.26it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.24it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 58.15it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.28it/s]
 51%|█████     | 509/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.20it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.23it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.35it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.25it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.32it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.38it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.27it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.26it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.31it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.19it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 58.09it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.27it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 593/1000 [00:10<00:06, 58.18it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.30it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.21it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.28it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.36it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.35it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.41it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.29it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 58.18it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.38it/s]
 65%|██████▌   | 653/1000 [00:11<00:05, 58.30it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.38it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.37it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.40it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.38it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.42it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.29it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.41it/s]
 71%|███████   | 707/1000 [00:12<00:05, 58.39it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.38it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.42it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.37it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.36it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.39it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.33it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.41it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.51it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.31it/s]
 77%|███████▋  | 767/1000 [00:13<00:03, 58.36it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.35it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.38it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.41it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.23it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.20it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.13it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.14it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.24it/s]
 83%|████████▎ | 827/1000 [00:14<00:02, 58.25it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.43it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.53it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.39it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.45it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.40it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.45it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.39it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.41it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.43it/s]
 89%|████████▊ | 887/1000 [00:15<00:01, 58.26it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.37it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.41it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.38it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.35it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.40it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.33it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.44it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.52it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.49it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.47it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.50it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.45it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.51it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.58it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.61it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.64it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.56it/s]
-2022-12-21,10:27:44 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:31, 79.09it/s]
  1%|          | 512/50176 [00:05<09:10, 90.14it/s]
  2%|▏         | 768/50176 [00:07<08:08, 101.19it/s]
  2%|▏         | 1024/50176 [00:10<07:56, 103.23it/s]
  3%|▎         | 1280/50176 [00:12<07:53, 103.32it/s]
  3%|▎         | 1536/50176 [00:15<08:17, 97.72it/s] 
  4%|▎         | 1792/50176 [00:18<08:04, 99.78it/s]
  4%|▍         | 2048/50176 [00:20<07:50, 102.33it/s]
  5%|▍         | 2304/50176 [00:23<07:49, 101.90it/s]
  5%|▌         | 2560/50176 [00:25<07:40, 103.31it/s]
  6%|▌         | 2816/50176 [00:28<07:47, 101.31it/s]
  6%|▌         | 3072/50176 [00:30<07:32, 104.21it/s]
  7%|▋         | 3328/50176 [00:32<07:20, 106.47it/s]
  7%|▋         | 3584/50176 [00:35<07:11, 107.88it/s]
  8%|▊         | 3840/50176 [00:37<07:28, 103.37it/s]
  8%|▊         | 4096/50176 [00:40<07:32, 101.88it/s]
  9%|▊         | 4352/50176 [00:42<07:25, 102.89it/s]
  9%|▉         | 4608/50176 [00:44<07:07, 106.49it/s]
 10%|▉         | 4864/50176 [00:47<07:02, 107.13it/s]
 10%|█         | 5120/50176 [00:49<06:51, 109.49it/s]
 11%|█         | 5376/50176 [00:51<06:54, 107.99it/s]
 11%|█         | 5632/50176 [00:54<06:53, 107.83it/s]
 12%|█▏        | 5888/50176 [00:56<06:55, 106.69it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.59it/s]
 13%|█▎        | 6400/50176 [01:01<06:51, 106.45it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.07it/s]
 14%|█▍        | 6912/50176 [01:06<06:41, 107.83it/s]
 14%|█▍        | 7168/50176 [01:08<06:21, 112.72it/s]
 15%|█▍        | 7424/50176 [01:10<06:14, 114.23it/s]
 15%|█▌        | 7680/50176 [01:12<06:06, 116.01it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.42it/s]
 16%|█▋        | 8192/50176 [01:17<06:05, 115.00it/s]
 17%|█▋        | 8448/50176 [01:19<05:58, 116.44it/s]
 17%|█▋        | 8704/50176 [01:21<06:05, 113.60it/s]
 18%|█▊        | 8960/50176 [01:23<06:04, 113.16it/s]
 18%|█▊        | 9216/50176 [01:26<05:56, 114.86it/s]
 19%|█▉        | 9472/50176 [01:28<05:57, 113.90it/s]
 19%|█▉        | 9728/50176 [01:30<06:00, 112.15it/s]
 20%|█▉        | 9984/50176 [01:33<05:57, 112.47it/s]
 20%|██        | 10240/50176 [01:35<05:42, 116.51it/s]
 21%|██        | 10496/50176 [01:37<05:41, 116.23it/s]
 21%|██▏       | 10752/50176 [01:39<05:53, 111.39it/s]
 22%|██▏       | 11008/50176 [01:42<05:53, 110.76it/s]
 22%|██▏       | 11264/50176 [01:44<05:40, 114.27it/s]
 23%|██▎       | 11520/50176 [01:46<05:34, 115.57it/s]
 23%|██▎       | 11776/50176 [01:48<05:27, 117.08it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.38it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.45it/s]
 25%|██▌       | 12544/50176 [01:55<05:22, 116.61it/s]
 26%|██▌       | 12800/50176 [01:57<05:13, 119.32it/s]
 26%|██▌       | 13056/50176 [01:59<05:11, 119.31it/s]
 27%|██▋       | 13312/50176 [02:01<05:11, 118.53it/s]
 27%|██▋       | 13568/50176 [02:04<05:27, 111.80it/s]
 28%|██▊       | 13824/50176 [02:06<05:29, 110.33it/s]
 28%|██▊       | 14080/50176 [02:08<05:28, 109.96it/s]
 29%|██▊       | 14336/50176 [02:11<05:32, 107.70it/s]
 29%|██▉       | 14592/50176 [02:13<05:19, 111.43it/s]
 30%|██▉       | 14848/50176 [02:15<05:28, 107.41it/s]
 30%|███       | 15104/50176 [02:18<05:28, 106.87it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.80it/s]
 31%|███       | 15616/50176 [02:22<05:12, 110.46it/s]
 32%|███▏      | 15872/50176 [02:25<05:17, 108.08it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.63it/s]
 33%|███▎      | 16384/50176 [02:29<05:07, 110.05it/s]
 33%|███▎      | 16640/50176 [02:32<05:21, 104.38it/s]
 34%|███▎      | 16896/50176 [02:34<05:11, 106.90it/s]
 34%|███▍      | 17152/50176 [02:37<05:03, 108.66it/s]
 35%|███▍      | 17408/50176 [02:39<04:55, 110.76it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.51it/s]
 36%|███▌      | 17920/50176 [02:43<04:40, 115.12it/s]
 36%|███▌      | 18176/50176 [02:45<04:37, 115.16it/s]
 37%|███▋      | 18432/50176 [02:48<04:38, 114.12it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.11it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.72it/s]
 38%|███▊      | 19200/50176 [02:55<04:43, 109.40it/s]
 39%|███▉      | 19456/50176 [02:57<04:48, 106.31it/s]
 39%|███▉      | 19712/50176 [03:00<04:51, 104.52it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.22it/s]
 40%|████      | 20224/50176 [03:04<04:32, 109.98it/s]
 41%|████      | 20480/50176 [03:06<04:20, 114.18it/s]
 41%|████▏     | 20736/50176 [03:09<04:17, 114.46it/s]
 42%|████▏     | 20992/50176 [03:11<04:04, 119.46it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.78it/s]
 43%|████▎     | 21504/50176 [03:15<04:10, 114.66it/s]
 43%|████▎     | 21760/50176 [03:18<04:08, 114.55it/s]
 44%|████▍     | 22016/50176 [03:19<03:55, 119.48it/s]
 44%|████▍     | 22272/50176 [03:22<03:55, 118.45it/s]
 45%|████▍     | 22528/50176 [03:24<03:54, 117.84it/s]
 45%|████▌     | 22784/50176 [03:26<04:03, 112.70it/s]
 46%|████▌     | 23040/50176 [03:29<04:03, 111.58it/s]
 46%|████▋     | 23296/50176 [03:31<04:07, 108.65it/s]
 47%|████▋     | 23552/50176 [03:34<04:09, 106.89it/s]
 47%|████▋     | 23808/50176 [03:36<03:55, 111.82it/s]
 48%|████▊     | 24064/50176 [03:38<03:49, 113.65it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.72it/s]
 49%|████▉     | 24576/50176 [03:43<03:46, 112.81it/s]
 49%|████▉     | 24832/50176 [03:45<03:41, 114.17it/s]
 50%|█████     | 25088/50176 [03:47<03:35, 116.40it/s]
 51%|█████     | 25344/50176 [03:49<03:30, 118.13it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 113.94it/s]
 52%|█████▏    | 25856/50176 [03:53<03:28, 116.42it/s]
 52%|█████▏    | 26112/50176 [03:56<03:27, 115.83it/s]
 53%|█████▎    | 26368/50176 [03:58<03:32, 112.29it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 114.98it/s]
 54%|█████▎    | 26880/50176 [04:02<03:24, 113.88it/s]
 54%|█████▍    | 27136/50176 [04:05<03:20, 114.71it/s]
 55%|█████▍    | 27392/50176 [04:07<03:15, 116.29it/s]
 55%|█████▌    | 27648/50176 [04:09<03:03, 122.70it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.39it/s]
 56%|█████▌    | 28160/50176 [04:14<03:20, 109.73it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.31it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.80it/s]
 58%|█████▊    | 28928/50176 [04:21<03:08, 112.64it/s]
 58%|█████▊    | 29184/50176 [04:23<03:09, 110.70it/s]
 59%|█████▊    | 29440/50176 [04:26<03:15, 105.96it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.28it/s]
 60%|█████▉    | 29952/50176 [04:31<03:11, 105.40it/s]
 60%|██████    | 30208/50176 [04:33<03:06, 107.25it/s]
 61%|██████    | 30464/50176 [04:35<02:56, 111.93it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.40it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.34it/s]
 62%|██████▏   | 31232/50176 [04:42<02:45, 114.52it/s]
 63%|██████▎   | 31488/50176 [04:44<02:40, 116.14it/s]
 63%|██████▎   | 31744/50176 [04:46<02:37, 117.26it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.66it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 117.05it/s]
 65%|██████▍   | 32512/50176 [04:53<02:38, 111.24it/s]
 65%|██████▌   | 32768/50176 [04:55<02:32, 114.51it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.73it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.91it/s]
 67%|██████▋   | 33536/50176 [05:02<02:30, 110.91it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.38it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.54it/s]
 68%|██████▊   | 34304/50176 [05:08<02:16, 116.51it/s]
 69%|██████▉   | 34560/50176 [05:11<02:19, 111.66it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.84it/s]
 70%|██████▉   | 35072/50176 [05:16<02:19, 108.64it/s]
 70%|███████   | 35328/50176 [05:18<02:13, 110.86it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 109.78it/s]
 71%|███████▏  | 35840/50176 [05:23<02:12, 108.26it/s]
 72%|███████▏  | 36096/50176 [05:25<02:07, 110.15it/s]
 72%|███████▏  | 36352/50176 [05:28<02:09, 107.11it/s]
 73%|███████▎  | 36608/50176 [05:30<02:05, 108.26it/s]
 73%|███████▎  | 36864/50176 [05:32<02:03, 107.44it/s]
 74%|███████▍  | 37120/50176 [05:35<02:02, 106.27it/s]
 74%|███████▍  | 37376/50176 [05:37<01:53, 112.41it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.37it/s]
 76%|███████▌  | 37888/50176 [05:41<01:51, 110.17it/s]
 76%|███████▌  | 38144/50176 [05:44<01:46, 112.73it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 110.93it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.31it/s]
 78%|███████▊  | 38912/50176 [05:51<01:42, 110.16it/s]
 78%|███████▊  | 39168/50176 [05:53<01:41, 108.96it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.94it/s]
 79%|███████▉  | 39680/50176 [05:58<01:37, 107.79it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.40it/s]
 80%|████████  | 40192/50176 [06:03<01:34, 105.95it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.16it/s]
 81%|████████  | 40704/50176 [06:08<01:33, 101.81it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.65it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.48it/s]
 83%|████████▎ | 41472/50176 [06:15<01:22, 106.00it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.48it/s]
 84%|████████▎ | 41984/50176 [06:20<01:15, 108.53it/s]
 84%|███████���▍ | 42240/50176 [06:22<01:13, 108.38it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 110.04it/s]
 85%|████████▌ | 42752/50176 [06:27<01:07, 109.40it/s]
 86%|████████▌ | 43008/50176 [06:29<01:05, 110.14it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.61it/s]
 87%|████████▋ | 43520/50176 [06:34<01:02, 105.87it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.10it/s]
 88%|████████▊ | 44032/50176 [06:39<00:58, 105.34it/s]
 88%|████████▊ | 44288/50176 [06:41<00:54, 107.73it/s]
 89%|████████▉ | 44544/50176 [06:44<00:54, 104.03it/s]
 89%|████████▉ | 44800/50176 [06:46<00:51, 104.96it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.11it/s]
 90%|█████████ | 45312/50176 [06:51<00:45, 106.35it/s]
 91%|█████████ | 45568/50176 [06:53<00:41, 110.10it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.33it/s]
 92%|█████████▏| 46080/50176 [06:58<00:39, 104.43it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.33it/s]
 93%|█████████▎| 46592/50176 [07:03<00:33, 108.39it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.00it/s]
 94%|█████████▍| 47104/50176 [07:08<00:29, 102.48it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.37it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.77it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.13it/s]
 96%|█████████▌| 48128/50176 [07:18<00:19, 104.99it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.23it/s]
 97%|█████████▋| 48640/50176 [07:23<00:14, 105.22it/s]
 97%|█████████▋| 48896/50176 [07:25<00:12, 106.28it/s]
 98%|█████████▊| 49152/50176 [07:28<00:09, 103.84it/s]
 98%|█████████▊| 49408/50176 [07:30<00:07, 103.53it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.73it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.87it/s] 
-2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6031	imagenet-zeroshot-val-top5: 0.8665

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_38.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:27:17 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:27:17 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:27:18 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt).
-2022-12-21,10:27:27 | INFO | Model:
-2022-12-21,10:27:27 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:27:27 | INFO | Params:
-2022-12-21,10:27:27 | INFO |   batch_size: 256
-2022-12-21,10:27:27 | INFO |   beta1: 0.9
-2022-12-21,10:27:27 | INFO |   beta2: 0.98
-2022-12-21,10:27:27 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/checkpoints
-2022-12-21,10:27:27 | INFO |   copy_codebase: False
-2022-12-21,10:27:27 | INFO |   csv_caption_key: title
-2022-12-21,10:27:27 | INFO |   csv_img_key: filepath
-2022-12-21,10:27:27 | INFO |   csv_separator: 	
-2022-12-21,10:27:27 | INFO |   dataset_resampled: False
-2022-12-21,10:27:27 | INFO |   dataset_type: auto
-2022-12-21,10:27:27 | INFO |   ddp_static_graph: False
-2022-12-21,10:27:27 | INFO |   debug: False
-2022-12-21,10:27:27 | INFO |   device: cuda:0
-2022-12-21,10:27:27 | INFO |   dist_backend: nccl
-2022-12-21,10:27:27 | INFO |   dist_url: env://
-2022-12-21,10:27:27 | INFO |   distributed: False
-2022-12-21,10:27:27 | INFO |   epochs: 32
-2022-12-21,10:27:27 | INFO |   eps: 1e-06
-2022-12-21,10:27:27 | INFO |   force_quick_gelu: False
-2022-12-21,10:27:27 | INFO |   gather_with_grad: False
-2022-12-21,10:27:27 | INFO |   grad_checkpointing: False
-2022-12-21,10:27:27 | INFO |   horovod: False
-2022-12-21,10:27:27 | INFO |   image_mean: None
-2022-12-21,10:27:27 | INFO |   image_std: None
-2022-12-21,10:27:27 | INFO |   imagenet_v2: None
-2022-12-21,10:27:27 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:27:27 | INFO |   local_loss: False
-2022-12-21,10:27:27 | INFO |   local_rank: 0
-2022-12-21,10:27:27 | INFO |   lock_image: False
-2022-12-21,10:27:27 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:27:27 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:27:27 | INFO |   log_level: 20
-2022-12-21,10:27:27 | INFO |   log_local: False
-2022-12-21,10:27:27 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_38.pt/out.log
-2022-12-21,10:27:27 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:27:27 | INFO |   lr: 0.0005
-2022-12-21,10:27:27 | INFO |   model: ViT-B-16
-2022-12-21,10:27:27 | INFO |   name: epoch_38.pt
-2022-12-21,10:27:27 | INFO |   no_set_device_rank: False
-2022-12-21,10:27:27 | INFO |   norm_gradient_clip: None
-2022-12-21,10:27:27 | INFO |   precision: amp_bfloat16
-2022-12-21,10:27:27 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_38.pt
-2022-12-21,10:27:27 | INFO |   pretrained_image: False
-2022-12-21,10:27:27 | INFO |   rank: 0
-2022-12-21,10:27:27 | INFO |   report_to: 
-2022-12-21,10:27:27 | INFO |   resume: None
-2022-12-21,10:27:27 | INFO |   save_frequency: 1
-2022-12-21,10:27:27 | INFO |   save_most_recent: False
-2022-12-21,10:27:27 | INFO |   seed: 0
-2022-12-21,10:27:27 | INFO |   skip_scheduler: False
-2022-12-21,10:27:27 | INFO |   tensorboard: False
-2022-12-21,10:27:27 | INFO |   tensorboard_path: 
-2022-12-21,10:27:27 | INFO |   torchscript: False
-2022-12-21,10:27:27 | INFO |   trace: False
-2022-12-21,10:27:27 | INFO |   train_data: None
-2022-12-21,10:27:27 | INFO |   train_num_samples: None
-2022-12-21,10:27:27 | INFO |   use_bn_sync: False
-2022-12-21,10:27:27 | INFO |   val_data: None
-2022-12-21,10:27:27 | INFO |   val_frequency: 1
-2022-12-21,10:27:27 | INFO |   val_num_samples: None
-2022-12-21,10:27:27 | INFO |   wandb: False
-2022-12-21,10:27:27 | INFO |   wandb_notes: 
-2022-12-21,10:27:27 | INFO |   warmup: 10000
-2022-12-21,10:27:27 | INFO |   wd: 0.2
-2022-12-21,10:27:27 | INFO |   workers: 1
-2022-12-21,10:27:27 | INFO |   world_size: 1
-2022-12-21,10:27:27 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:27:27 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:27:27 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:03,  1.65it/s]
  1%|          | 6/1000 [00:00<01:31, 10.88it/s]
  1%|          | 12/1000 [00:00<00:47, 20.80it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.55it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.65it/s]
  3%|▎         | 30/1000 [00:01<00:22, 42.22it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.38it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.43it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.73it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.55it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.33it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.94it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.30it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.78it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.06it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.31it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.52it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.78it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.92it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.81it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.84it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.80it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.84it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.92it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.95it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.82it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.46it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.53it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.53it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.49it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.48it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.43it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.24it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.13it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.22it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.43it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.43it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.42it/s]
 25%|██▌       | 252/1000 [00:04<00:13, 57.52it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.46it/s]
 26%|██▋       | 264/1000 [00:05<00:15, 48.60it/s]
 27%|██▋       | 270/1000 [00:05<00:14, 50.87it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 52.72it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 53.84it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 55.02it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.01it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.57it/s]
 31%|███       | 306/1000 [00:05<00:12, 56.95it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.36it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.53it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.56it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.48it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.62it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.60it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.78it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.62it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.68it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.77it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.90it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.76it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.46it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.55it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.83it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.92it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.91it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.92it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.94it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.05it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.04it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.83it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.95it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.92it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.85it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.01it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 57.92it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.72it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.78it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.72it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.80it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.74it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.83it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.87it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.95it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.89it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.97it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.98it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.88it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.89it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.93it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.85it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.65it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.84it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.85it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.80it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.92it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.85it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.85it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.96it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.96it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.05it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.86it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.80it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.96it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.93it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.02it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.89it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.87it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.96it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.00it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.05it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.95it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.00it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.00it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.97it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.95it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.96it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.99it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.98it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.95it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.04it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.89it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.01it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.97it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.01it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.04it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.84it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.84it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.89it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.87it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.84it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.86it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.95it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.09it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.13it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.04it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.09it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.03it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.03it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.98it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.03it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.04it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.91it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.02it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.01it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.98it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.00it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.98it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.96it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.01it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.01it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.03it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.06it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.01it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.08it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.15it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.86it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.90it/s]
-2022-12-21,10:27:45 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:06, 91.28it/s]
  1%|          | 512/50176 [00:05<08:36, 96.22it/s]
  2%|▏         | 768/50176 [00:07<07:50, 104.92it/s]
  2%|▏         | 1024/50176 [00:09<07:44, 105.93it/s]
  3%|▎         | 1280/50176 [00:12<07:46, 104.71it/s]
  3%|▎         | 1536/50176 [00:15<08:14, 98.37it/s] 
  4%|▎         | 1792/50176 [00:17<08:01, 100.42it/s]
  4%|▍         | 2048/50176 [00:20<07:48, 102.71it/s]
  5%|▍         | 2304/50176 [00:22<07:49, 102.06it/s]
  5%|▌         | 2560/50176 [00:25<07:40, 103.34it/s]
  6%|▌         | 2816/50176 [00:27<07:45, 101.66it/s]
  6%|▌         | 3072/50176 [00:30<07:32, 104.19it/s]
  7%|▋         | 3328/50176 [00:32<07:19, 106.58it/s]
  7%|▋         | 3584/50176 [00:34<07:10, 108.21it/s]
  8%|▊         | 3840/50176 [00:37<07:27, 103.65it/s]
  8%|▊         | 4096/50176 [00:39<07:32, 101.91it/s]
  9%|▊         | 4352/50176 [00:42<07:24, 103.14it/s]
  9%|▉         | 4608/50176 [00:44<07:08, 106.39it/s]
 10%|▉         | 4864/50176 [00:46<07:02, 107.31it/s]
 10%|█         | 5120/50176 [00:49<06:52, 109.15it/s]
 11%|█         | 5376/50176 [00:51<06:54, 107.97it/s]
 11%|█         | 5632/50176 [00:53<06:52, 107.87it/s]
 12%|█▏        | 5888/50176 [00:56<06:55, 106.56it/s]
 12%|█▏        | 6144/50176 [00:59<07:13, 101.53it/s]
 13%|█▎        | 6400/50176 [01:01<06:50, 106.61it/s]
 13%|█▎        | 6656/50176 [01:03<06:35, 110.04it/s]
 14%|█▍        | 6912/50176 [01:05<06:41, 107.83it/s]
 14%|█▍        | 7168/50176 [01:07<06:20, 112.92it/s]
 15%|█▍        | 7424/50176 [01:10<06:13, 114.35it/s]
 15%|█▌        | 7680/50176 [01:12<06:07, 115.77it/s]
 16%|█▌        | 7936/50176 [01:14<05:59, 117.36it/s]
 16%|█▋        | 8192/50176 [01:16<06:05, 114.74it/s]
 17%|█▋        | 8448/50176 [01:18<05:58, 116.30it/s]
 17%|█▋        | 8704/50176 [01:21<06:05, 113.60it/s]
 18%|█▊        | 8960/50176 [01:23<06:04, 113.08it/s]
 18%|█▊        | 9216/50176 [01:25<05:57, 114.73it/s]
 19%|█▉        | 9472/50176 [01:27<05:57, 114.01it/s]
 19%|█▉        | 9728/50176 [01:30<06:00, 112.05it/s]
 20%|█▉        | 9984/50176 [01:32<05:57, 112.39it/s]
 20%|██        | 10240/50176 [01:34<05:42, 116.53it/s]
 21%|██        | 10496/50176 [01:36<05:41, 116.25it/s]
 21%|██▏       | 10752/50176 [01:39<05:54, 111.34it/s]
 22%|██▏       | 11008/50176 [01:41<05:53, 110.78it/s]
 22%|██▏       | 11264/50176 [01:43<05:40, 114.21it/s]
 23%|██▎       | 11520/50176 [01:45<05:34, 115.59it/s]
 23%|██▎       | 11776/50176 [01:48<05:28, 117.06it/s]
 24%|██▍       | 12032/50176 [01:50<05:30, 115.43it/s]
 24%|██▍       | 12288/50176 [01:52<05:19, 118.51it/s]
 25%|██▌       | 12544/50176 [01:54<05:22, 116.64it/s]
 26%|██▌       | 12800/50176 [01:56<05:13, 119.24it/s]
 26%|██▌       | 13056/50176 [01:58<05:10, 119.43it/s]
 27%|██▋       | 13312/50176 [02:01<05:10, 118.59it/s]
 27%|██▋       | 13568/50176 [02:03<05:27, 111.68it/s]
 28%|██▊       | 13824/50176 [02:06<05:29, 110.27it/s]
 28%|██▊       | 14080/50176 [02:08<05:28, 109.74it/s]
 29%|██▊       | 14336/50176 [02:10<05:32, 107.72it/s]
 29%|██▉       | 14592/50176 [02:12<05:18, 111.59it/s]
 30%|██▉       | 14848/50176 [02:15<05:29, 107.38it/s]
 30%|███       | 15104/50176 [02:17<05:28, 106.85it/s]
 31%|███       | 15360/50176 [02:20<05:20, 108.79it/s]
 31%|███       | 15616/50176 [02:22<05:12, 110.48it/s]
 32%|███▏      | 15872/50176 [02:24<05:17, 108.06it/s]
 32%|███▏      | 16128/50176 [02:27<05:10, 109.58it/s]
 33%|███▎      | 16384/50176 [02:29<05:06, 110.09it/s]
 33%|███▎      | 16640/50176 [02:32<05:20, 104.50it/s]
 34%|███▎      | 16896/50176 [02:34<05:11, 106.99it/s]
 34%|███▍      | 17152/50176 [02:36<05:03, 108.63it/s]
 35%|███▍      | 17408/50176 [02:38<04:55, 110.71it/s]
 35%|███▌      | 17664/50176 [02:41<04:46, 113.48it/s]
 36%|███▌      | 17920/50176 [02:43<04:39, 115.21it/s]
 36%|███▌      | 18176/50176 [02:45<04:38, 115.05it/s]
 37%|███▋      | 18432/50176 [02:47<04:37, 114.23it/s]
 37%|███▋      | 18688/50176 [02:50<04:38, 113.06it/s]
 38%|███▊      | 18944/50176 [02:52<04:37, 112.71it/s]
 38%|███▊      | 19200/50176 [02:54<04:43, 109.36it/s]
 39%|███▉      | 19456/50176 [02:57<04:49, 106.29it/s]
 39%|███▉      | 19712/50176 [02:59<04:51, 104.53it/s]
 40%|███▉      | 19968/50176 [03:02<04:47, 105.13it/s]
 40%|████      | 20224/50176 [03:04<04:32, 109.96it/s]
 41%|████      | 20480/50176 [03:06<04:19, 114.23it/s]
 41%|████▏     | 20736/50176 [03:08<04:17, 114.44it/s]
 42%|████▏     | 20992/50176 [03:10<04:04, 119.41it/s]
 42%|████▏     | 21248/50176 [03:13<04:16, 112.88it/s]
 43%|████▎     | 21504/50176 [03:15<04:10, 114.60it/s]
 43%|████▎     | 21760/50176 [03:17<04:08, 114.56it/s]
 44%|████▍     | 22016/50176 [03:19<03:55, 119.43it/s]
 44%|████▍     | 22272/50176 [03:21<03:55, 118.55it/s]
 45%|████▍     | 22528/50176 [03:23<03:54, 117.96it/s]
 45%|████▌     | 22784/50176 [03:26<04:02, 112.88it/s]
 46%|████▌     | 23040/50176 [03:28<04:03, 111.61it/s]
 46%|████▋     | 23296/50176 [03:31<04:08, 108.37it/s]
 47%|████▋     | 23552/50176 [03:33<04:09, 106.87it/s]
 47%|████▋     | 23808/50176 [03:35<03:55, 111.84it/s]
 48%|████▊     | 24064/50176 [03:37<03:49, 113.63it/s]
 48%|████▊     | 24320/50176 [03:40<03:53, 110.72it/s]
 49%|████▉     | 24576/50176 [03:42<03:46, 112.80it/s]
 49%|████▉     | 24832/50176 [03:44<03:42, 114.07it/s]
 50%|█████     | 25088/50176 [03:46<03:35, 116.39it/s]
 51%|█████     | 25344/50176 [03:48<03:30, 118.15it/s]
 51%|█████     | 25600/50176 [03:51<03:35, 114.05it/s]
 52%|█████▏    | 25856/50176 [03:53<03:28, 116.40it/s]
 52%|█████▏    | 26112/50176 [03:55<03:28, 115.67it/s]
 53%|█████▎    | 26368/50176 [03:58<03:31, 112.41it/s]
 53%|█████▎    | 26624/50176 [04:00<03:24, 115.31it/s]
 54%|█████▎    | 26880/50176 [04:02<03:23, 114.22it/s]
 54%|█████▍    | 27136/50176 [04:04<03:21, 114.62it/s]
 55%|█████▍    | 27392/50176 [04:06<03:16, 116.20it/s]
 55%|█████▌    | 27648/50176 [04:08<03:03, 122.59it/s]
 56%|█████▌    | 27904/50176 [04:11<03:16, 113.22it/s]
 56%|█████▌    | 28160/50176 [04:13<03:20, 109.88it/s]
 57%|█████▋    | 28416/50176 [04:16<03:15, 111.36it/s]
 57%|█████▋    | 28672/50176 [04:18<03:14, 110.74it/s]
 58%|█████▊    | 28928/50176 [04:20<03:08, 112.52it/s]
 58%|█████▊    | 29184/50176 [04:23<03:09, 110.91it/s]
 59%|█████▊    | 29440/50176 [04:25<03:15, 105.83it/s]
 59%|█████▉    | 29696/50176 [04:28<03:16, 104.23it/s]
 60%|█████▉    | 29952/50176 [04:30<03:11, 105.69it/s]
 60%|██████    | 30208/50176 [04:32<03:06, 107.18it/s]
 61%|██████    | 30464/50176 [04:34<02:56, 111.90it/s]
 61%|██████    | 30720/50176 [04:37<02:53, 112.28it/s]
 62%|██████▏   | 30976/50176 [04:39<02:50, 112.60it/s]
 62%|██████▏   | 31232/50176 [04:41<02:45, 114.57it/s]
 63%|██████▎   | 31488/50176 [04:43<02:41, 115.96it/s]
 63%|██████▎   | 31744/50176 [04:45<02:37, 117.16it/s]
 64%|██████▍   | 32000/50176 [04:48<02:38, 114.81it/s]
 64%|██████▍   | 32256/50176 [04:50<02:33, 116.93it/s]
 65%|██████▍   | 32512/50176 [04:52<02:38, 111.20it/s]
 65%|██████▌   | 32768/50176 [04:54<02:32, 114.46it/s]
 66%|██████▌   | 33024/50176 [04:57<02:32, 112.73it/s]
 66%|██████▋   | 33280/50176 [04:59<02:28, 113.91it/s]
 67%|██████▋   | 33536/50176 [05:01<02:30, 110.91it/s]
 67%|██████▋   | 33792/50176 [05:04<02:27, 111.31it/s]
 68%|██████▊   | 34048/50176 [05:06<02:23, 112.51it/s]
 68%|██████▊   | 34304/50176 [05:08<02:16, 116.64it/s]
 69%|██████▉   | 34560/50176 [05:10<02:19, 112.00it/s]
 69%|██████▉   | 34816/50176 [05:13<02:21, 108.70it/s]
 70%|██████▉   | 35072/50176 [05:15<02:19, 108.57it/s]
 70%|███████   | 35328/50176 [05:18<02:13, 110.86it/s]
 71%|███████   | 35584/50176 [05:20<02:12, 110.00it/s]
 71%|███████▏  | 35840/50176 [05:22<02:12, 108.46it/s]
 72%|███████▏  | 36096/50176 [05:25<02:07, 110.13it/s]
 72%|███████▏  | 36352/50176 [05:27<02:09, 107.11it/s]
 73%|███████▎  | 36608/50176 [05:29<02:05, 108.12it/s]
 73%|███████▎  | 36864/50176 [05:32<02:04, 107.34it/s]
 74%|███████▍  | 37120/50176 [05:34<02:02, 106.33it/s]
 74%|███████▍  | 37376/50176 [05:36<01:53, 112.28it/s]
 75%|███████▌  | 37632/50176 [05:39<01:53, 110.51it/s]
 76%|███████▌  | 37888/50176 [05:41<01:51, 110.03it/s]
 76%|███████▌  | 38144/50176 [05:43<01:46, 112.92it/s]
 77%|███████▋  | 38400/50176 [05:46<01:46, 110.99it/s]
 77%|███████▋  | 38656/50176 [05:48<01:44, 110.40it/s]
 78%|███████▊  | 38912/50176 [05:50<01:42, 110.00it/s]
 78%|███████▊  | 39168/50176 [05:53<01:41, 108.87it/s]
 79%|███████▊  | 39424/50176 [05:55<01:37, 109.83it/s]
 79%|███████▉  | 39680/50176 [05:57<01:37, 108.08it/s]
 80%|███████▉  | 39936/50176 [06:00<01:34, 108.25it/s]
 80%|████████  | 40192/50176 [06:02<01:34, 105.88it/s]
 81%|████████  | 40448/50176 [06:05<01:34, 103.11it/s]
 81%|████████  | 40704/50176 [06:08<01:33, 101.74it/s]
 82%|████████▏ | 40960/50176 [06:10<01:26, 106.58it/s]
 82%|████████▏ | 41216/50176 [06:12<01:23, 107.44it/s]
 83%|████████▎ | 41472/50176 [06:15<01:22, 105.96it/s]
 83%|████████▎ | 41728/50176 [06:17<01:18, 107.75it/s]
 84%|████████▎ | 41984/50176 [06:19<01:15, 108.45it/s]
 84%|████████▍ | 42240/50176 [06:21<01:13, 108.44it/s]
 85%|████████▍ | 42496/50176 [06:24<01:09, 110.01it/s]
 85%|████████▌ | 42752/50176 [06:26<01:07, 109.59it/s]
 86%|████████▌ | 43008/50176 [06:28<01:05, 110.01it/s]
 86%|████████▌ | 43264/50176 [06:31<01:03, 108.48it/s]
 87%|████████▋ | 43520/50176 [06:33<01:02, 105.99it/s]
 87%|████████▋ | 43776/50176 [06:36<01:00, 105.04it/s]
 88%|████████▊ | 44032/50176 [06:38<00:58, 105.29it/s]
 88%|████████▊ | 44288/50176 [06:41<00:54, 107.69it/s]
 89%|████████▉ | 44544/50176 [06:43<00:54, 104.08it/s]
 89%|████████▉ | 44800/50176 [06:46<00:51, 104.91it/s]
 90%|████████▉ | 45056/50176 [06:48<00:47, 108.04it/s]
 90%|█████████ | 45312/50176 [06:50<00:45, 106.23it/s]
 91%|█████████ | 45568/50176 [06:52<00:41, 110.17it/s]
 91%|█████████▏| 45824/50176 [06:55<00:40, 106.34it/s]
 92%|█████████▏| 46080/50176 [06:58<00:39, 104.44it/s]
 92%|█████████▏| 46336/50176 [07:00<00:36, 104.54it/s]
 93%|█████████▎| 46592/50176 [07:02<00:33, 108.35it/s]
 93%|█████████▎| 46848/50176 [07:05<00:31, 107.18it/s]
 94%|█████████▍| 47104/50176 [07:07<00:29, 102.41it/s]
 94%|█████████▍| 47360/50176 [07:10<00:27, 101.34it/s]
 95%|█████████▍| 47616/50176 [07:13<00:25, 99.86it/s] 
 95%|█████████▌| 47872/50176 [07:15<00:22, 102.04it/s]
 96%|█████████▌| 48128/50176 [07:17<00:19, 105.15it/s]
 96%|█████████▋| 48384/50176 [07:20<00:17, 105.33it/s]
 97%|█████████▋| 48640/50176 [07:22<00:14, 105.28it/s]
 97%|█████████▋| 48896/50176 [07:24<00:12, 106.12it/s]
 98%|█████████▊| 49152/50176 [07:27<00:09, 103.97it/s]
 98%|█████████▊| 49408/50176 [07:30<00:07, 103.50it/s]
 99%|█████████▉| 49664/50176 [07:32<00:04, 103.72it/s]
 99%|█████████▉| 49920/50176 [07:35<00:02, 97.96it/s] 
-2022-12-21,10:35:21 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:35:21 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6101	imagenet-zeroshot-val-top5: 0.8697

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_39.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt).
-2022-12-21,10:35:42 | INFO | Model:
-2022-12-21,10:35:42 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:35:42 | INFO | Params:
-2022-12-21,10:35:42 | INFO |   batch_size: 256
-2022-12-21,10:35:42 | INFO |   beta1: 0.9
-2022-12-21,10:35:42 | INFO |   beta2: 0.98
-2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/checkpoints
-2022-12-21,10:35:42 | INFO |   copy_codebase: False
-2022-12-21,10:35:42 | INFO |   csv_caption_key: title
-2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
-2022-12-21,10:35:42 | INFO |   csv_separator: 	
-2022-12-21,10:35:42 | INFO |   dataset_resampled: False
-2022-12-21,10:35:42 | INFO |   dataset_type: auto
-2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
-2022-12-21,10:35:42 | INFO |   debug: False
-2022-12-21,10:35:42 | INFO |   device: cuda:0
-2022-12-21,10:35:42 | INFO |   dist_backend: nccl
-2022-12-21,10:35:42 | INFO |   dist_url: env://
-2022-12-21,10:35:42 | INFO |   distributed: False
-2022-12-21,10:35:42 | INFO |   epochs: 32
-2022-12-21,10:35:42 | INFO |   eps: 1e-06
-2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
-2022-12-21,10:35:42 | INFO |   gather_with_grad: False
-2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
-2022-12-21,10:35:42 | INFO |   horovod: False
-2022-12-21,10:35:42 | INFO |   image_mean: None
-2022-12-21,10:35:42 | INFO |   image_std: None
-2022-12-21,10:35:42 | INFO |   imagenet_v2: None
-2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:35:42 | INFO |   local_loss: False
-2022-12-21,10:35:42 | INFO |   local_rank: 0
-2022-12-21,10:35:42 | INFO |   lock_image: False
-2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:35:42 | INFO |   log_level: 20
-2022-12-21,10:35:42 | INFO |   log_local: False
-2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_39.pt/out.log
-2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:35:42 | INFO |   lr: 0.0005
-2022-12-21,10:35:42 | INFO |   model: ViT-B-16
-2022-12-21,10:35:42 | INFO |   name: epoch_39.pt
-2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
-2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
-2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
-2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_39.pt
-2022-12-21,10:35:42 | INFO |   pretrained_image: False
-2022-12-21,10:35:42 | INFO |   rank: 0
-2022-12-21,10:35:42 | INFO |   report_to: 
-2022-12-21,10:35:42 | INFO |   resume: None
-2022-12-21,10:35:42 | INFO |   save_frequency: 1
-2022-12-21,10:35:42 | INFO |   save_most_recent: False
-2022-12-21,10:35:42 | INFO |   seed: 0
-2022-12-21,10:35:42 | INFO |   skip_scheduler: False
-2022-12-21,10:35:42 | INFO |   tensorboard: False
-2022-12-21,10:35:42 | INFO |   tensorboard_path: 
-2022-12-21,10:35:42 | INFO |   torchscript: False
-2022-12-21,10:35:42 | INFO |   trace: False
-2022-12-21,10:35:42 | INFO |   train_data: None
-2022-12-21,10:35:42 | INFO |   train_num_samples: None
-2022-12-21,10:35:42 | INFO |   use_bn_sync: False
-2022-12-21,10:35:42 | INFO |   val_data: None
-2022-12-21,10:35:42 | INFO |   val_frequency: 1
-2022-12-21,10:35:42 | INFO |   val_num_samples: None
-2022-12-21,10:35:42 | INFO |   wandb: False
-2022-12-21,10:35:42 | INFO |   wandb_notes: 
-2022-12-21,10:35:42 | INFO |   warmup: 10000
-2022-12-21,10:35:42 | INFO |   wd: 0.2
-2022-12-21,10:35:42 | INFO |   workers: 1
-2022-12-21,10:35:42 | INFO |   world_size: 1
-2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:35:42 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:35:42 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:43,  1.55it/s]
  1%|          | 6/1000 [00:00<01:37, 10.23it/s]
  1%|          | 11/1000 [00:00<00:54, 18.21it/s]
  2%|▏         | 17/1000 [00:00<00:36, 27.18it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.62it/s]
  3%|▎         | 29/1000 [00:01<00:23, 40.63it/s]
  4%|▎         | 35/1000 [00:01<00:21, 45.10it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.48it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.83it/s]
  5%|▌         | 53/1000 [00:01<00:17, 52.73it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.97it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.87it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.55it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.09it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.42it/s]
  9%|▉         | 89/1000 [00:02<00:16, 56.86it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.10it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.30it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.56it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.68it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 57.70it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.65it/s]
 13%|█▎        | 131/1000 [00:02<00:15, 57.61it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.56it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.60it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.72it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.62it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.51it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.54it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.54it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.39it/s]
 19%|█▉        | 191/1000 [00:03<00:14, 57.38it/s]
 20%|█▉        | 197/1000 [00:04<00:14, 57.32it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.21it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.08it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.13it/s]
 22%|██▏       | 221/1000 [00:04<00:15, 51.39it/s]
 23%|██▎       | 227/1000 [00:04<00:14, 53.15it/s]
 23%|██▎       | 233/1000 [00:04<00:14, 54.40it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 55.12it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 55.74it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 56.25it/s]
 26%|██▌       | 257/1000 [00:05<00:13, 56.74it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 56.99it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 56.82it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.25it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.66it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.69it/s]
 30%|███       | 305/1000 [00:06<00:12, 57.51it/s]
 31%|███       | 311/1000 [00:06<00:11, 57.71it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 57.75it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 57.74it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.56it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.65it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 57.71it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 57.83it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.64it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 57.76it/s]
 36%|███▋      | 365/1000 [00:07<00:11, 57.51it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 57.72it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.61it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.44it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.55it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.70it/s]
 41%|████      | 407/1000 [00:07<00:10, 57.79it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 57.79it/s]
 42%|████▏     | 419/1000 [00:07<00:10, 57.82it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.77it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.92it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 57.92it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.73it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 57.82it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 57.79it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 57.77it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 57.90it/s]
 48%|████▊     | 479/1000 [00:09<00:09, 57.82it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 57.80it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.77it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.67it/s]
 50%|█████     | 503/1000 [00:09<00:08, 57.78it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.66it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.71it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 57.76it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 57.86it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 57.76it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 57.81it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 57.87it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.75it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.74it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 57.84it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.63it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.61it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 57.81it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 57.76it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.71it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 57.82it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.73it/s]
 61%|██████    | 611/1000 [00:11<00:06, 57.78it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 57.87it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 57.86it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 57.75it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.67it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.59it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 57.81it/s]
 65%|██████▌   | 653/1000 [00:12<00:06, 57.74it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 57.86it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 57.85it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 57.86it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 57.90it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 57.86it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.70it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.62it/s]
 70%|███████   | 701/1000 [00:12<00:05, 57.76it/s]
 71%|███████   | 707/1000 [00:12<00:05, 57.79it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 57.81it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 57.86it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 57.82it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 57.78it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 57.84it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 57.80it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 57.87it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.01it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 57.81it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 57.86it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 57.86it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 57.90it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 57.93it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.70it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.71it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.74it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.75it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 57.73it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 57.81it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 57.81it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 57.99it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.08it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 57.91it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 57.96it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 57.91it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 57.97it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 57.80it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 57.79it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 57.85it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.72it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 57.84it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 57.88it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 57.85it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 57.83it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 57.88it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.66it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 57.84it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 57.95it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 57.95it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 57.95it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 57.97it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 57.94it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.01it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.07it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.11it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.13it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.06it/s]
-2022-12-21,10:36:00 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:26, 79.65it/s]
  1%|          | 512/50176 [00:05<09:27, 87.56it/s]
  2%|▏         | 768/50176 [00:08<08:36, 95.57it/s]
  2%|▏         | 1024/50176 [00:10<08:25, 97.16it/s]
  3%|▎         | 1280/50176 [00:13<08:19, 97.84it/s]
  3%|▎         | 1536/50176 [00:16<08:55, 90.79it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.15it/s]
  4%|▍         | 2048/50176 [00:21<08:22, 95.78it/s]
  5%|▍         | 2304/50176 [00:24<08:21, 95.38it/s]
  5%|▌         | 2560/50176 [00:26<08:02, 98.77it/s]
  6%|▌         | 2816/50176 [00:29<08:13, 95.95it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.82it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.33it/s]
  7%|▋         | 3584/50176 [00:37<07:41, 100.95it/s]
  8%|▊         | 3840/50176 [00:39<07:49, 98.77it/s] 
  8%|▊         | 4096/50176 [00:42<08:07, 94.62it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.21it/s]
  9%|▉         | 4608/50176 [00:47<07:43, 98.26it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.34it/s]
 10%|█         | 5120/50176 [00:53<07:30, 99.97it/s]
 11%|█         | 5376/50176 [00:55<07:34, 98.61it/s]
 11%|█         | 5632/50176 [00:58<07:32, 98.36it/s]
 12%|█▏        | 5888/50176 [01:00<07:29, 98.62it/s]
 12%|█▏        | 6144/50176 [01:04<07:57, 92.26it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.85it/s]
 13%|█▎        | 6656/50176 [01:09<07:20, 98.80it/s]
 14%|█▍        | 6912/50176 [01:11<07:26, 96.80it/s]
 14%|█▍        | 7168/50176 [01:14<07:21, 97.48it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.86it/s]
 15%|█▌        | 7680/50176 [01:19<07:00, 100.98it/s]
 16%|█▌        | 7936/50176 [01:21<06:49, 103.08it/s]
 16%|█▋        | 8192/50176 [01:24<06:47, 103.02it/s]
 17%|█▋        | 8448/50176 [01:26<06:42, 103.55it/s]
 17%|█▋        | 8704/50176 [01:29<06:39, 103.77it/s]
 18%|█▊        | 8960/50176 [01:31<06:50, 100.30it/s]
 18%|█▊        | 9216/50176 [01:34<06:47, 100.55it/s]
 19%|█▉        | 9472/50176 [01:36<06:40, 101.67it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.22it/s]
 20%|█▉        | 9984/50176 [01:41<06:36, 101.28it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.47it/s]
 21%|██        | 10496/50176 [01:46<06:19, 104.62it/s]
 21%|██▏       | 10752/50176 [01:49<06:24, 102.50it/s]
 22%|██▏       | 11008/50176 [01:51<06:25, 101.51it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.35it/s]
 23%|██▎       | 11520/50176 [01:56<06:17, 102.33it/s]
 23%|██▎       | 11776/50176 [01:59<06:04, 105.27it/s]
 24%|██▍       | 12032/50176 [02:01<06:05, 104.50it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.43it/s]
 25%|██▌       | 12544/50176 [02:06<06:05, 102.87it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.67it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.74it/s]
 27%|██▋       | 13312/50176 [02:13<05:50, 105.12it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.86it/s] 
 28%|██▊       | 13824/50176 [02:19<06:19, 95.67it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.68it/s]
 29%|██▊       | 14336/50176 [02:25<06:16, 95.12it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.54it/s]
 30%|██▉       | 14848/50176 [02:30<06:16, 93.74it/s]
 30%|███       | 15104/50176 [02:33<06:07, 95.51it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.44it/s]
 31%|███       | 15616/50176 [02:38<05:47, 99.44it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.76it/s]
 32%|███▏      | 16128/50176 [02:43<05:42, 99.35it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.59it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.14it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.50it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.23it/s]
 35%|███▍      | 17408/50176 [02:56<05:30, 99.09it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.47it/s]
 36%|███▌      | 17920/50176 [03:00<05:12, 103.22it/s]
 36%|███▌      | 18176/50176 [03:03<05:11, 102.86it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.58it/s]
 37%|███▋      | 18688/50176 [03:08<05:12, 100.90it/s]
 38%|███▊      | 18944/50176 [03:11<05:11, 100.28it/s]
 38%|███▊      | 19200/50176 [03:13<05:19, 97.00it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.78it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.38it/s]
 40%|███▉      | 19968/50176 [03:22<05:18, 94.91it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.68it/s]
 41%|████      | 20480/50176 [03:26<04:55, 100.64it/s]
 41%|████▏     | 20736/50176 [03:29<04:51, 101.11it/s]
 42%|████▏     | 20992/50176 [03:31<04:38, 104.95it/s]
 42%|████▏     | 21248/50176 [03:34<04:41, 102.81it/s]
 43%|████▎     | 21504/50176 [03:36<04:35, 103.90it/s]
 43%|████▎     | 21760/50176 [03:39<04:32, 104.34it/s]
 44%|████▍     | 22016/50176 [03:41<04:23, 106.77it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.82it/s]
 45%|████▍     | 22528/50176 [03:46<04:18, 106.91it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.44it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.50it/s]
 46%|████▋     | 23296/50176 [03:54<04:30, 99.29it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.36it/s]
 47%|████▋     | 23808/50176 [03:59<04:17, 102.42it/s]
 48%|████▊     | 24064/50176 [04:01<04:13, 102.88it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.17it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 102.89it/s]
 49%|████▉     | 24832/50176 [04:08<03:59, 105.66it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 105.98it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.89it/s]
 51%|█████     | 25600/50176 [04:16<03:53, 105.20it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.33it/s]
 52%|█████▏    | 26112/50176 [04:21<03:50, 104.61it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.62it/s]
 53%|█████▎    | 26624/50176 [04:25<03:40, 106.58it/s]
 54%|█████▎    | 26880/50176 [04:28<03:44, 103.59it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.53it/s]
 55%|█████▍    | 27392/50176 [04:33<03:36, 105.16it/s]
 55%|█████▌    | 27648/50176 [04:35<03:25, 109.80it/s]
 56%|█████▌    | 27904/50176 [04:38<03:34, 103.99it/s]
 56%|█████▌    | 28160/50176 [04:40<03:34, 102.77it/s]
 57%|█████▋    | 28416/50176 [04:43<03:32, 102.32it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.79it/s]
 58%|█████▊    | 28928/50176 [04:48<03:28, 102.07it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.53it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.86it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.96it/s]
 60%|█████▉    | 29952/50176 [04:59<03:34, 94.25it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.09it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.82it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.46it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.39it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.70it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 106.06it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.31it/s]
 64%|██████▍   | 32000/50176 [05:18<02:49, 107.33it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.60it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.44it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.72it/s]
 66%|██████▌   | 33024/50176 [05:28<02:47, 102.17it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.52it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.16it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.34it/s]
 68%|██████▊   | 34048/50176 [05:38<02:31, 106.13it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.57it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.34it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.58it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.95it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.46it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.88it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.22it/s]
 72%|███████▏  | 36096/50176 [05:57<02:16, 103.47it/s]
 72%|███████▏  | 36352/50176 [06:00<02:15, 101.69it/s]
 73%|███████▎  | 36608/50176 [06:03<02:14, 100.83it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.44it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.62it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.27it/s]
 75%|███████▌  | 37632/50176 [06:12<01:58, 106.30it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.23it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.59it/s]
 77%|███████▋  | 38400/50176 [06:20<01:53, 103.79it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.39it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.08it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.73it/s]
 79%|███████▊  | 39424/50176 [06:30<01:45, 102.15it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.71it/s]
 80%|███████▉  | 39936/50176 [06:35<01:39, 103.18it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.82it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.58it/s] 
 81%|████████  | 40704/50176 [06:43<01:36, 97.95it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.50it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.81it/s]
 83%|████████▎ | 41472/50176 [06:50<01:24, 102.78it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 105.00it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.33it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.69it/s]
 85%|████████▍ | 42496/50176 [06:59<01:12, 106.64it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 103.98it/s]
 86%|████████▌ | 43008/50176 [07:04<01:09, 103.52it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.38it/s]
 87%|████████▋ | 43520/50176 [07:10<01:06, 100.63it/s]
 87%|████████▋ | 43776/50176 [07:12<01:04, 99.99it/s] 
 88%|████████▊ | 44032/50176 [07:15<01:00, 102.27it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.78it/s]
 89%|████████▉ | 44544/50176 [07:20<00:55, 101.38it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.18it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.24it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.08it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.47it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 104.08it/s]
 92%|█████████▏| 46080/50176 [07:35<00:40, 101.43it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.34it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.48it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 102.07it/s]
 94%|█████████▍| 47104/50176 [07:45<00:31, 98.48it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.81it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.00it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.25it/s]
 96%|█████████▌| 48128/50176 [07:55<00:19, 104.06it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.31it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.15it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.35it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.47it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.64it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.51it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 95.80it/s] 
-2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6116	imagenet-zeroshot-val-top5: 0.8690

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_4.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:35:34 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt).
-2022-12-21,10:35:42 | INFO | Model:
-2022-12-21,10:35:42 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:35:42 | INFO | Params:
-2022-12-21,10:35:42 | INFO |   batch_size: 256
-2022-12-21,10:35:42 | INFO |   beta1: 0.9
-2022-12-21,10:35:42 | INFO |   beta2: 0.98
-2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/checkpoints
-2022-12-21,10:35:42 | INFO |   copy_codebase: False
-2022-12-21,10:35:42 | INFO |   csv_caption_key: title
-2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
-2022-12-21,10:35:42 | INFO |   csv_separator: 	
-2022-12-21,10:35:42 | INFO |   dataset_resampled: False
-2022-12-21,10:35:42 | INFO |   dataset_type: auto
-2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
-2022-12-21,10:35:42 | INFO |   debug: False
-2022-12-21,10:35:42 | INFO |   device: cuda:0
-2022-12-21,10:35:42 | INFO |   dist_backend: nccl
-2022-12-21,10:35:42 | INFO |   dist_url: env://
-2022-12-21,10:35:42 | INFO |   distributed: False
-2022-12-21,10:35:42 | INFO |   epochs: 32
-2022-12-21,10:35:42 | INFO |   eps: 1e-06
-2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
-2022-12-21,10:35:42 | INFO |   gather_with_grad: False
-2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
-2022-12-21,10:35:42 | INFO |   horovod: False
-2022-12-21,10:35:42 | INFO |   image_mean: None
-2022-12-21,10:35:42 | INFO |   image_std: None
-2022-12-21,10:35:42 | INFO |   imagenet_v2: None
-2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:35:42 | INFO |   local_loss: False
-2022-12-21,10:35:42 | INFO |   local_rank: 0
-2022-12-21,10:35:42 | INFO |   lock_image: False
-2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:35:42 | INFO |   log_level: 20
-2022-12-21,10:35:42 | INFO |   log_local: False
-2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_4.pt/out.log
-2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:35:42 | INFO |   lr: 0.0005
-2022-12-21,10:35:42 | INFO |   model: ViT-B-16
-2022-12-21,10:35:42 | INFO |   name: epoch_4.pt
-2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
-2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
-2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
-2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_4.pt
-2022-12-21,10:35:42 | INFO |   pretrained_image: False
-2022-12-21,10:35:42 | INFO |   rank: 0
-2022-12-21,10:35:42 | INFO |   report_to: 
-2022-12-21,10:35:42 | INFO |   resume: None
-2022-12-21,10:35:42 | INFO |   save_frequency: 1
-2022-12-21,10:35:42 | INFO |   save_most_recent: False
-2022-12-21,10:35:42 | INFO |   seed: 0
-2022-12-21,10:35:42 | INFO |   skip_scheduler: False
-2022-12-21,10:35:42 | INFO |   tensorboard: False
-2022-12-21,10:35:42 | INFO |   tensorboard_path: 
-2022-12-21,10:35:42 | INFO |   torchscript: False
-2022-12-21,10:35:42 | INFO |   trace: False
-2022-12-21,10:35:42 | INFO |   train_data: None
-2022-12-21,10:35:42 | INFO |   train_num_samples: None
-2022-12-21,10:35:42 | INFO |   use_bn_sync: False
-2022-12-21,10:35:42 | INFO |   val_data: None
-2022-12-21,10:35:42 | INFO |   val_frequency: 1
-2022-12-21,10:35:42 | INFO |   val_num_samples: None
-2022-12-21,10:35:42 | INFO |   wandb: False
-2022-12-21,10:35:42 | INFO |   wandb_notes: 
-2022-12-21,10:35:42 | INFO |   warmup: 10000
-2022-12-21,10:35:42 | INFO |   wd: 0.2
-2022-12-21,10:35:42 | INFO |   workers: 1
-2022-12-21,10:35:42 | INFO |   world_size: 1
-2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:35:43 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:25,  1.46it/s]
  1%|          | 6/1000 [00:00<01:41,  9.81it/s]
  1%|          | 12/1000 [00:00<00:52, 19.00it/s]
  2%|▏         | 18/1000 [00:01<00:36, 27.23it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.41it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.26it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.78it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.19it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.72it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.48it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.82it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.75it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.43it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.83it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.35it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.67it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.06it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.25it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.51it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.65it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.64it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.59it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.62it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.57it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.61it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.68it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.53it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.39it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.38it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.31it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.23it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.18it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.14it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.97it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.98it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.93it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.16it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.14it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.07it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.14it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.24it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.32it/s]
 26%|██▋       | 264/1000 [00:05<00:14, 51.32it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 52.84it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 54.09it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 55.15it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 55.90it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 56.55it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.87it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.08it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.35it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.45it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.45it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.34it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.47it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.63it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.73it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.52it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.48it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.59it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.69it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.59it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.26it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.30it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.40it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.55it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.64it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.61it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.62it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.67it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.81it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.81it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.60it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.67it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.73it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.71it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.65it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.79it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.70it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.66it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.67it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.56it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.62it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.52it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.55it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.59it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.69it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.64it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.76it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.77it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.66it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.67it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.72it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.59it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.52it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.65it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.64it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.55it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.66it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.59it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.60it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.69it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.67it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.77it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.59it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.53it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.67it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.62it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.71it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.74it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.65it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.70it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.71it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.75it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.64it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.72it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.72it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.68it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.65it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.62it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.65it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.69it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.67it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.75it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.82it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.67it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.76it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.63it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.66it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.71it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.52it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.52it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.58it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.56it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.55it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.56it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.66it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.80it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.84it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.76it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.81it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.76it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.78it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.71it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.76it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.76it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.60it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.70it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.71it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.66it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.68it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.68it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.68it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.78it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.82it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.81it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.83it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.86it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.80it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.87it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.94it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.97it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.85it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.17it/s]
-2022-12-21,10:36:01 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:14, 89.99it/s]
  1%|          | 512/50176 [00:05<09:01, 91.68it/s]
  2%|▏         | 768/50176 [00:07<08:24, 97.99it/s]
  2%|▏         | 1024/50176 [00:10<08:19, 98.39it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.47it/s]
  3%|▎         | 1536/50176 [00:16<08:52, 91.29it/s]
  4%|▎         | 1792/50176 [00:18<08:37, 93.41it/s]
  4%|▍         | 2048/50176 [00:21<08:21, 95.94it/s]
  5%|▍         | 2304/50176 [00:24<08:21, 95.46it/s]
  5%|▌         | 2560/50176 [00:26<08:02, 98.66it/s]
  6%|▌         | 2816/50176 [00:29<08:12, 96.07it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.90it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.26it/s]
  7%|▋         | 3584/50176 [00:36<07:41, 101.01it/s]
  8%|▊         | 3840/50176 [00:39<07:49, 98.65it/s] 
  8%|▊         | 4096/50176 [00:42<08:07, 94.58it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.20it/s]
  9%|▉         | 4608/50176 [00:47<07:44, 98.21it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.36it/s]
 10%|█         | 5120/50176 [00:52<07:30, 100.10it/s]
 11%|█         | 5376/50176 [00:55<07:34, 98.58it/s] 
 11%|█         | 5632/50176 [00:58<07:33, 98.32it/s]
 12%|█▏        | 5888/50176 [01:00<07:29, 98.63it/s]
 12%|█▏        | 6144/50176 [01:03<07:57, 92.26it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.84it/s]
 13%|█▎        | 6656/50176 [01:08<07:20, 98.73it/s]
 14%|█▍        | 6912/50176 [01:11<07:27, 96.75it/s]
 14%|█▍        | 7168/50176 [01:14<07:20, 97.53it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.81it/s]
 15%|█▌        | 7680/50176 [01:19<07:01, 100.92it/s]
 16%|█▌        | 7936/50176 [01:21<06:49, 103.13it/s]
 16%|█▋        | 8192/50176 [01:23<06:47, 103.15it/s]
 17%|█▋        | 8448/50176 [01:26<06:43, 103.51it/s]
 17%|█▋        | 8704/50176 [01:28<06:39, 103.71it/s]
 18%|█▊        | 8960/50176 [01:31<06:51, 100.27it/s]
 18%|█▊        | 9216/50176 [01:34<06:47, 100.56it/s]
 19%|█▉        | 9472/50176 [01:36<06:39, 101.85it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.15it/s]
 20%|█▉        | 9984/50176 [01:41<06:36, 101.39it/s]
 20%|██        | 10240/50176 [01:43<06:25, 103.57it/s]
 21%|██        | 10496/50176 [01:46<06:19, 104.54it/s]
 21%|██▏       | 10752/50176 [01:48<06:25, 102.31it/s]
 22%|██▏       | 11008/50176 [01:51<06:26, 101.40it/s]
 22%|██▏       | 11264/50176 [01:53<06:20, 102.20it/s]
 23%|██▎       | 11520/50176 [01:56<06:16, 102.57it/s]
 23%|██▎       | 11776/50176 [01:58<06:04, 105.44it/s]
 24%|██▍       | 12032/50176 [02:01<06:05, 104.49it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.52it/s]
 25%|██▌       | 12544/50176 [02:06<06:05, 102.88it/s]
 26%|██▌       | 12800/50176 [02:08<06:07, 101.79it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.76it/s]
 27%|██▋       | 13312/50176 [02:13<05:51, 104.98it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.83it/s] 
 28%|██▊       | 13824/50176 [02:19<06:20, 95.63it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.70it/s]
 29%|██▊       | 14336/50176 [02:24<06:16, 95.19it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.64it/s]
 30%|██▉       | 14848/50176 [02:30<06:17, 93.60it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.52it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.43it/s]
 31%|███       | 15616/50176 [02:37<05:47, 99.51it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.76it/s]
 32%|███▏      | 16128/50176 [02:42<05:42, 99.39it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.64it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.09it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.44it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.36it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.12it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.57it/s]
 36%|███▌      | 17920/50176 [03:00<05:11, 103.40it/s]
 36%|███▌      | 18176/50176 [03:03<05:11, 102.74it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.70it/s]
 37%|███▋      | 18688/50176 [03:08<05:11, 101.16it/s]
 38%|███▊      | 18944/50176 [03:10<05:11, 100.12it/s]
 38%|███▊      | 19200/50176 [03:13<05:20, 96.65it/s] 
 39%|███▉      | 19456/50176 [03:16<05:21, 95.52it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.45it/s]
 40%|███▉      | 19968/50176 [03:21<05:18, 94.80it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.71it/s]
 41%|████      | 20480/50176 [03:26<04:55, 100.62it/s]
 41%|████▏     | 20736/50176 [03:29<04:51, 101.01it/s]
 42%|████▏     | 20992/50176 [03:31<04:38, 104.90it/s]
 42%|████▏     | 21248/50176 [03:33<04:41, 102.80it/s]
 43%|████▎     | 21504/50176 [03:36<04:35, 103.90it/s]
 43%|████▎     | 21760/50176 [03:38<04:32, 104.36it/s]
 44%|████▍     | 22016/50176 [03:41<04:23, 106.69it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.74it/s]
 45%|████▍     | 22528/50176 [03:45<04:18, 106.84it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.55it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.45it/s]
 46%|████▋     | 23296/50176 [03:53<04:30, 99.23it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.48it/s]
 47%|████▋     | 23808/50176 [03:58<04:17, 102.59it/s]
 48%|████▊     | 24064/50176 [04:01<04:13, 102.87it/s]
 48%|████▊     | 24320/50176 [04:03<04:15, 101.17it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 102.98it/s]
 49%|████▉     | 24832/50176 [04:08<04:00, 105.59it/s]
 50%|█████     | 25088/50176 [04:10<03:56, 105.98it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.70it/s]
 51%|█████     | 25600/50176 [04:15<03:53, 105.42it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.31it/s]
 52%|█████▏    | 26112/50176 [04:20<03:49, 104.63it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.72it/s]
 53%|█████▎    | 26624/50176 [04:25<03:41, 106.53it/s]
 54%|█████▎    | 26880/50176 [04:28<03:45, 103.40it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.74it/s]
 55%|█████▍    | 27392/50176 [04:32<03:36, 105.15it/s]
 55%|█████▌    | 27648/50176 [04:34<03:24, 109.94it/s]
 56%|█████▌    | 27904/50176 [04:37<03:34, 103.70it/s]
 56%|█████▌    | 28160/50176 [04:40<03:34, 102.72it/s]
 57%|█████▋    | 28416/50176 [04:42<03:32, 102.33it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.79it/s]
 58%|█████▊    | 28928/50176 [04:47<03:27, 102.21it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.53it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.83it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.98it/s]
 60%|█████▉    | 29952/50176 [04:58<03:34, 94.24it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.19it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.86it/s]
 61%|██████    | 30720/50176 [05:06<03:12, 101.31it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.52it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.65it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 105.96it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.22it/s]
 64%|██████▍   | 32000/50176 [05:18<02:48, 107.64it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.65it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.40it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.72it/s]
 66%|██████▌   | 33024/50176 [05:28<02:47, 102.10it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.66it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.17it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.28it/s]
 68%|██████▊   | 34048/50176 [05:37<02:31, 106.16it/s]
 68%|██████▊   | 34304/50176 [05:39<02:24, 109.54it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.28it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.69it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.69it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.60it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.90it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.04it/s]
 72%|███████▏  | 36096/50176 [05:57<02:15, 103.60it/s]
 72%|███████▏  | 36352/50176 [06:00<02:16, 101.64it/s]
 73%|███████▎  | 36608/50176 [06:02<02:14, 100.79it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.58it/s] 
 74%|███████▍  | 37120/50176 [06:07<02:11, 99.52it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.18it/s]
 75%|███████▌  | 37632/50176 [06:12<01:57, 106.36it/s]
 76%|███████▌  | 37888/50176 [06:14<01:54, 107.20it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.52it/s]
 77%|███████▋  | 38400/50176 [06:19<01:53, 103.79it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.39it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 104.06it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.66it/s]
 79%|███████▊  | 39424/50176 [06:29<01:45, 102.27it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.71it/s]
 80%|███████▉  | 39936/50176 [06:34<01:39, 103.24it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.87it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.67it/s] 
 81%|████████  | 40704/50176 [06:42<01:36, 97.77it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.47it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.92it/s]
 83%|████████▎ | 41472/50176 [06:49<01:24, 102.96it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 104.99it/s]
 84%|████████▎ | 41984/50176 [06:54<01:19, 103.55it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.69it/s]
 85%|████████▍ | 42496/50176 [06:59<01:11, 106.86it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 104.18it/s]
 86%|████████▌ | 43008/50176 [07:04<01:09, 103.80it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.51it/s]
 87%|████████▋ | 43520/50176 [07:09<01:06, 100.56it/s]
 87%|████████▋ | 43776/50176 [07:12<01:03, 100.24it/s]
 88%|████████▊ | 44032/50176 [07:14<00:59, 102.51it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.81it/s]
 89%|████████▉ | 44544/50176 [07:19<00:55, 101.44it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.13it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.20it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.18it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.31it/s]
 91%|█████████▏| 45824/50176 [07:31<00:41, 104.23it/s]
 92%|█████████▏| 46080/50176 [07:34<00:40, 101.41it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.46it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.44it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 101.99it/s]
 94%|█████████▍| 47104/50176 [07:44<00:31, 98.29it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.73it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.05it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.49it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 104.17it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.36it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.28it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.27it/s]
 98%|█████████▊| 49152/50176 [08:04<00:10, 100.42it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.45it/s]
 99%|█████████▉| 49664/50176 [08:09<00:05, 100.47it/s]
 99%|█████████▉| 49920/50176 [08:12<00:02, 95.96it/s] 
-2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4249	imagenet-zeroshot-val-top5: 0.7049

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_40.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt).
-2022-12-21,10:35:42 | INFO | Model:
-2022-12-21,10:35:42 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:35:42 | INFO | Params:
-2022-12-21,10:35:42 | INFO |   batch_size: 256
-2022-12-21,10:35:42 | INFO |   beta1: 0.9
-2022-12-21,10:35:42 | INFO |   beta2: 0.98
-2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/checkpoints
-2022-12-21,10:35:42 | INFO |   copy_codebase: False
-2022-12-21,10:35:42 | INFO |   csv_caption_key: title
-2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
-2022-12-21,10:35:42 | INFO |   csv_separator: 	
-2022-12-21,10:35:42 | INFO |   dataset_resampled: False
-2022-12-21,10:35:42 | INFO |   dataset_type: auto
-2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
-2022-12-21,10:35:42 | INFO |   debug: False
-2022-12-21,10:35:42 | INFO |   device: cuda:0
-2022-12-21,10:35:42 | INFO |   dist_backend: nccl
-2022-12-21,10:35:42 | INFO |   dist_url: env://
-2022-12-21,10:35:42 | INFO |   distributed: False
-2022-12-21,10:35:42 | INFO |   epochs: 32
-2022-12-21,10:35:42 | INFO |   eps: 1e-06
-2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
-2022-12-21,10:35:42 | INFO |   gather_with_grad: False
-2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
-2022-12-21,10:35:42 | INFO |   horovod: False
-2022-12-21,10:35:42 | INFO |   image_mean: None
-2022-12-21,10:35:42 | INFO |   image_std: None
-2022-12-21,10:35:42 | INFO |   imagenet_v2: None
-2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:35:42 | INFO |   local_loss: False
-2022-12-21,10:35:42 | INFO |   local_rank: 0
-2022-12-21,10:35:42 | INFO |   lock_image: False
-2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:35:42 | INFO |   log_level: 20
-2022-12-21,10:35:42 | INFO |   log_local: False
-2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_40.pt/out.log
-2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:35:42 | INFO |   lr: 0.0005
-2022-12-21,10:35:42 | INFO |   model: ViT-B-16
-2022-12-21,10:35:42 | INFO |   name: epoch_40.pt
-2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
-2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
-2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
-2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_40.pt
-2022-12-21,10:35:42 | INFO |   pretrained_image: False
-2022-12-21,10:35:42 | INFO |   rank: 0
-2022-12-21,10:35:42 | INFO |   report_to: 
-2022-12-21,10:35:42 | INFO |   resume: None
-2022-12-21,10:35:42 | INFO |   save_frequency: 1
-2022-12-21,10:35:42 | INFO |   save_most_recent: False
-2022-12-21,10:35:42 | INFO |   seed: 0
-2022-12-21,10:35:42 | INFO |   skip_scheduler: False
-2022-12-21,10:35:42 | INFO |   tensorboard: False
-2022-12-21,10:35:42 | INFO |   tensorboard_path: 
-2022-12-21,10:35:42 | INFO |   torchscript: False
-2022-12-21,10:35:42 | INFO |   trace: False
-2022-12-21,10:35:42 | INFO |   train_data: None
-2022-12-21,10:35:42 | INFO |   train_num_samples: None
-2022-12-21,10:35:42 | INFO |   use_bn_sync: False
-2022-12-21,10:35:42 | INFO |   val_data: None
-2022-12-21,10:35:42 | INFO |   val_frequency: 1
-2022-12-21,10:35:42 | INFO |   val_num_samples: None
-2022-12-21,10:35:42 | INFO |   wandb: False
-2022-12-21,10:35:42 | INFO |   wandb_notes: 
-2022-12-21,10:35:42 | INFO |   warmup: 10000
-2022-12-21,10:35:42 | INFO |   wd: 0.2
-2022-12-21,10:35:42 | INFO |   workers: 1
-2022-12-21,10:35:42 | INFO |   world_size: 1
-2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:35:43 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:41,  1.56it/s]
  1%|          | 6/1000 [00:00<01:35, 10.38it/s]
  1%|          | 12/1000 [00:00<00:49, 20.12it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.69it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.86it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.43it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.77it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.89it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.35it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.92it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.25it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.19it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.85it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.33it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.86it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.16it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.52it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.68it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.94it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.85it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.85it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.94it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.92it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.99it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.10it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.97it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.84it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.85it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.78it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.70it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.68it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.40it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.30it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.99it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.30it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.37it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.45it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.57it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.70it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.77it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.60it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.64it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.84it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.97it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.18it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.16it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.14it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.27it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.26it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.15it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.94it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.02it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.13it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.00it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.82it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.88it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 57.98it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.08it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.99it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.67it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.87it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.03it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.11it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.12it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.13it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.27it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.26it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.05it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.18it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.15it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.11it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.26it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.18it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.16it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.13it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.04it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.08it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.18it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.11it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.22it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.18it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.05it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.01it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.15it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.13it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.17it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.08it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.20it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.29it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.10it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.04it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.20it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.14it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.24it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.30it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.23it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.28it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.29it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.17it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.25it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.22it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.23it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.16it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.18it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.19it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.27it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.34it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.19it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.27it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.23it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.28it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.32it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.08it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.09it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.14it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.11it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.86it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.94it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.08it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.31it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.29it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.23it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.24it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.19it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.24it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.23it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.12it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.22it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.22it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.17it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.20it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.21it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.21it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.34it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.34it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.34it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.37it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.30it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.37it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.44it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.46it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.45it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.40it/s]
-2022-12-21,10:36:01 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:05, 82.39it/s]
  1%|          | 512/50176 [00:05<09:20, 88.65it/s]
  2%|▏         | 768/50176 [00:08<08:37, 95.42it/s]
  2%|▏         | 1024/50176 [00:10<08:26, 97.05it/s]
  3%|▎         | 1280/50176 [00:13<08:19, 97.84it/s]
  3%|▎         | 1536/50176 [00:16<08:54, 90.93it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.09it/s]
  4%|▍         | 2048/50176 [00:21<08:22, 95.70it/s]
  5%|▍         | 2304/50176 [00:24<08:23, 95.12it/s]
  5%|▌         | 2560/50176 [00:26<08:02, 98.79it/s]
  6%|▌         | 2816/50176 [00:29<08:12, 96.17it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.92it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.42it/s]
  7%|▋         | 3584/50176 [00:37<07:41, 100.89it/s]
  8%|▊         | 3840/50176 [00:39<07:49, 98.72it/s] 
  8%|▊         | 4096/50176 [00:42<08:06, 94.69it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.22it/s]
  9%|▉         | 4608/50176 [00:47<07:44, 98.14it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.42it/s]
 10%|█         | 5120/50176 [00:53<07:31, 99.70it/s]
 11%|█         | 5376/50176 [00:55<07:33, 98.76it/s]
 11%|█         | 5632/50176 [00:58<07:32, 98.37it/s]
 12%|█▏        | 5888/50176 [01:00<07:28, 98.71it/s]
 12%|█▏        | 6144/50176 [01:04<07:56, 92.35it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.90it/s]
 13%|█▎        | 6656/50176 [01:08<07:21, 98.68it/s]
 14%|█▍        | 6912/50176 [01:11<07:26, 96.82it/s]
 14%|█▍        | 7168/50176 [01:14<07:21, 97.41it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.79it/s]
 15%|█▌        | 7680/50176 [01:19<07:00, 100.96it/s]
 16%|█▌        | 7936/50176 [01:21<06:49, 103.07it/s]
 16%|█▋        | 8192/50176 [01:24<06:47, 103.11it/s]
 17%|█▋        | 8448/50176 [01:26<06:43, 103.43it/s]
 17%|█▋        | 8704/50176 [01:28<06:39, 103.69it/s]
 18%|█▊        | 8960/50176 [01:31<06:51, 100.17it/s]
 18%|█▊        | 9216/50176 [01:34<06:46, 100.65it/s]
 19%|█▉        | 9472/50176 [01:36<06:39, 101.78it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.27it/s]
 20%|█▉        | 9984/50176 [01:41<06:38, 100.92it/s]
 20%|██        | 10240/50176 [01:44<06:25, 103.59it/s]
 21%|██        | 10496/50176 [01:46<06:18, 104.70it/s]
 21%|██▏       | 10752/50176 [01:49<06:24, 102.40it/s]
 22%|██▏       | 11008/50176 [01:51<06:25, 101.60it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.15it/s]
 23%|██▎       | 11520/50176 [01:56<06:16, 102.60it/s]
 23%|██▎       | 11776/50176 [01:58<06:05, 105.05it/s]
 24%|██▍       | 12032/50176 [02:01<06:06, 104.20it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.27it/s]
 25%|██▌       | 12544/50176 [02:06<06:06, 102.78it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.66it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.77it/s]
 27%|██▋       | 13312/50176 [02:13<05:51, 104.94it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.78it/s] 
 28%|██▊       | 13824/50176 [02:19<06:20, 95.63it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.74it/s]
 29%|██▊       | 14336/50176 [02:25<06:16, 95.14it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.62it/s]
 30%|██▉       | 14848/50176 [02:30<06:17, 93.66it/s]
 30%|███       | 15104/50176 [02:33<06:07, 95.56it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.35it/s]
 31%|███       | 15616/50176 [02:38<05:47, 99.45it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.77it/s]
 32%|███▏      | 16128/50176 [02:43<05:42, 99.32it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.67it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.06it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.54it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.26it/s]
 35%|███▍      | 17408/50176 [02:56<05:30, 99.03it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.49it/s]
 36%|███▌      | 17920/50176 [03:00<05:12, 103.38it/s]
 36%|███▌      | 18176/50176 [03:03<05:11, 102.77it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.62it/s]
 37%|███▋      | 18688/50176 [03:08<05:11, 100.95it/s]
 38%|███▊      | 18944/50176 [03:11<05:11, 100.27it/s]
 38%|███▊      | 19200/50176 [03:13<05:19, 96.86it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.84it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.36it/s]
 40%|███▉      | 19968/50176 [03:22<05:17, 95.04it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.79it/s]
 41%|████      | 20480/50176 [03:26<04:54, 100.92it/s]
 41%|████▏     | 20736/50176 [03:29<04:51, 101.04it/s]
 42%|████▏     | 20992/50176 [03:31<04:38, 104.86it/s]
 42%|████▏     | 21248/50176 [03:34<04:41, 102.84it/s]
 43%|████▎     | 21504/50176 [03:36<04:36, 103.78it/s]
 43%|████▎     | 21760/50176 [03:39<04:32, 104.28it/s]
 44%|████▍     | 22016/50176 [03:41<04:24, 106.63it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.87it/s]
 45%|████▍     | 22528/50176 [03:46<04:18, 107.01it/s]
 45%|████▌     | 22784/50176 [03:48<04:25, 103.34it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.71it/s]
 46%|████▋     | 23296/50176 [03:54<04:30, 99.29it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.27it/s]
 47%|████▋     | 23808/50176 [03:59<04:17, 102.42it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.75it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.10it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 102.90it/s]
 49%|████▉     | 24832/50176 [04:08<04:00, 105.57it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 105.94it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.93it/s]
 51%|█████     | 25600/50176 [04:16<03:53, 105.24it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.45it/s]
 52%|█████▏    | 26112/50176 [04:20<03:50, 104.61it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.61it/s]
 53%|█████▎    | 26624/50176 [04:25<03:41, 106.46it/s]
 54%|█████▎    | 26880/50176 [04:28<03:44, 103.64it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.65it/s]
 55%|█████▍    | 27392/50176 [04:33<03:36, 105.02it/s]
 55%|█████▌    | 27648/50176 [04:35<03:24, 109.91it/s]
 56%|█████▌    | 27904/50176 [04:38<03:33, 104.16it/s]
 56%|█████▌    | 28160/50176 [04:40<03:34, 102.73it/s]
 57%|█████▋    | 28416/50176 [04:43<03:32, 102.41it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.84it/s]
 58%|█████▊    | 28928/50176 [04:48<03:28, 102.06it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.43it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.81it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.93it/s]
 60%|█████▉    | 29952/50176 [04:59<03:34, 94.32it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.12it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.97it/s]
 61%|██████    | 30720/50176 [05:06<03:12, 101.24it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.54it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.58it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 106.01it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.14it/s]
 64%|██████▍   | 32000/50176 [05:18<02:49, 107.40it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.65it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.38it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.72it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 102.09it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.58it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.17it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.46it/s]
 68%|██████▊   | 34048/50176 [05:38<02:31, 106.12it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.61it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.51it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.64it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.80it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.48it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.82it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.17it/s]
 72%|███████▏  | 36096/50176 [05:57<02:16, 103.50it/s]
 72%|███████▏  | 36352/50176 [06:00<02:16, 101.54it/s]
 73%|███████▎  | 36608/50176 [06:03<02:14, 100.97it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.50it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.52it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.35it/s]
 75%|███████▌  | 37632/50176 [06:12<01:57, 106.32it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.13it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.58it/s]
 77%|███████▋  | 38400/50176 [06:20<01:53, 103.71it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.34it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 104.05it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.75it/s]
 79%|███████▊  | 39424/50176 [06:30<01:45, 102.22it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.69it/s]
 80%|███████▉  | 39936/50176 [06:35<01:39, 103.24it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.80it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.65it/s] 
 81%|████████  | 40704/50176 [06:43<01:36, 97.91it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.41it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.77it/s]
 83%|████████▎ | 41472/50176 [06:50<01:24, 102.86it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 104.94it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.43it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.68it/s]
 85%|████████▍ | 42496/50176 [06:59<01:11, 106.89it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 103.87it/s]
 86%|████████▌ | 43008/50176 [07:04<01:08, 103.89it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.53it/s]
 87%|████████▋ | 43520/50176 [07:10<01:06, 100.83it/s]
 87%|████████▋ | 43776/50176 [07:12<01:03, 100.27it/s]
 88%|████████▊ | 44032/50176 [07:15<00:59, 102.61it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.69it/s]
 89%|████████▉ | 44544/50176 [07:20<00:55, 101.60it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.27it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.21it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.04it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.57it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 103.94it/s]
 92%|█████████▏| 46080/50176 [07:34<00:40, 101.43it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.44it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.59it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 102.09it/s]
 94%|█████████▍| 47104/50176 [07:45<00:31, 98.26it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.73it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.07it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.52it/s]
 96%|█████████▌| 48128/50176 [07:55<00:19, 104.07it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.29it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.20it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.32it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.47it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.64it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.48it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 95.90it/s] 
-2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6105	imagenet-zeroshot-val-top5: 0.8693

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_41.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:35:32 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:35:32 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:35:33 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt).
-2022-12-21,10:35:42 | INFO | Model:
-2022-12-21,10:35:42 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:35:42 | INFO | Params:
-2022-12-21,10:35:42 | INFO |   batch_size: 256
-2022-12-21,10:35:42 | INFO |   beta1: 0.9
-2022-12-21,10:35:42 | INFO |   beta2: 0.98
-2022-12-21,10:35:42 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/checkpoints
-2022-12-21,10:35:42 | INFO |   copy_codebase: False
-2022-12-21,10:35:42 | INFO |   csv_caption_key: title
-2022-12-21,10:35:42 | INFO |   csv_img_key: filepath
-2022-12-21,10:35:42 | INFO |   csv_separator: 	
-2022-12-21,10:35:42 | INFO |   dataset_resampled: False
-2022-12-21,10:35:42 | INFO |   dataset_type: auto
-2022-12-21,10:35:42 | INFO |   ddp_static_graph: False
-2022-12-21,10:35:42 | INFO |   debug: False
-2022-12-21,10:35:42 | INFO |   device: cuda:0
-2022-12-21,10:35:42 | INFO |   dist_backend: nccl
-2022-12-21,10:35:42 | INFO |   dist_url: env://
-2022-12-21,10:35:42 | INFO |   distributed: False
-2022-12-21,10:35:42 | INFO |   epochs: 32
-2022-12-21,10:35:42 | INFO |   eps: 1e-06
-2022-12-21,10:35:42 | INFO |   force_quick_gelu: False
-2022-12-21,10:35:42 | INFO |   gather_with_grad: False
-2022-12-21,10:35:42 | INFO |   grad_checkpointing: False
-2022-12-21,10:35:42 | INFO |   horovod: False
-2022-12-21,10:35:42 | INFO |   image_mean: None
-2022-12-21,10:35:42 | INFO |   image_std: None
-2022-12-21,10:35:42 | INFO |   imagenet_v2: None
-2022-12-21,10:35:42 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:35:42 | INFO |   local_loss: False
-2022-12-21,10:35:42 | INFO |   local_rank: 0
-2022-12-21,10:35:42 | INFO |   lock_image: False
-2022-12-21,10:35:42 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:35:42 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:35:42 | INFO |   log_level: 20
-2022-12-21,10:35:42 | INFO |   log_local: False
-2022-12-21,10:35:42 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_41.pt/out.log
-2022-12-21,10:35:42 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:35:42 | INFO |   lr: 0.0005
-2022-12-21,10:35:42 | INFO |   model: ViT-B-16
-2022-12-21,10:35:42 | INFO |   name: epoch_41.pt
-2022-12-21,10:35:42 | INFO |   no_set_device_rank: False
-2022-12-21,10:35:42 | INFO |   norm_gradient_clip: None
-2022-12-21,10:35:42 | INFO |   precision: amp_bfloat16
-2022-12-21,10:35:42 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_41.pt
-2022-12-21,10:35:42 | INFO |   pretrained_image: False
-2022-12-21,10:35:42 | INFO |   rank: 0
-2022-12-21,10:35:42 | INFO |   report_to: 
-2022-12-21,10:35:42 | INFO |   resume: None
-2022-12-21,10:35:42 | INFO |   save_frequency: 1
-2022-12-21,10:35:42 | INFO |   save_most_recent: False
-2022-12-21,10:35:42 | INFO |   seed: 0
-2022-12-21,10:35:42 | INFO |   skip_scheduler: False
-2022-12-21,10:35:42 | INFO |   tensorboard: False
-2022-12-21,10:35:42 | INFO |   tensorboard_path: 
-2022-12-21,10:35:42 | INFO |   torchscript: False
-2022-12-21,10:35:42 | INFO |   trace: False
-2022-12-21,10:35:42 | INFO |   train_data: None
-2022-12-21,10:35:42 | INFO |   train_num_samples: None
-2022-12-21,10:35:42 | INFO |   use_bn_sync: False
-2022-12-21,10:35:42 | INFO |   val_data: None
-2022-12-21,10:35:42 | INFO |   val_frequency: 1
-2022-12-21,10:35:42 | INFO |   val_num_samples: None
-2022-12-21,10:35:42 | INFO |   wandb: False
-2022-12-21,10:35:42 | INFO |   wandb_notes: 
-2022-12-21,10:35:42 | INFO |   warmup: 10000
-2022-12-21,10:35:42 | INFO |   wd: 0.2
-2022-12-21,10:35:42 | INFO |   workers: 1
-2022-12-21,10:35:42 | INFO |   world_size: 1
-2022-12-21,10:35:42 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:35:43 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:35:43 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:43,  1.42it/s]
  1%|          | 6/1000 [00:00<01:43,  9.61it/s]
  1%|          | 12/1000 [00:00<00:51, 19.13it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.71it/s]
  2%|▏         | 24/1000 [00:01<00:27, 34.93it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.79it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.26it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.65it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.20it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.68it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.05it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.10it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.84it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.29it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.81it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.14it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.53it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.69it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.95it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.08it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.05it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.97it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.99it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.96it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.02it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.10it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.00it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.86it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.88it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.82it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.76it/s]
 19%|█▉        | 192/1000 [00:04<00:13, 57.74it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.67it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.45it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.33it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.42it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.64it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.56it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.61it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.71it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.82it/s]
 26%|██▋       | 264/1000 [00:05<00:13, 52.59it/s]
 27%|██▋       | 270/1000 [00:05<00:13, 53.91it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 55.04it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 55.98it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 56.64it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.25it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.50it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.69it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.94it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.00it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.98it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.85it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.88it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.01it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.11it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.93it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 58.05it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.15it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.06it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.75it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.81it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.92it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.06it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.13it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.12it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.14it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.17it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.28it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.25it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.03it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.11it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.07it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.22it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 58.14it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.12it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.10it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.02it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.06it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.05it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.10it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.20it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.14it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.24it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.21it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.10it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.10it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.05it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.00it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.15it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.15it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.05it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.16it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.08it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.09it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.19it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.29it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.79it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.82it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.03it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 58.04it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.23it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.22it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.23it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.27it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.16it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.26it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.24it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.22it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.20it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.14it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.16it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.16it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.23it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.32it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.17it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.28it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.22it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.26it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.28it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.06it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.05it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.00it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.02it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.13it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.26it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.32it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.29it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.23it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.19it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.89it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.05it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.11it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.02it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.17it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.13it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.16it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.14it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.13it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.23it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.29it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.28it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.31it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.34it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.28it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.40it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.42it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.38it/s]
-2022-12-21,10:36:01 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<09:50, 84.56it/s]
  1%|          | 512/50176 [00:05<09:13, 89.78it/s]
  2%|▏         | 768/50176 [00:08<08:32, 96.38it/s]
  2%|▏         | 1024/50176 [00:10<08:23, 97.57it/s]
  3%|▎         | 1280/50176 [00:13<08:19, 97.98it/s]
  3%|▎         | 1536/50176 [00:16<08:54, 90.93it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.17it/s]
  4%|▍         | 2048/50176 [00:21<08:22, 95.71it/s]
  5%|▍         | 2304/50176 [00:24<08:21, 95.38it/s]
  5%|▌         | 2560/50176 [00:26<08:01, 98.82it/s]
  6%|▌         | 2816/50176 [00:29<08:12, 96.16it/s]
  6%|▌         | 3072/50176 [00:32<08:11, 95.80it/s]
  7%|▋         | 3328/50176 [00:34<07:56, 98.34it/s]
  7%|▋         | 3584/50176 [00:37<07:41, 100.88it/s]
  8%|▊         | 3840/50176 [00:39<07:48, 98.80it/s] 
  8%|▊         | 4096/50176 [00:42<08:06, 94.62it/s]
  9%|▊         | 4352/50176 [00:45<08:01, 95.26it/s]
  9%|▉         | 4608/50176 [00:47<07:43, 98.23it/s]
 10%|▉         | 4864/50176 [00:50<07:40, 98.32it/s]
 10%|█         | 5120/50176 [00:52<07:30, 100.05it/s]
 11%|█         | 5376/50176 [00:55<07:34, 98.59it/s] 
 11%|█         | 5632/50176 [00:58<07:33, 98.25it/s]
 12%|█▏        | 5888/50176 [01:00<07:28, 98.78it/s]
 12%|█▏        | 6144/50176 [01:03<07:57, 92.25it/s]
 13%|█▎        | 6400/50176 [01:06<07:46, 93.85it/s]
 13%|█▎        | 6656/50176 [01:08<07:20, 98.75it/s]
 14%|█▍        | 6912/50176 [01:11<07:27, 96.75it/s]
 14%|█▍        | 7168/50176 [01:14<07:21, 97.39it/s]
 15%|█▍        | 7424/50176 [01:16<07:12, 98.92it/s]
 15%|█▌        | 7680/50176 [01:19<07:00, 101.00it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.98it/s]
 16%|█▋        | 8192/50176 [01:23<06:46, 103.20it/s]
 17%|█▋        | 8448/50176 [01:26<06:43, 103.45it/s]
 17%|█▋        | 8704/50176 [01:28<06:39, 103.68it/s]
 18%|█▊        | 8960/50176 [01:31<06:50, 100.36it/s]
 18%|█▊        | 9216/50176 [01:34<06:47, 100.53it/s]
 19%|█▉        | 9472/50176 [01:36<06:40, 101.71it/s]
 19%|█▉        | 9728/50176 [01:39<06:43, 100.17it/s]
 20%|█▉        | 9984/50176 [01:41<06:36, 101.36it/s]
 20%|██        | 10240/50176 [01:44<06:26, 103.44it/s]
 21%|██        | 10496/50176 [01:46<06:19, 104.53it/s]
 21%|██▏       | 10752/50176 [01:49<06:25, 102.33it/s]
 22%|██▏       | 11008/50176 [01:51<06:25, 101.49it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.19it/s]
 23%|██▎       | 11520/50176 [01:56<06:16, 102.54it/s]
 23%|██▎       | 11776/50176 [01:58<06:04, 105.35it/s]
 24%|██▍       | 12032/50176 [02:01<06:05, 104.34it/s]
 24%|██▍       | 12288/50176 [02:03<05:59, 105.45it/s]
 25%|██▌       | 12544/50176 [02:06<06:05, 102.84it/s]
 26%|██▌       | 12800/50176 [02:08<06:07, 101.79it/s]
 26%|██▌       | 13056/50176 [02:11<05:57, 103.73it/s]
 27%|██▋       | 13312/50176 [02:13<05:51, 104.95it/s]
 27%|██▋       | 13568/50176 [02:16<06:10, 98.87it/s] 
 28%|██▊       | 13824/50176 [02:19<06:20, 95.62it/s]
 28%|██▊       | 14080/50176 [02:22<06:17, 95.68it/s]
 29%|██▊       | 14336/50176 [02:24<06:16, 95.24it/s]
 29%|██▉       | 14592/50176 [02:27<06:08, 96.50it/s]
 30%|██▉       | 14848/50176 [02:30<06:17, 93.69it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.54it/s]
 31%|███       | 15360/50176 [02:35<05:57, 97.35it/s]
 31%|███       | 15616/50176 [02:37<05:47, 99.48it/s]
 32%|███▏      | 15872/50176 [02:40<05:50, 97.82it/s]
 32%|███▏      | 16128/50176 [02:43<05:42, 99.32it/s]
 33%|███▎      | 16384/50176 [02:45<05:39, 99.67it/s]
 33%|███▎      | 16640/50176 [02:48<05:41, 98.08it/s]
 34%|███▎      | 16896/50176 [02:50<05:34, 99.47it/s]
 34%|███▍      | 17152/50176 [02:53<05:29, 100.37it/s]
 35%|███▍      | 17408/50176 [02:56<05:30, 99.13it/s] 
 35%|███▌      | 17664/50176 [02:58<05:20, 101.53it/s]
 36%|███▌      | 17920/50176 [03:00<05:12, 103.23it/s]
 36%|███▌      | 18176/50176 [03:03<05:10, 102.90it/s]
 37%|███▋      | 18432/50176 [03:05<05:03, 104.60it/s]
 37%|███▋      | 18688/50176 [03:08<05:11, 100.95it/s]
 38%|███▊      | 18944/50176 [03:10<05:11, 100.17it/s]
 38%|███▊      | 19200/50176 [03:13<05:19, 96.81it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.77it/s]
 39%|███▉      | 19712/50176 [03:19<05:22, 94.43it/s]
 40%|███▉      | 19968/50176 [03:22<05:18, 94.92it/s]
 40%|████      | 20224/50176 [03:24<05:00, 99.72it/s]
 41%|████      | 20480/50176 [03:26<04:54, 100.68it/s]
 41%|████▏     | 20736/50176 [03:29<04:50, 101.17it/s]
 42%|████▏     | 20992/50176 [03:31<04:37, 105.01it/s]
 42%|████▏     | 21248/50176 [03:34<04:41, 102.86it/s]
 43%|████▎     | 21504/50176 [03:36<04:36, 103.86it/s]
 43%|████▎     | 21760/50176 [03:38<04:32, 104.34it/s]
 44%|████▍     | 22016/50176 [03:41<04:24, 106.57it/s]
 44%|████▍     | 22272/50176 [03:43<04:21, 106.87it/s]
 45%|████▍     | 22528/50176 [03:46<04:18, 106.89it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.43it/s]
 46%|████▌     | 23040/50176 [03:51<04:24, 102.44it/s]
 46%|████▋     | 23296/50176 [03:53<04:30, 99.19it/s] 
 47%|████▋     | 23552/50176 [03:56<04:33, 97.46it/s]
 47%|████▋     | 23808/50176 [03:58<04:17, 102.53it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.76it/s]
 48%|████▊     | 24320/50176 [04:04<04:15, 101.09it/s]
 49%|████▉     | 24576/50176 [04:06<04:08, 103.01it/s]
 49%|████▉     | 24832/50176 [04:08<04:00, 105.52it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 105.92it/s]
 51%|█████     | 25344/50176 [04:13<03:54, 105.81it/s]
 51%|█████     | 25600/50176 [04:15<03:53, 105.26it/s]
 52%|█████▏    | 25856/50176 [04:18<03:50, 105.36it/s]
 52%|█████▏    | 26112/50176 [04:20<03:50, 104.59it/s]
 53%|█████▎    | 26368/50176 [04:23<03:47, 104.73it/s]
 53%|█████▎    | 26624/50176 [04:25<03:41, 106.48it/s]
 54%|█████▎    | 26880/50176 [04:28<03:45, 103.52it/s]
 54%|█████▍    | 27136/50176 [04:30<03:42, 103.62it/s]
 55%|█████▍    | 27392/50176 [04:33<03:36, 105.07it/s]
 55%|█████▌    | 27648/50176 [04:35<03:25, 109.86it/s]
 56%|█████▌    | 27904/50176 [04:37<03:34, 103.84it/s]
 56%|█████▌    | 28160/50176 [04:40<03:33, 102.93it/s]
 57%|█████▋    | 28416/50176 [04:43<03:32, 102.33it/s]
 57%|█████▋    | 28672/50176 [04:45<03:33, 100.78it/s]
 58%|█████▊    | 28928/50176 [04:48<03:28, 102.11it/s]
 58%|█████▊    | 29184/50176 [04:50<03:24, 102.60it/s]
 59%|█████▊    | 29440/50176 [04:53<03:36, 95.93it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:35, 94.91it/s]
 60%|█████▉    | 29952/50176 [04:59<03:34, 94.25it/s]
 60%|██████    | 30208/50176 [05:01<03:25, 97.08it/s]
 61%|██████    | 30464/50176 [05:03<03:13, 101.83it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.35it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.52it/s]
 62%|██████▏   | 31232/50176 [05:11<03:02, 103.76it/s]
 63%|██████▎   | 31488/50176 [05:13<02:56, 105.99it/s]
 63%|██████▎   | 31744/50176 [05:15<02:53, 106.16it/s]
 64%|██████▍   | 32000/50176 [05:18<02:49, 107.44it/s]
 64%|██████▍   | 32256/50176 [05:20<02:46, 107.60it/s]
 65%|██████▍   | 32512/50176 [05:23<02:55, 100.52it/s]
 65%|██████▌   | 32768/50176 [05:25<02:47, 103.68it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 102.08it/s]
 66%|██████▋   | 33280/50176 [05:30<02:41, 104.53it/s]
 67%|██████▋   | 33536/50176 [05:33<02:41, 103.17it/s]
 67%|██████▋   | 33792/50176 [05:35<02:38, 103.29it/s]
 68%|██████▊   | 34048/50176 [05:37<02:31, 106.21it/s]
 68%|██████▊   | 34304/50176 [05:40<02:24, 109.56it/s]
 69%|██████▉   | 34560/50176 [05:42<02:28, 105.31it/s]
 69%|██████▉   | 34816/50176 [05:45<02:25, 105.59it/s]
 70%|██████▉   | 35072/50176 [05:47<02:25, 103.84it/s]
 70%|███████   | 35328/50176 [05:50<02:23, 103.50it/s]
 71%|███████   | 35584/50176 [05:52<02:20, 103.91it/s]
 71%|███████▏  | 35840/50176 [05:55<02:21, 101.26it/s]
 72%|███████▏  | 36096/50176 [05:57<02:16, 103.46it/s]
 72%|███████▏  | 36352/50176 [06:00<02:16, 101.65it/s]
 73%|███████▎  | 36608/50176 [06:02<02:14, 100.82it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.47it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.55it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.31it/s]
 75%|███████▌  | 37632/50176 [06:12<01:57, 106.32it/s]
 76%|███████▌  | 37888/50176 [06:14<01:54, 107.33it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 106.67it/s]
 77%|███████▋  | 38400/50176 [06:19<01:53, 103.83it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.43it/s]
 78%|███████▊  | 38912/50176 [06:24<01:48, 103.95it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.71it/s]
 79%|███████▊  | 39424/50176 [06:30<01:45, 102.25it/s]
 79%|███████▉  | 39680/50176 [06:32<01:44, 100.72it/s]
 80%|███████▉  | 39936/50176 [06:35<01:39, 103.12it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.86it/s]
 81%|████████  | 40448/50176 [06:40<01:37, 99.55it/s] 
 81%|████████  | 40704/50176 [06:42<01:36, 97.94it/s]
 82%|████████▏ | 40960/50176 [06:45<01:29, 102.48it/s]
 82%|████████▏ | 41216/50176 [06:47<01:26, 103.88it/s]
 83%|████████▎ | 41472/50176 [06:50<01:24, 102.77it/s]
 83%|████████▎ | 41728/50176 [06:52<01:20, 105.08it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.41it/s]
 84%|████████▍ | 42240/50176 [06:57<01:16, 103.74it/s]
 85%|████████▍ | 42496/50176 [06:59<01:11, 106.92it/s]
 85%|████████▌ | 42752/50176 [07:02<01:11, 104.19it/s]
 86%|████████▌ | 43008/50176 [07:04<01:09, 103.65it/s]
 86%|████████▌ | 43264/50176 [07:07<01:07, 102.55it/s]
 87%|████████▋ | 43520/50176 [07:10<01:06, 100.75it/s]
 87%|████████▋ | 43776/50176 [07:12<01:03, 100.22it/s]
 88%|████████▊ | 44032/50176 [07:14<01:00, 102.36it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.83it/s]
 89%|████████▉ | 44544/50176 [07:19<00:55, 101.40it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 103.20it/s]
 90%|████████▉ | 45056/50176 [07:24<00:49, 104.30it/s]
 90%|█████████ | 45312/50176 [07:27<00:47, 102.22it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 105.37it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 104.05it/s]
 92%|█████████▏| 46080/50176 [07:34<00:40, 101.51it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.45it/s]
 93%|█████████▎| 46592/50176 [07:39<00:34, 104.45it/s]
 93%|█████████▎| 46848/50176 [07:42<00:32, 102.05it/s]
 94%|█████████▍| 47104/50176 [07:45<00:31, 98.41it/s] 
 94%|█████████▍| 47360/50176 [07:47<00:28, 97.77it/s]
 95%|█████████▍| 47616/50176 [07:50<00:26, 98.02it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 101.37it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 104.11it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 104.33it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 105.17it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 104.33it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.45it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.60it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.51it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 95.80it/s] 
-2022-12-21,10:44:14 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:44:14 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6177	imagenet-zeroshot-val-top5: 0.8741

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_42.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt).
-2022-12-21,10:44:38 | INFO | Model:
-2022-12-21,10:44:38 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:44:38 | INFO | Params:
-2022-12-21,10:44:38 | INFO |   batch_size: 256
-2022-12-21,10:44:38 | INFO |   beta1: 0.9
-2022-12-21,10:44:38 | INFO |   beta2: 0.98
-2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/checkpoints
-2022-12-21,10:44:38 | INFO |   copy_codebase: False
-2022-12-21,10:44:38 | INFO |   csv_caption_key: title
-2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
-2022-12-21,10:44:38 | INFO |   csv_separator: 	
-2022-12-21,10:44:38 | INFO |   dataset_resampled: False
-2022-12-21,10:44:38 | INFO |   dataset_type: auto
-2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
-2022-12-21,10:44:38 | INFO |   debug: False
-2022-12-21,10:44:38 | INFO |   device: cuda:0
-2022-12-21,10:44:38 | INFO |   dist_backend: nccl
-2022-12-21,10:44:38 | INFO |   dist_url: env://
-2022-12-21,10:44:38 | INFO |   distributed: False
-2022-12-21,10:44:38 | INFO |   epochs: 32
-2022-12-21,10:44:38 | INFO |   eps: 1e-06
-2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
-2022-12-21,10:44:38 | INFO |   gather_with_grad: False
-2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
-2022-12-21,10:44:38 | INFO |   horovod: False
-2022-12-21,10:44:38 | INFO |   image_mean: None
-2022-12-21,10:44:38 | INFO |   image_std: None
-2022-12-21,10:44:38 | INFO |   imagenet_v2: None
-2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:44:38 | INFO |   local_loss: False
-2022-12-21,10:44:38 | INFO |   local_rank: 0
-2022-12-21,10:44:38 | INFO |   lock_image: False
-2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:44:38 | INFO |   log_level: 20
-2022-12-21,10:44:38 | INFO |   log_local: False
-2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_42.pt/out.log
-2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:44:38 | INFO |   lr: 0.0005
-2022-12-21,10:44:38 | INFO |   model: ViT-B-16
-2022-12-21,10:44:38 | INFO |   name: epoch_42.pt
-2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
-2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
-2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
-2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_42.pt
-2022-12-21,10:44:38 | INFO |   pretrained_image: False
-2022-12-21,10:44:38 | INFO |   rank: 0
-2022-12-21,10:44:38 | INFO |   report_to: 
-2022-12-21,10:44:38 | INFO |   resume: None
-2022-12-21,10:44:38 | INFO |   save_frequency: 1
-2022-12-21,10:44:38 | INFO |   save_most_recent: False
-2022-12-21,10:44:38 | INFO |   seed: 0
-2022-12-21,10:44:38 | INFO |   skip_scheduler: False
-2022-12-21,10:44:38 | INFO |   tensorboard: False
-2022-12-21,10:44:38 | INFO |   tensorboard_path: 
-2022-12-21,10:44:38 | INFO |   torchscript: False
-2022-12-21,10:44:38 | INFO |   trace: False
-2022-12-21,10:44:38 | INFO |   train_data: None
-2022-12-21,10:44:38 | INFO |   train_num_samples: None
-2022-12-21,10:44:38 | INFO |   use_bn_sync: False
-2022-12-21,10:44:38 | INFO |   val_data: None
-2022-12-21,10:44:38 | INFO |   val_frequency: 1
-2022-12-21,10:44:38 | INFO |   val_num_samples: None
-2022-12-21,10:44:38 | INFO |   wandb: False
-2022-12-21,10:44:38 | INFO |   wandb_notes: 
-2022-12-21,10:44:38 | INFO |   warmup: 10000
-2022-12-21,10:44:38 | INFO |   wd: 0.2
-2022-12-21,10:44:38 | INFO |   workers: 1
-2022-12-21,10:44:38 | INFO |   world_size: 1
-2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:44:38 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:06,  1.50it/s]
  1%|          | 6/1000 [00:00<01:39, 10.03it/s]
  1%|          | 12/1000 [00:00<00:49, 19.80it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.29it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.35it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.15it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.35it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.43it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.05it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.95it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.32it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.34it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.86it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.38it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.90it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.23it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.63it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.84it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.09it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.23it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.18it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.17it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.08it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.04it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.13it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.15it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.02it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.91it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.90it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.85it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.81it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.74it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.56it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.46it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.56it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.55it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.74it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.69it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.61it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.67it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.78it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.89it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.93it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.77it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.81it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 58.01it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.11it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.31it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.27it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.23it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.36it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.32it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.24it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.05it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.11it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.18it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.28it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.90it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.92it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.05it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.18it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.08it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.79it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.88it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 58.00it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.16it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.24it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.24it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.25it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.27it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.38it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.38it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.17it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.23it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.28it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.26it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.21it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.36it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.28it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.25it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.25it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.17it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.21it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.13it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.98it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.05it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.17it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.13it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.25it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.27it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.16it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.17it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.24it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.13it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.07it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.23it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.13it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.24it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.15it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.29it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.29it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.38it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.19it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.13it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.27it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.23it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.32it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.37it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.30it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.34it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.36it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.23it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.31it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.30it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.29it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.21it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.16it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.26it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.24it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.32it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.40it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.24it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.34it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.30it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.32it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.36it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.17it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.23it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.19it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.19it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.23it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.32it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.42it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.46it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.39it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.43it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.39it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.39it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.33it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.38it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.36it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.24it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.32it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.33it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.30it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.32it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.28it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.26it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.37it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.41it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.42it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.43it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.44it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.39it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.48it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.54it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.56it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.48it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.19it/s]
-2022-12-21,10:44:56 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<09:57, 83.60it/s]
  1%|          | 512/50176 [00:05<09:13, 89.73it/s]
  2%|▏         | 768/50176 [00:08<08:28, 97.15it/s]
  2%|▏         | 1024/50176 [00:10<08:22, 97.90it/s]
  3%|▎         | 1280/50176 [00:13<08:18, 98.01it/s]
  3%|▎         | 1536/50176 [00:16<08:42, 93.17it/s]
  4%|▎         | 1792/50176 [00:19<08:35, 93.84it/s]
  4%|▍         | 2048/50176 [00:21<08:16, 97.03it/s]
  5%|▍         | 2304/50176 [00:23<08:03, 98.98it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.46it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.07it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.80it/s]
  7%|▋         | 3328/50176 [00:34<07:43, 100.99it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.01it/s]
  8%|▊         | 3840/50176 [00:39<07:41, 100.51it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.88it/s] 
  9%|▊         | 4352/50176 [00:44<07:52, 96.97it/s]
  9%|▉         | 4608/50176 [00:47<07:37, 99.66it/s]
 10%|▉         | 4864/50176 [00:49<07:33, 99.98it/s]
 10%|█         | 5120/50176 [00:52<07:23, 101.66it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.27it/s]
 11%|█         | 5632/50176 [00:57<07:19, 101.46it/s]
 12%|█▏        | 5888/50176 [00:59<07:18, 101.07it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.46it/s] 
 13%|█▎        | 6400/50176 [01:05<07:32, 96.64it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.82it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.67it/s] 
 14%|█▍        | 7168/50176 [01:12<07:14, 98.91it/s]
 15%|█▍        | 7424/50176 [01:15<07:04, 100.70it/s]
 15%|█▌        | 7680/50176 [01:17<06:54, 102.62it/s]
 16%|█▌        | 7936/50176 [01:20<06:43, 104.62it/s]
 16%|█▋        | 8192/50176 [01:22<06:41, 104.52it/s]
 17%|█▋        | 8448/50176 [01:24<06:37, 105.02it/s]
 17%|█▋        | 8704/50176 [01:27<06:38, 104.02it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.09it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 102.96it/s]
 19%|█▉        | 9472/50176 [01:34<06:30, 104.13it/s]
 19%|█▉        | 9728/50176 [01:37<06:36, 102.06it/s]
 20%|█▉        | 9984/50176 [01:40<06:44, 99.27it/s] 
 20%|██        | 10240/50176 [01:42<06:28, 102.87it/s]
 21%|██        | 10496/50176 [01:45<06:22, 103.87it/s]
 21%|██▏       | 10752/50176 [01:47<06:35, 99.74it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.75it/s]
 22%|██▏       | 11264/50176 [01:52<06:23, 101.40it/s]
 23%|██▎       | 11520/50176 [01:55<06:16, 102.57it/s]
 23%|██▎       | 11776/50176 [01:57<06:19, 101.13it/s]
 24%|██▍       | 12032/50176 [02:00<06:11, 102.56it/s]
 24%|██▍       | 12288/50176 [02:02<06:00, 104.99it/s]
 25%|██▌       | 12544/50176 [02:05<06:04, 103.20it/s]
 26%|██▌       | 12800/50176 [02:07<06:05, 102.14it/s]
 26%|██▌       | 13056/50176 [02:10<06:07, 100.87it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 102.02it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.29it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.84it/s]
 28%|██▊       | 14080/50176 [02:21<06:18, 95.33it/s]
 29%|██▊       | 14336/50176 [02:23<06:16, 95.25it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.81it/s]
 30%|██▉       | 14848/50176 [02:29<06:16, 93.72it/s]
 30%|███       | 15104/50176 [02:31<06:09, 94.81it/s]
 31%|███       | 15360/50176 [02:34<06:04, 95.51it/s]
 31%|███       | 15616/50176 [02:36<05:52, 97.93it/s]
 32%|███▏      | 15872/50176 [02:39<05:55, 96.56it/s]
 32%|███▏      | 16128/50176 [02:42<05:46, 98.39it/s]
 33%|███▎      | 16384/50176 [02:44<05:39, 99.39it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.88it/s]
 34%|███▎      | 16896/50176 [02:49<05:36, 98.77it/s]
 34%|███▍      | 17152/50176 [02:52<05:30, 99.93it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.29it/s]
 35%|███▌      | 17664/50176 [02:57<05:18, 101.98it/s]
 36%|███▌      | 17920/50176 [02:59<05:13, 102.89it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.62it/s]
 37%|███▋      | 18432/50176 [03:04<05:07, 103.36it/s]
 37%|███▋      | 18688/50176 [03:07<05:12, 100.92it/s]
 38%|███▊      | 18944/50176 [03:10<05:10, 100.64it/s]
 38%|███▊      | 19200/50176 [03:12<05:17, 97.68it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.60it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.99it/s]
 40%|███▉      | 19968/50176 [03:20<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:22<04:50, 103.22it/s]
 41%|████      | 20480/50176 [03:25<04:51, 101.99it/s]
 41%|████▏     | 20736/50176 [03:27<04:45, 102.95it/s]
 42%|████▏     | 20992/50176 [03:30<04:32, 106.94it/s]
 42%|████▏     | 21248/50176 [03:32<04:41, 102.78it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.62it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.23it/s]
 44%|████▍     | 22016/50176 [03:39<04:19, 108.62it/s]
 44%|████▍     | 22272/50176 [03:42<04:15, 109.14it/s]
 45%|████▍     | 22528/50176 [03:44<04:16, 107.59it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.11it/s]
 46%|████▌     | 23040/50176 [03:49<04:22, 103.53it/s]
 46%|████▋     | 23296/50176 [03:52<04:25, 101.38it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.15it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.91it/s]
 48%|████▊     | 24064/50176 [03:59<04:09, 104.79it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.97it/s]
 49%|████▉     | 24576/50176 [04:04<04:07, 103.40it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.73it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.12it/s]
 51%|█████     | 25344/50176 [04:11<03:49, 108.15it/s]
 51%|█████     | 25600/50176 [04:15<04:16, 95.70it/s] 
 52%|█████▏    | 25856/50176 [04:17<04:15, 95.22it/s]
 52%|█████▏    | 26112/50176 [04:20<04:08, 96.90it/s]
 53%|█████▎    | 26368/50176 [04:22<03:59, 99.25it/s]
 53%|█████▎    | 26624/50176 [04:25<03:53, 101.00it/s]
 54%|█████▎    | 26880/50176 [04:28<03:56, 98.43it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.60it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.13it/s]
 55%|█████▌    | 27648/50176 [04:34<03:27, 108.34it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.31it/s]
 56%|█████▌    | 28160/50176 [04:40<03:41, 99.51it/s] 
 57%|█████▋    | 28416/50176 [04:42<03:33, 101.81it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 101.43it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.19it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.38it/s]
 59%|█████▊    | 29440/50176 [04:53<03:31, 98.27it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.60it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.38it/s]
 60%|██████    | 30208/50176 [05:00<03:20, 99.47it/s]
 61%|██████    | 30464/50176 [05:03<03:07, 105.41it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.48it/s]
 62%|██████▏   | 30976/50176 [05:07<03:01, 105.99it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.03it/s]
 63%|██████▎   | 31488/50176 [05:12<02:50, 109.30it/s]
 63%|██████▎   | 31744/50176 [05:14<02:50, 107.99it/s]
 64%|██████▍   | 32000/50176 [05:17<02:46, 109.20it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.88it/s]
 65%|██████▍   | 32512/50176 [05:22<02:50, 103.70it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 107.05it/s]
 66%|██████▌   | 33024/50176 [05:26<02:39, 107.69it/s]
 66%|██████▋   | 33280/50176 [05:29<02:37, 107.29it/s]
 67%|██████▋   | 33536/50176 [05:31<02:42, 102.27it/s]
 67%|██████▋   | 33792/50176 [05:34<02:37, 103.84it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 106.00it/s]
 68%|██████▊   | 34304/50176 [05:38<02:26, 108.54it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.38it/s]
 69%|██████▉   | 34816/50176 [05:44<02:27, 104.09it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.79it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.09it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.78it/s]
 71%|███████▏  | 35840/50176 [05:53<02:18, 103.39it/s]
 72%|███████▏  | 36096/50176 [05:56<02:13, 105.10it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.46it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 104.56it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.72it/s]
 74%|███████▍  | 37120/50176 [06:06<02:06, 103.35it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 106.93it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.11it/s]
 76%|███████▌  | 37888/50176 [06:13<01:55, 106.71it/s]
 76%|███████▌  | 38144/50176 [06:15<01:52, 107.30it/s]
 77%|███████▋  | 38400/50176 [06:18<01:52, 104.55it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.51it/s]
 78%|███████▊  | 38912/50176 [06:22<01:46, 105.31it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.47it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.13it/s]
 79%|███████▉  | 39680/50176 [06:30<01:38, 106.81it/s]
 80%|███████▉  | 39936/50176 [06:32<01:33, 109.59it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.15it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 104.09it/s]
 81%|████████  | 40704/50176 [06:39<01:33, 101.79it/s]
 82%|████████▏ | 40960/50176 [06:42<01:26, 106.33it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 104.85it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.05it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.73it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.92it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.42it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.36it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 107.97it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.12it/s]
 86%|████████▌ | 43264/50176 [07:03<01:05, 106.02it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.30it/s]
 87%|████████▋ | 43776/50176 [07:09<01:02, 102.55it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 104.28it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.81it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.71it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.55it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 105.04it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.76it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 107.99it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.71it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.13it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.07it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.09it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.01it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 99.92it/s] 
 94%|█████████▍| 47360/50176 [07:43<00:28, 99.83it/s]
 95%|█████████▍| 47616/50176 [07:46<00:26, 98.46it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 101.00it/s]
 96%|█████████▌| 48128/50176 [07:51<00:20, 102.40it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.52it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.56it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.13it/s]
 98%|█████████▊| 49152/50176 [08:01<00:10, 101.95it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.34it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.40it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 96.05it/s] 
-2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6169	imagenet-zeroshot-val-top5: 0.8739

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_43.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt).
-2022-12-21,10:44:38 | INFO | Model:
-2022-12-21,10:44:38 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:44:38 | INFO | Params:
-2022-12-21,10:44:38 | INFO |   batch_size: 256
-2022-12-21,10:44:38 | INFO |   beta1: 0.9
-2022-12-21,10:44:38 | INFO |   beta2: 0.98
-2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/checkpoints
-2022-12-21,10:44:38 | INFO |   copy_codebase: False
-2022-12-21,10:44:38 | INFO |   csv_caption_key: title
-2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
-2022-12-21,10:44:38 | INFO |   csv_separator: 	
-2022-12-21,10:44:38 | INFO |   dataset_resampled: False
-2022-12-21,10:44:38 | INFO |   dataset_type: auto
-2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
-2022-12-21,10:44:38 | INFO |   debug: False
-2022-12-21,10:44:38 | INFO |   device: cuda:0
-2022-12-21,10:44:38 | INFO |   dist_backend: nccl
-2022-12-21,10:44:38 | INFO |   dist_url: env://
-2022-12-21,10:44:38 | INFO |   distributed: False
-2022-12-21,10:44:38 | INFO |   epochs: 32
-2022-12-21,10:44:38 | INFO |   eps: 1e-06
-2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
-2022-12-21,10:44:38 | INFO |   gather_with_grad: False
-2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
-2022-12-21,10:44:38 | INFO |   horovod: False
-2022-12-21,10:44:38 | INFO |   image_mean: None
-2022-12-21,10:44:38 | INFO |   image_std: None
-2022-12-21,10:44:38 | INFO |   imagenet_v2: None
-2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:44:38 | INFO |   local_loss: False
-2022-12-21,10:44:38 | INFO |   local_rank: 0
-2022-12-21,10:44:38 | INFO |   lock_image: False
-2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:44:38 | INFO |   log_level: 20
-2022-12-21,10:44:38 | INFO |   log_local: False
-2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_43.pt/out.log
-2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:44:38 | INFO |   lr: 0.0005
-2022-12-21,10:44:38 | INFO |   model: ViT-B-16
-2022-12-21,10:44:38 | INFO |   name: epoch_43.pt
-2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
-2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
-2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
-2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_43.pt
-2022-12-21,10:44:38 | INFO |   pretrained_image: False
-2022-12-21,10:44:38 | INFO |   rank: 0
-2022-12-21,10:44:38 | INFO |   report_to: 
-2022-12-21,10:44:38 | INFO |   resume: None
-2022-12-21,10:44:38 | INFO |   save_frequency: 1
-2022-12-21,10:44:38 | INFO |   save_most_recent: False
-2022-12-21,10:44:38 | INFO |   seed: 0
-2022-12-21,10:44:38 | INFO |   skip_scheduler: False
-2022-12-21,10:44:38 | INFO |   tensorboard: False
-2022-12-21,10:44:38 | INFO |   tensorboard_path: 
-2022-12-21,10:44:38 | INFO |   torchscript: False
-2022-12-21,10:44:38 | INFO |   trace: False
-2022-12-21,10:44:38 | INFO |   train_data: None
-2022-12-21,10:44:38 | INFO |   train_num_samples: None
-2022-12-21,10:44:38 | INFO |   use_bn_sync: False
-2022-12-21,10:44:38 | INFO |   val_data: None
-2022-12-21,10:44:38 | INFO |   val_frequency: 1
-2022-12-21,10:44:38 | INFO |   val_num_samples: None
-2022-12-21,10:44:38 | INFO |   wandb: False
-2022-12-21,10:44:38 | INFO |   wandb_notes: 
-2022-12-21,10:44:38 | INFO |   warmup: 10000
-2022-12-21,10:44:38 | INFO |   wd: 0.2
-2022-12-21,10:44:38 | INFO |   workers: 1
-2022-12-21,10:44:38 | INFO |   world_size: 1
-2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:44:38 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:10,  1.64it/s]
  1%|          | 6/1000 [00:00<01:32, 10.79it/s]
  1%|          | 12/1000 [00:00<00:48, 20.49it/s]
  2%|▏         | 18/1000 [00:00<00:33, 28.97it/s]
  2%|▏         | 24/1000 [00:01<00:27, 36.13it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.83it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.15it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.36it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.42it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.21it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.48it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.90it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.39it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.95it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.29it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.70it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.87it/s]
 11%|█         | 108/1000 [00:02<00:15, 58.12it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.25it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 58.16it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 58.17it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 58.11it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 58.14it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.21it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.23it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 58.11it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.99it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 58.00it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.97it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.81it/s]
 19%|█▉        | 192/1000 [00:03<00:13, 57.79it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.47it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.31it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.45it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.68it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.71it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.64it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.72it/s]
 25%|██▌       | 252/1000 [00:04<00:12, 57.82it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.88it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.94it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.78it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.84it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 58.03it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 58.14it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.35it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.33it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.30it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.43it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.42it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.32it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 58.14it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.21it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.32it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.41it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 58.17it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 58.17it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.25it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.32it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.20it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.88it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.95it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 58.03it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.12it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.23it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.25it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.25it/s]
 43%|████▎     | 426/1000 [00:07<00:09, 58.29it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.42it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.42it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.17it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.24it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.31it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.29it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.23it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.38it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.30it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.28it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.24it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.16it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.22it/s]
 51%|█████     | 510/1000 [00:09<00:08, 58.15it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.24it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.28it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.37it/s]
 53%|█████▎    | 534/1000 [00:09<00:07, 58.32it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.41it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.37it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.27it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.30it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.36it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.25it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 58.19it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.33it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.31it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.23it/s]
 60%|██████    | 600/1000 [00:10<00:06, 58.33it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.24it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.23it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.35it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.35it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.42it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.25it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 58.19it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.33it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.27it/s]
 66%|██████▌   | 660/1000 [00:11<00:05, 58.35it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.39it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.31it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.28it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.27it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.32it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.22it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.33it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.31it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.29it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.31it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.27it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.29it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.30it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.28it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.36it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.44it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.23it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.34it/s]
 77%|███████▋  | 774/1000 [00:13<00:03, 58.32it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.37it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.43it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.08it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.11it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.18it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.16it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.19it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.31it/s]
 83%|████████▎ | 834/1000 [00:14<00:02, 58.45it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.49it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.40it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.43it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.38it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.35it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.29it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.35it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.23it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.34it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.31it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.35it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.33it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.31it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.40it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.44it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.44it/s]
 95%|█████████▍| 948/1000 [00:16<00:00, 58.47it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.50it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.43it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.51it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.58it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.61it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.59it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.08it/s]
-2022-12-21,10:44:56 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:42, 85.63it/s]
  1%|          | 512/50176 [00:05<09:07, 90.79it/s]
  2%|▏         | 768/50176 [00:08<08:26, 97.61it/s]
  2%|▏         | 1024/50176 [00:10<08:21, 98.05it/s]
  3%|▎         | 1280/50176 [00:13<08:18, 98.18it/s]
  3%|▎         | 1536/50176 [00:16<08:41, 93.21it/s]
  4%|▎         | 1792/50176 [00:18<08:34, 93.95it/s]
  4%|▍         | 2048/50176 [00:21<08:15, 97.14it/s]
  5%|▍         | 2304/50176 [00:23<08:04, 98.81it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.63it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.24it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.81it/s]
  7%|▋         | 3328/50176 [00:34<07:43, 101.00it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.06it/s]
  8%|▊         | 3840/50176 [00:39<07:41, 100.49it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.91it/s] 
  9%|▊         | 4352/50176 [00:44<07:52, 96.98it/s]
  9%|▉         | 4608/50176 [00:47<07:38, 99.47it/s]
 10%|▉         | 4864/50176 [00:49<07:33, 100.00it/s]
 10%|█         | 5120/50176 [00:52<07:22, 101.76it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.30it/s]
 11%|█         | 5632/50176 [00:57<07:18, 101.51it/s]
 12%|█▏        | 5888/50176 [00:59<07:17, 101.30it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.50it/s] 
 13%|█▎        | 6400/50176 [01:05<07:33, 96.63it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.79it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.54it/s] 
 14%|█▍        | 7168/50176 [01:12<07:14, 98.94it/s]
 15%|█▍        | 7424/50176 [01:15<07:04, 100.65it/s]
 15%|█▌        | 7680/50176 [01:17<06:54, 102.56it/s]
 16%|█▌        | 7936/50176 [01:19<06:43, 104.69it/s]
 16%|█▋        | 8192/50176 [01:22<06:41, 104.61it/s]
 17%|█▋        | 8448/50176 [01:24<06:37, 105.03it/s]
 17%|█▋        | 8704/50176 [01:27<06:38, 104.13it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.15it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 102.94it/s]
 19%|█▉        | 9472/50176 [01:34<06:31, 104.03it/s]
 19%|█▉        | 9728/50176 [01:37<06:36, 102.09it/s]
 20%|█▉        | 9984/50176 [01:40<06:45, 99.17it/s] 
 20%|██        | 10240/50176 [01:42<06:28, 102.86it/s]
 21%|██        | 10496/50176 [01:44<06:22, 103.70it/s]
 21%|██▏       | 10752/50176 [01:47<06:35, 99.77it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.95it/s]
 22%|██▏       | 11264/50176 [01:52<06:23, 101.42it/s]
 23%|██▎       | 11520/50176 [01:55<06:17, 102.52it/s]
 23%|██▎       | 11776/50176 [01:57<06:19, 101.12it/s]
 24%|██▍       | 12032/50176 [02:00<06:12, 102.53it/s]
 24%|██▍       | 12288/50176 [02:02<06:01, 104.95it/s]
 25%|██▌       | 12544/50176 [02:05<06:04, 103.27it/s]
 26%|██▌       | 12800/50176 [02:07<06:06, 102.05it/s]
 26%|██▌       | 13056/50176 [02:10<06:07, 100.92it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 102.02it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.35it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.81it/s]
 28%|██▊       | 14080/50176 [02:21<06:18, 95.34it/s]
 29%|██▊       | 14336/50176 [02:23<06:16, 95.25it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.81it/s]
 30%|██▉       | 14848/50176 [02:29<06:16, 93.74it/s]
 30%|███       | 15104/50176 [02:31<06:09, 94.86it/s]
 31%|███       | 15360/50176 [02:34<06:04, 95.50it/s]
 31%|███       | 15616/50176 [02:36<05:53, 97.85it/s]
 32%|███▏      | 15872/50176 [02:39<05:55, 96.58it/s]
 32%|███▏      | 16128/50176 [02:42<05:46, 98.34it/s]
 33%|███▎      | 16384/50176 [02:44<05:40, 99.33it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.85it/s]
 34%|███▎      | 16896/50176 [02:49<05:36, 98.85it/s]
 34%|███▍      | 17152/50176 [02:52<05:30, 99.86it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.25it/s]
 35%|███▌      | 17664/50176 [02:57<05:18, 102.01it/s]
 36%|███▌      | 17920/50176 [02:59<05:13, 102.94it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.63it/s]
 37%|███▋      | 18432/50176 [03:04<05:07, 103.37it/s]
 37%|███▋      | 18688/50176 [03:07<05:11, 100.98it/s]
 38%|███▊      | 18944/50176 [03:09<05:10, 100.65it/s]
 38%|███▊      | 19200/50176 [03:12<05:17, 97.61it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.62it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.99it/s]
 40%|███▉      | 19968/50176 [03:20<05:09, 97.47it/s]
 40%|████      | 20224/50176 [03:22<04:50, 103.20it/s]
 41%|████      | 20480/50176 [03:25<04:51, 101.95it/s]
 41%|████▏     | 20736/50176 [03:27<04:46, 102.91it/s]
 42%|████▏     | 20992/50176 [03:30<04:32, 106.93it/s]
 42%|████▏     | 21248/50176 [03:32<04:41, 102.73it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.57it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.16it/s]
 44%|████▍     | 22016/50176 [03:39<04:19, 108.63it/s]
 44%|████▍     | 22272/50176 [03:42<04:15, 109.20it/s]
 45%|████▍     | 22528/50176 [03:44<04:16, 107.59it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.04it/s]
 46%|████▌     | 23040/50176 [03:49<04:22, 103.52it/s]
 46%|████▋     | 23296/50176 [03:52<04:24, 101.46it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.11it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.89it/s]
 48%|████▊     | 24064/50176 [03:59<04:09, 104.83it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.96it/s]
 49%|████▉     | 24576/50176 [04:04<04:07, 103.43it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.75it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.22it/s]
 51%|█████     | 25344/50176 [04:11<03:49, 108.08it/s]
 51%|█████     | 25600/50176 [04:15<04:16, 95.66it/s] 
 52%|█████▏    | 25856/50176 [04:17<04:15, 95.21it/s]
 52%|█████▏    | 26112/50176 [04:20<04:07, 97.09it/s]
 53%|█████▎    | 26368/50176 [04:22<03:59, 99.35it/s]
 53%|█████▎    | 26624/50176 [04:25<03:53, 100.93it/s]
 54%|█████▎    | 26880/50176 [04:28<03:56, 98.38it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.60it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.13it/s]
 55%|█████▌    | 27648/50176 [04:34<03:27, 108.34it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.44it/s]
 56%|█████▌    | 28160/50176 [04:40<03:41, 99.40it/s] 
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.64it/s]
 57%|█████▋    | 28672/50176 [04:45<03:31, 101.50it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.24it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.39it/s]
 59%|█████▊    | 29440/50176 [04:53<03:31, 98.23it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.67it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.35it/s]
 60%|██████    | 30208/50176 [05:00<03:20, 99.51it/s]
 61%|██████    | 30464/50176 [05:02<03:07, 105.38it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.41it/s]
 62%|██████▏   | 30976/50176 [05:07<03:01, 106.02it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.14it/s]
 63%|██████▎   | 31488/50176 [05:12<02:51, 109.26it/s]
 63%|██████▎   | 31744/50176 [05:14<02:50, 107.96it/s]
 64%|██████▍   | 32000/50176 [05:17<02:46, 109.17it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.76it/s]
 65%|██████▍   | 32512/50176 [05:22<02:50, 103.73it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 107.05it/s]
 66%|██████▌   | 33024/50176 [05:26<02:39, 107.47it/s]
 66%|██████▋   | 33280/50176 [05:29<02:37, 107.03it/s]
 67%|██████▋   | 33536/50176 [05:31<02:42, 102.17it/s]
 67%|██████▋   | 33792/50176 [05:34<02:38, 103.67it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 106.01it/s]
 68%|██████▊   | 34304/50176 [05:38<02:26, 108.41it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.29it/s]
 69%|██████▉   | 34816/50176 [05:43<02:27, 104.04it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.77it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.21it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.72it/s]
 71%|███████▏  | 35840/50176 [05:53<02:18, 103.22it/s]
 72%|███████▏  | 36096/50176 [05:56<02:14, 105.07it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.34it/s]
 73%|███████▎  | 36608/50176 [06:00<02:09, 104.71it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.78it/s]
 74%|███████▍  | 37120/50176 [06:06<02:06, 103.45it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 106.92it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.12it/s]
 76%|███████▌  | 37888/50176 [06:13<01:55, 106.73it/s]
 76%|███████▌  | 38144/50176 [06:15<01:52, 107.22it/s]
 77%|███████▋  | 38400/50176 [06:17<01:52, 104.63it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.31it/s]
 78%|███████▊  | 38912/50176 [06:22<01:46, 105.28it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.50it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.28it/s]
 79%|███████▉  | 39680/50176 [06:29<01:38, 106.96it/s]
 80%|███████▉  | 39936/50176 [06:32<01:33, 109.55it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.04it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 104.22it/s]
 81%|████████  | 40704/50176 [06:39<01:33, 101.68it/s]
 82%|████████▏ | 40960/50176 [06:42<01:26, 106.26it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 104.85it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 105.01it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.78it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.89it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.25it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.38it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 108.13it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.19it/s]
 86%|████████▌ | 43264/50176 [07:03<01:05, 106.05it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.28it/s]
 87%|████████▋ | 43776/50176 [07:08<01:02, 102.70it/s]
 88%|████████▊ | 44032/50176 [07:11<00:59, 103.94it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.87it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.59it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.67it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 105.13it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.87it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 108.09it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.73it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.07it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.14it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.31it/s]
 93%|█████████▎| 46848/50176 [07:38<00:32, 103.74it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 99.77it/s] 
 94%|█████████▍| 47360/50176 [07:43<00:28, 99.51it/s]
 95%|█████████▍| 47616/50176 [07:46<00:26, 98.42it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 101.11it/s]
 96%|█████████▌| 48128/50176 [07:51<00:20, 102.38it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.39it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.41it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.17it/s]
 98%|█████████▊| 49152/50176 [08:01<00:10, 101.93it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.53it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.22it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 96.17it/s] 
-2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6219	imagenet-zeroshot-val-top5: 0.8762

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_44.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt).
-2022-12-21,10:44:37 | INFO | Model:
-2022-12-21,10:44:37 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:44:37 | INFO | Params:
-2022-12-21,10:44:37 | INFO |   batch_size: 256
-2022-12-21,10:44:37 | INFO |   beta1: 0.9
-2022-12-21,10:44:37 | INFO |   beta2: 0.98
-2022-12-21,10:44:37 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/checkpoints
-2022-12-21,10:44:37 | INFO |   copy_codebase: False
-2022-12-21,10:44:37 | INFO |   csv_caption_key: title
-2022-12-21,10:44:37 | INFO |   csv_img_key: filepath
-2022-12-21,10:44:37 | INFO |   csv_separator: 	
-2022-12-21,10:44:37 | INFO |   dataset_resampled: False
-2022-12-21,10:44:37 | INFO |   dataset_type: auto
-2022-12-21,10:44:37 | INFO |   ddp_static_graph: False
-2022-12-21,10:44:37 | INFO |   debug: False
-2022-12-21,10:44:37 | INFO |   device: cuda:0
-2022-12-21,10:44:37 | INFO |   dist_backend: nccl
-2022-12-21,10:44:37 | INFO |   dist_url: env://
-2022-12-21,10:44:37 | INFO |   distributed: False
-2022-12-21,10:44:37 | INFO |   epochs: 32
-2022-12-21,10:44:37 | INFO |   eps: 1e-06
-2022-12-21,10:44:37 | INFO |   force_quick_gelu: False
-2022-12-21,10:44:37 | INFO |   gather_with_grad: False
-2022-12-21,10:44:37 | INFO |   grad_checkpointing: False
-2022-12-21,10:44:37 | INFO |   horovod: False
-2022-12-21,10:44:37 | INFO |   image_mean: None
-2022-12-21,10:44:37 | INFO |   image_std: None
-2022-12-21,10:44:37 | INFO |   imagenet_v2: None
-2022-12-21,10:44:37 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:44:37 | INFO |   local_loss: False
-2022-12-21,10:44:37 | INFO |   local_rank: 0
-2022-12-21,10:44:37 | INFO |   lock_image: False
-2022-12-21,10:44:37 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:44:37 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:44:37 | INFO |   log_level: 20
-2022-12-21,10:44:37 | INFO |   log_local: False
-2022-12-21,10:44:37 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_44.pt/out.log
-2022-12-21,10:44:37 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:44:37 | INFO |   lr: 0.0005
-2022-12-21,10:44:37 | INFO |   model: ViT-B-16
-2022-12-21,10:44:37 | INFO |   name: epoch_44.pt
-2022-12-21,10:44:37 | INFO |   no_set_device_rank: False
-2022-12-21,10:44:37 | INFO |   norm_gradient_clip: None
-2022-12-21,10:44:37 | INFO |   precision: amp_bfloat16
-2022-12-21,10:44:37 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_44.pt
-2022-12-21,10:44:37 | INFO |   pretrained_image: False
-2022-12-21,10:44:37 | INFO |   rank: 0
-2022-12-21,10:44:37 | INFO |   report_to: 
-2022-12-21,10:44:37 | INFO |   resume: None
-2022-12-21,10:44:37 | INFO |   save_frequency: 1
-2022-12-21,10:44:37 | INFO |   save_most_recent: False
-2022-12-21,10:44:37 | INFO |   seed: 0
-2022-12-21,10:44:37 | INFO |   skip_scheduler: False
-2022-12-21,10:44:37 | INFO |   tensorboard: False
-2022-12-21,10:44:37 | INFO |   tensorboard_path: 
-2022-12-21,10:44:37 | INFO |   torchscript: False
-2022-12-21,10:44:37 | INFO |   trace: False
-2022-12-21,10:44:37 | INFO |   train_data: None
-2022-12-21,10:44:37 | INFO |   train_num_samples: None
-2022-12-21,10:44:37 | INFO |   use_bn_sync: False
-2022-12-21,10:44:37 | INFO |   val_data: None
-2022-12-21,10:44:37 | INFO |   val_frequency: 1
-2022-12-21,10:44:37 | INFO |   val_num_samples: None
-2022-12-21,10:44:37 | INFO |   wandb: False
-2022-12-21,10:44:37 | INFO |   wandb_notes: 
-2022-12-21,10:44:37 | INFO |   warmup: 10000
-2022-12-21,10:44:37 | INFO |   wd: 0.2
-2022-12-21,10:44:37 | INFO |   workers: 1
-2022-12-21,10:44:37 | INFO |   world_size: 1
-2022-12-21,10:44:37 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:44:38 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:07,  1.50it/s]
  1%|          | 6/1000 [00:00<01:40,  9.90it/s]
  1%|          | 11/1000 [00:00<00:55, 17.87it/s]
  2%|▏         | 16/1000 [00:00<00:39, 24.88it/s]
  2%|▏         | 22/1000 [00:01<00:29, 33.01it/s]
  3%|▎         | 28/1000 [00:01<00:24, 39.51it/s]
  3%|▎         | 34/1000 [00:01<00:21, 44.47it/s]
  4%|▍         | 40/1000 [00:01<00:19, 48.23it/s]
  5%|▍         | 46/1000 [00:01<00:18, 50.85it/s]
  5%|▌         | 52/1000 [00:01<00:17, 52.82it/s]
  6%|▌         | 58/1000 [00:01<00:17, 54.15it/s]
  6%|▋         | 64/1000 [00:01<00:16, 55.21it/s]
  7%|▋         | 70/1000 [00:01<00:16, 55.93it/s]
  8%|▊         | 76/1000 [00:02<00:16, 56.49it/s]
  8%|▊         | 82/1000 [00:02<00:16, 57.07it/s]
  9%|▉         | 88/1000 [00:02<00:15, 57.39it/s]
  9%|▉         | 94/1000 [00:02<00:15, 57.61it/s]
 10%|█         | 100/1000 [00:02<00:15, 57.83it/s]
 11%|█         | 106/1000 [00:02<00:15, 58.08it/s]
 11%|█         | 112/1000 [00:02<00:15, 58.18it/s]
 12%|█▏        | 118/1000 [00:02<00:15, 58.19it/s]
 12%|█▏        | 124/1000 [00:02<00:15, 58.06it/s]
 13%|█▎        | 130/1000 [00:02<00:14, 58.16it/s]
 14%|█▎        | 136/1000 [00:03<00:14, 58.16it/s]
 14%|█▍        | 142/1000 [00:03<00:14, 58.06it/s]
 15%|█▍        | 148/1000 [00:03<00:14, 58.15it/s]
 15%|█▌        | 154/1000 [00:03<00:14, 58.22it/s]
 16%|█▌        | 160/1000 [00:03<00:14, 58.10it/s]
 17%|█▋        | 166/1000 [00:03<00:14, 58.04it/s]
 17%|█▋        | 172/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 178/1000 [00:03<00:14, 57.99it/s]
 18%|█▊        | 184/1000 [00:03<00:14, 57.84it/s]
 19%|█▉        | 190/1000 [00:03<00:14, 57.68it/s]
 20%|█▉        | 196/1000 [00:04<00:13, 57.68it/s]
 20%|██        | 202/1000 [00:04<00:13, 57.65it/s]
 21%|██        | 208/1000 [00:04<00:13, 57.38it/s]
 21%|██▏       | 214/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 220/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 226/1000 [00:04<00:13, 57.63it/s]
 23%|██▎       | 232/1000 [00:04<00:13, 57.76it/s]
 24%|██▍       | 238/1000 [00:04<00:13, 57.70it/s]
 24%|██▍       | 244/1000 [00:04<00:13, 57.60it/s]
 25%|██▌       | 250/1000 [00:05<00:12, 57.70it/s]
 26%|██▌       | 256/1000 [00:05<00:12, 57.92it/s]
 26%|██▌       | 262/1000 [00:05<00:12, 57.96it/s]
 27%|██▋       | 268/1000 [00:05<00:12, 57.87it/s]
 27%|██▋       | 274/1000 [00:05<00:13, 52.08it/s]
 28%|██▊       | 280/1000 [00:05<00:13, 53.83it/s]
 29%|██▊       | 286/1000 [00:05<00:12, 55.06it/s]
 29%|██▉       | 292/1000 [00:05<00:12, 56.05it/s]
 30%|██▉       | 298/1000 [00:05<00:12, 56.71it/s]
 30%|███       | 304/1000 [00:05<00:12, 57.19it/s]
 31%|███       | 310/1000 [00:06<00:11, 57.60it/s]
 32%|███▏      | 316/1000 [00:06<00:11, 57.84it/s]
 32%|███▏      | 322/1000 [00:06<00:11, 58.01it/s]
 33%|███▎      | 328/1000 [00:06<00:11, 57.89it/s]
 33%|███▎      | 334/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 340/1000 [00:06<00:11, 58.09it/s]
 35%|███▍      | 346/1000 [00:06<00:11, 58.28it/s]
 35%|███▌      | 352/1000 [00:06<00:11, 58.15it/s]
 36%|███▌      | 358/1000 [00:06<00:11, 58.24it/s]
 36%|███▋      | 364/1000 [00:07<00:10, 58.25it/s]
 37%|███▋      | 370/1000 [00:07<00:10, 58.26it/s]
 38%|███▊      | 376/1000 [00:07<00:10, 58.19it/s]
 38%|███▊      | 382/1000 [00:07<00:10, 57.99it/s]
 39%|███▉      | 388/1000 [00:07<00:10, 57.90it/s]
 39%|███▉      | 394/1000 [00:07<00:10, 58.00it/s]
 40%|████      | 400/1000 [00:07<00:10, 58.19it/s]
 41%|████      | 406/1000 [00:07<00:10, 58.22it/s]
 41%|████      | 412/1000 [00:07<00:10, 58.11it/s]
 42%|████▏     | 418/1000 [00:07<00:10, 58.20it/s]
 42%|████▏     | 424/1000 [00:08<00:09, 58.14it/s]
 43%|████▎     | 430/1000 [00:08<00:09, 58.30it/s]
 44%|████▎     | 436/1000 [00:08<00:09, 58.36it/s]
 44%|████▍     | 442/1000 [00:08<00:09, 58.15it/s]
 45%|████▍     | 448/1000 [00:08<00:09, 58.19it/s]
 45%|████▌     | 454/1000 [00:08<00:09, 58.27it/s]
 46%|████▌     | 460/1000 [00:08<00:09, 58.26it/s]
 47%|████▋     | 466/1000 [00:08<00:09, 58.35it/s]
 47%|████▋     | 472/1000 [00:08<00:09, 58.32it/s]
 48%|████▊     | 478/1000 [00:08<00:08, 58.37it/s]
 48%|████▊     | 484/1000 [00:09<00:08, 58.21it/s]
 49%|████▉     | 490/1000 [00:09<00:08, 58.21it/s]
 50%|████▉     | 496/1000 [00:09<00:08, 58.16it/s]
 50%|█████     | 502/1000 [00:09<00:08, 58.22it/s]
 51%|█████     | 508/1000 [00:09<00:08, 58.15it/s]
 51%|█████▏    | 514/1000 [00:09<00:08, 58.20it/s]
 52%|█████▏    | 520/1000 [00:09<00:08, 58.27it/s]
 53%|█████▎    | 526/1000 [00:09<00:08, 58.32it/s]
 53%|█████▎    | 532/1000 [00:09<00:08, 58.30it/s]
 54%|█████▍    | 538/1000 [00:10<00:07, 58.35it/s]
 54%|█████▍    | 544/1000 [00:10<00:07, 58.43it/s]
 55%|█████▌    | 550/1000 [00:10<00:07, 58.34it/s]
 56%|█████▌    | 556/1000 [00:10<00:07, 58.08it/s]
 56%|█████▌    | 562/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 568/1000 [00:10<00:07, 58.11it/s]
 57%|█████▋    | 574/1000 [00:10<00:07, 58.06it/s]
 58%|█████▊    | 580/1000 [00:10<00:07, 58.22it/s]
 59%|█████▊    | 586/1000 [00:10<00:07, 58.22it/s]
 59%|█████▉    | 592/1000 [00:10<00:07, 58.16it/s]
 60%|█████▉    | 598/1000 [00:11<00:06, 58.21it/s]
 60%|██████    | 604/1000 [00:11<00:06, 58.12it/s]
 61%|██████    | 610/1000 [00:11<00:06, 58.26it/s]
 62%|██████▏   | 616/1000 [00:11<00:06, 58.26it/s]
 62%|██████▏   | 622/1000 [00:11<00:06, 58.33it/s]
 63%|██████▎   | 628/1000 [00:11<00:06, 58.39it/s]
 63%|██████▎   | 634/1000 [00:11<00:06, 58.24it/s]
 64%|██████▍   | 640/1000 [00:11<00:06, 58.17it/s]
 65%|██████▍   | 646/1000 [00:11<00:06, 58.31it/s]
 65%|██████▌   | 652/1000 [00:11<00:05, 58.31it/s]
 66%|██████▌   | 658/1000 [00:12<00:05, 58.35it/s]
 66%|██████▋   | 664/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 670/1000 [00:12<00:05, 58.31it/s]
 68%|██████▊   | 676/1000 [00:12<00:05, 58.37it/s]
 68%|██████▊   | 682/1000 [00:12<00:05, 58.34it/s]
 69%|██████▉   | 688/1000 [00:12<00:05, 58.40it/s]
 69%|██████▉   | 694/1000 [00:12<00:05, 58.26it/s]
 70%|███████   | 700/1000 [00:12<00:05, 58.37it/s]
 71%|███████   | 706/1000 [00:12<00:05, 58.30it/s]
 71%|███████   | 712/1000 [00:12<00:04, 58.36it/s]
 72%|███████▏  | 718/1000 [00:13<00:04, 58.34it/s]
 72%|███████▏  | 724/1000 [00:13<00:04, 58.30it/s]
 73%|███████▎  | 730/1000 [00:13<00:04, 58.27it/s]
 74%|███████▎  | 736/1000 [00:13<00:04, 58.33it/s]
 74%|███████▍  | 742/1000 [00:13<00:04, 58.26it/s]
 75%|███████▍  | 748/1000 [00:13<00:04, 58.16it/s]
 75%|███████▌  | 754/1000 [00:13<00:04, 58.34it/s]
 76%|███████▌  | 760/1000 [00:13<00:04, 58.22it/s]
 77%|███████▋  | 766/1000 [00:13<00:04, 58.30it/s]
 77%|███████▋  | 772/1000 [00:14<00:03, 58.30it/s]
 78%|███████▊  | 778/1000 [00:14<00:03, 58.37it/s]
 78%|███████▊  | 784/1000 [00:14<00:03, 58.37it/s]
 79%|███████▉  | 790/1000 [00:14<00:03, 58.21it/s]
 80%|███████▉  | 796/1000 [00:14<00:03, 58.27it/s]
 80%|████████  | 802/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 808/1000 [00:14<00:03, 58.21it/s]
 81%|████████▏ | 814/1000 [00:14<00:03, 58.20it/s]
 82%|████████▏ | 820/1000 [00:14<00:03, 58.30it/s]
 83%|████████▎ | 826/1000 [00:14<00:02, 58.26it/s]
 83%|████████▎ | 832/1000 [00:15<00:02, 58.41it/s]
 84%|████████▍ | 838/1000 [00:15<00:02, 58.51it/s]
 84%|████████▍ | 844/1000 [00:15<00:02, 58.40it/s]
 85%|████████▌ | 850/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 856/1000 [00:15<00:02, 58.39it/s]
 86%|████████▌ | 862/1000 [00:15<00:02, 58.37it/s]
 87%|████████▋ | 868/1000 [00:15<00:02, 58.31it/s]
 87%|████████▋ | 874/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 880/1000 [00:15<00:02, 58.36it/s]
 89%|████████▊ | 886/1000 [00:15<00:01, 58.23it/s]
 89%|████████▉ | 892/1000 [00:16<00:01, 58.33it/s]
 90%|████████▉ | 898/1000 [00:16<00:01, 58.38it/s]
 90%|█████████ | 904/1000 [00:16<00:01, 58.35it/s]
 91%|█████████ | 910/1000 [00:16<00:01, 58.32it/s]
 92%|█████████▏| 916/1000 [00:16<00:01, 58.30it/s]
 92%|█████████▏| 922/1000 [00:16<00:01, 58.25it/s]
 93%|█████████▎| 928/1000 [00:16<00:01, 58.41it/s]
 93%|█████████▎| 934/1000 [00:16<00:01, 58.48it/s]
 94%|█████████▍| 940/1000 [00:16<00:01, 58.50it/s]
 95%|█████████▍| 946/1000 [00:17<00:00, 58.38it/s]
 95%|█████████▌| 952/1000 [00:17<00:00, 58.42it/s]
 96%|█████████▌| 958/1000 [00:17<00:00, 58.37it/s]
 96%|█████████▋| 964/1000 [00:17<00:00, 58.46it/s]
 97%|█████████▋| 970/1000 [00:17<00:00, 58.48it/s]
 98%|█████████▊| 976/1000 [00:17<00:00, 58.51it/s]
 98%|█████████▊| 982/1000 [00:17<00:00, 58.54it/s]
 99%|█████████▉| 988/1000 [00:17<00:00, 58.46it/s]
 99%|█████████▉| 994/1000 [00:17<00:00, 58.41it/s]
-2022-12-21,10:44:55 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:36, 78.39it/s]
  1%|          | 512/50176 [00:06<09:33, 86.55it/s]
  2%|▏         | 768/50176 [00:08<08:38, 95.22it/s]
  2%|▏         | 1024/50176 [00:10<08:27, 96.82it/s]
  3%|▎         | 1280/50176 [00:13<08:23, 97.18it/s]
  3%|▎         | 1536/50176 [00:16<08:45, 92.61it/s]
  4%|▎         | 1792/50176 [00:19<08:37, 93.42it/s]
  4%|▍         | 2048/50176 [00:21<08:17, 96.80it/s]
  5%|▍         | 2304/50176 [00:24<08:05, 98.69it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.59it/s]
  6%|▌         | 2816/50176 [00:29<08:03, 97.92it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.78it/s]
  7%|▋         | 3328/50176 [00:34<07:43, 101.05it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.03it/s]
  8%|▊         | 3840/50176 [00:39<07:41, 100.37it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.86it/s] 
  9%|▊         | 4352/50176 [00:44<07:52, 96.98it/s]
  9%|▉         | 4608/50176 [00:47<07:38, 99.46it/s]
 10%|▉         | 4864/50176 [00:49<07:32, 100.03it/s]
 10%|█         | 5120/50176 [00:52<07:23, 101.68it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.30it/s]
 11%|█         | 5632/50176 [00:57<07:19, 101.44it/s]
 12%|█▏        | 5888/50176 [00:59<07:17, 101.27it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.47it/s] 
 13%|█▎        | 6400/50176 [01:05<07:33, 96.63it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.94it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.53it/s] 
 14%|█▍        | 7168/50176 [01:13<07:14, 98.88it/s]
 15%|█▍        | 7424/50176 [01:15<07:04, 100.75it/s]
 15%|█▌        | 7680/50176 [01:17<06:53, 102.88it/s]
 16%|█▌        | 7936/50176 [01:20<06:44, 104.50it/s]
 16%|█▋        | 8192/50176 [01:22<06:40, 104.92it/s]
 17%|█▋        | 8448/50176 [01:25<06:37, 104.88it/s]
 17%|█▋        | 8704/50176 [01:27<06:37, 104.24it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.11it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 102.92it/s]
 19%|█▉        | 9472/50176 [01:35<06:31, 103.97it/s]
 19%|█▉        | 9728/50176 [01:37<06:37, 101.81it/s]
 20%|█▉        | 9984/50176 [01:40<06:44, 99.48it/s] 
 20%|██        | 10240/50176 [01:42<06:29, 102.56it/s]
 21%|██        | 10496/50176 [01:45<06:21, 103.91it/s]
 21%|██▏       | 10752/50176 [01:48<06:34, 99.84it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.72it/s]
 22%|██▏       | 11264/50176 [01:53<06:23, 101.45it/s]
 23%|██▎       | 11520/50176 [01:55<06:16, 102.55it/s]
 23%|██▎       | 11776/50176 [01:58<06:19, 101.10it/s]
 24%|██▍       | 12032/50176 [02:00<06:12, 102.51it/s]
 24%|██▍       | 12288/50176 [02:02<06:01, 104.89it/s]
 25%|██▌       | 12544/50176 [02:05<06:03, 103.40it/s]
 26%|██▌       | 12800/50176 [02:07<06:05, 102.34it/s]
 26%|██▌       | 13056/50176 [02:10<06:08, 100.85it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 102.05it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.34it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.89it/s]
 28%|██▊       | 14080/50176 [02:21<06:18, 95.31it/s]
 29%|██▊       | 14336/50176 [02:24<06:16, 95.26it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.87it/s]
 30%|██▉       | 14848/50176 [02:29<06:17, 93.60it/s]
 30%|███       | 15104/50176 [02:32<06:10, 94.63it/s]
 31%|███       | 15360/50176 [02:34<06:04, 95.43it/s]
 31%|███       | 15616/50176 [02:37<05:52, 97.94it/s]
 32%|███▏      | 15872/50176 [02:39<05:56, 96.33it/s]
 32%|███▏      | 16128/50176 [02:42<05:45, 98.44it/s]
 33%|███▎      | 16384/50176 [02:44<05:40, 99.32it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.89it/s]
 34%|███▎      | 16896/50176 [02:50<05:36, 98.80it/s]
 34%|███▍      | 17152/50176 [02:52<05:30, 99.85it/s]
 35%|███▍      | 17408/50176 [02:55<05:30, 99.19it/s]
 35%|███▌      | 17664/50176 [02:57<05:18, 102.10it/s]
 36%|███▌      | 17920/50176 [03:00<05:13, 102.79it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.67it/s]
 37%|███▋      | 18432/50176 [03:05<05:06, 103.46it/s]
 37%|███▋      | 18688/50176 [03:07<05:12, 100.85it/s]
 38%|███▊      | 18944/50176 [03:10<05:10, 100.71it/s]
 38%|███▊      | 19200/50176 [03:13<05:17, 97.65it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.58it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.88it/s]
 40%|███▉      | 19968/50176 [03:21<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:23<04:49, 103.34it/s]
 41%|████      | 20480/50176 [03:25<04:51, 101.94it/s]
 41%|████▏     | 20736/50176 [03:28<04:45, 102.99it/s]
 42%|████▏     | 20992/50176 [03:30<04:33, 106.80it/s]
 42%|████▏     | 21248/50176 [03:33<04:41, 102.77it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.62it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.21it/s]
 44%|████▍     | 22016/50176 [03:40<04:19, 108.54it/s]
 44%|████▍     | 22272/50176 [03:42<04:15, 109.21it/s]
 45%|████▍     | 22528/50176 [03:44<04:16, 107.61it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.19it/s]
 46%|████▌     | 23040/50176 [03:50<04:21, 103.64it/s]
 46%|████▋     | 23296/50176 [03:52<04:25, 101.34it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.22it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.86it/s]
 48%|████▊     | 24064/50176 [04:00<04:09, 104.67it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.94it/s]
 49%|████▉     | 24576/50176 [04:05<04:07, 103.32it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.77it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.26it/s]
 51%|█████     | 25344/50176 [04:12<03:49, 108.12it/s]
 51%|█████     | 25600/50176 [04:15<04:16, 95.79it/s] 
 52%|█████▏    | 25856/50176 [04:18<04:15, 95.16it/s]
 52%|█████▏    | 26112/50176 [04:20<04:07, 97.07it/s]
 53%|█████▎    | 26368/50176 [04:23<04:00, 99.14it/s]
 53%|█████▎    | 26624/50176 [04:25<03:53, 100.98it/s]
 54%|█████▎    | 26880/50176 [04:28<03:56, 98.52it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.59it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.16it/s]
 55%|█████▌    | 27648/50176 [04:35<03:28, 108.16it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.23it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 99.76it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:33, 101.99it/s]
 57%|█████▋    | 28672/50176 [04:45<03:31, 101.49it/s]
 58%|█████▊    | 28928/50176 [04:47<03:26, 103.10it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.26it/s]
 59%|█████▊    | 29440/50176 [04:53<03:31, 98.25it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:32, 96.52it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.29it/s]
 60%|██████    | 30208/50176 [05:01<03:20, 99.46it/s]
 61%|██████    | 30464/50176 [05:03<03:06, 105.67it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.68it/s]
 62%|██████▏   | 30976/50176 [05:08<03:01, 105.92it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 105.94it/s]
 63%|██████▎   | 31488/50176 [05:12<02:50, 109.36it/s]
 63%|██████▎   | 31744/50176 [05:15<02:51, 107.76it/s]
 64%|██████▍   | 32000/50176 [05:17<02:46, 109.25it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.81it/s]
 65%|██████▍   | 32512/50176 [05:22<02:50, 103.64it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 107.09it/s]
 66%|██████▌   | 33024/50176 [05:27<02:39, 107.60it/s]
 66%|██████▋   | 33280/50176 [05:29<02:37, 107.26it/s]
 67%|██████▋   | 33536/50176 [05:32<02:42, 102.32it/s]
 67%|██████▋   | 33792/50176 [05:34<02:37, 103.88it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 105.95it/s]
 68%|██████▊   | 34304/50176 [05:39<02:26, 108.49it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.50it/s]
 69%|██████▉   | 34816/50176 [05:44<02:27, 104.13it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.76it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.09it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.56it/s]
 71%|███████▏  | 35840/50176 [05:54<02:19, 103.12it/s]
 72%|███████▏  | 36096/50176 [05:56<02:14, 105.03it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.26it/s]
 73%|███████▎  | 36608/50176 [06:01<02:09, 104.76it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.88it/s]
 74%|███████▍  | 37120/50176 [06:06<02:06, 103.43it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 107.09it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.18it/s]
 76%|███████▌  | 37888/50176 [06:13<01:55, 106.79it/s]
 76%|███████▌  | 38144/50176 [06:15<01:51, 107.43it/s]
 77%|███████▋  | 38400/50176 [06:18<01:52, 104.40it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.63it/s]
 78%|███████▊  | 38912/50176 [06:23<01:46, 105.32it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.47it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.16it/s]
 79%|███████▉  | 39680/50176 [06:30<01:38, 106.92it/s]
 80%|███████▉  | 39936/50176 [06:32<01:33, 109.64it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.10it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 103.85it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.67it/s]
 82%|████████▏ | 40960/50176 [06:42<01:26, 106.41it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 105.05it/s]
 83%|████████▎ | 41472/50176 [06:47<01:22, 104.93it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.79it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.96it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.50it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.34it/s]
 85%|████████▌ | 42752/50176 [06:59<01:08, 107.82it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.07it/s]
 86%|████████▌ | 43264/50176 [07:04<01:05, 106.14it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.41it/s]
 87%|████████▋ | 43776/50176 [07:09<01:02, 102.37it/s]
 88%|████████▊ | 44032/50176 [07:11<00:59, 104.05it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 108.08it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.69it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.46it/s]
 90%|████████▉ | 45056/50176 [07:21<00:48, 105.14it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.75it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 107.83it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.56it/s]
 92%|█████████▏| 46080/50176 [07:31<00:39, 103.28it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 104.12it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.17it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.01it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 100.00it/s]
 94%|█████████▍| 47360/50176 [07:43<00:28, 99.92it/s] 
 95%|█████████▍| 47616/50176 [07:46<00:26, 98.34it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.98it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 102.45it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.54it/s]
 97%|█████████▋| 48640/50176 [07:56<00:14, 104.73it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.06it/s]
 98%|█████████▊| 49152/50176 [08:01<00:10, 101.93it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.34it/s]
 99%|█████████▉| 49664/50176 [08:06<00:05, 101.38it/s]
 99%|█████████▉| 49920/50176 [08:09<00:02, 96.06it/s] 
-2022-12-21,10:53:06 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:53:06 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6259	imagenet-zeroshot-val-top5: 0.8789

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_45.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:44:28 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:44:28 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:44:29 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt).
-2022-12-21,10:44:38 | INFO | Model:
-2022-12-21,10:44:38 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:44:38 | INFO | Params:
-2022-12-21,10:44:38 | INFO |   batch_size: 256
-2022-12-21,10:44:38 | INFO |   beta1: 0.9
-2022-12-21,10:44:38 | INFO |   beta2: 0.98
-2022-12-21,10:44:38 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/checkpoints
-2022-12-21,10:44:38 | INFO |   copy_codebase: False
-2022-12-21,10:44:38 | INFO |   csv_caption_key: title
-2022-12-21,10:44:38 | INFO |   csv_img_key: filepath
-2022-12-21,10:44:38 | INFO |   csv_separator: 	
-2022-12-21,10:44:38 | INFO |   dataset_resampled: False
-2022-12-21,10:44:38 | INFO |   dataset_type: auto
-2022-12-21,10:44:38 | INFO |   ddp_static_graph: False
-2022-12-21,10:44:38 | INFO |   debug: False
-2022-12-21,10:44:38 | INFO |   device: cuda:0
-2022-12-21,10:44:38 | INFO |   dist_backend: nccl
-2022-12-21,10:44:38 | INFO |   dist_url: env://
-2022-12-21,10:44:38 | INFO |   distributed: False
-2022-12-21,10:44:38 | INFO |   epochs: 32
-2022-12-21,10:44:38 | INFO |   eps: 1e-06
-2022-12-21,10:44:38 | INFO |   force_quick_gelu: False
-2022-12-21,10:44:38 | INFO |   gather_with_grad: False
-2022-12-21,10:44:38 | INFO |   grad_checkpointing: False
-2022-12-21,10:44:38 | INFO |   horovod: False
-2022-12-21,10:44:38 | INFO |   image_mean: None
-2022-12-21,10:44:38 | INFO |   image_std: None
-2022-12-21,10:44:38 | INFO |   imagenet_v2: None
-2022-12-21,10:44:38 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:44:38 | INFO |   local_loss: False
-2022-12-21,10:44:38 | INFO |   local_rank: 0
-2022-12-21,10:44:38 | INFO |   lock_image: False
-2022-12-21,10:44:38 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:44:38 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:44:38 | INFO |   log_level: 20
-2022-12-21,10:44:38 | INFO |   log_local: False
-2022-12-21,10:44:38 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_45.pt/out.log
-2022-12-21,10:44:38 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:44:38 | INFO |   lr: 0.0005
-2022-12-21,10:44:38 | INFO |   model: ViT-B-16
-2022-12-21,10:44:38 | INFO |   name: epoch_45.pt
-2022-12-21,10:44:38 | INFO |   no_set_device_rank: False
-2022-12-21,10:44:38 | INFO |   norm_gradient_clip: None
-2022-12-21,10:44:38 | INFO |   precision: amp_bfloat16
-2022-12-21,10:44:38 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_45.pt
-2022-12-21,10:44:38 | INFO |   pretrained_image: False
-2022-12-21,10:44:38 | INFO |   rank: 0
-2022-12-21,10:44:38 | INFO |   report_to: 
-2022-12-21,10:44:38 | INFO |   resume: None
-2022-12-21,10:44:38 | INFO |   save_frequency: 1
-2022-12-21,10:44:38 | INFO |   save_most_recent: False
-2022-12-21,10:44:38 | INFO |   seed: 0
-2022-12-21,10:44:38 | INFO |   skip_scheduler: False
-2022-12-21,10:44:38 | INFO |   tensorboard: False
-2022-12-21,10:44:38 | INFO |   tensorboard_path: 
-2022-12-21,10:44:38 | INFO |   torchscript: False
-2022-12-21,10:44:38 | INFO |   trace: False
-2022-12-21,10:44:38 | INFO |   train_data: None
-2022-12-21,10:44:38 | INFO |   train_num_samples: None
-2022-12-21,10:44:38 | INFO |   use_bn_sync: False
-2022-12-21,10:44:38 | INFO |   val_data: None
-2022-12-21,10:44:38 | INFO |   val_frequency: 1
-2022-12-21,10:44:38 | INFO |   val_num_samples: None
-2022-12-21,10:44:38 | INFO |   wandb: False
-2022-12-21,10:44:38 | INFO |   wandb_notes: 
-2022-12-21,10:44:38 | INFO |   warmup: 10000
-2022-12-21,10:44:38 | INFO |   wd: 0.2
-2022-12-21,10:44:38 | INFO |   workers: 1
-2022-12-21,10:44:38 | INFO |   world_size: 1
-2022-12-21,10:44:38 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:44:38 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:44:38 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:55,  1.40it/s]
  1%|          | 6/1000 [00:00<01:45,  9.45it/s]
  1%|          | 12/1000 [00:00<00:53, 18.49it/s]
  2%|▏         | 18/1000 [00:01<00:37, 26.43it/s]
  2%|▏         | 24/1000 [00:01<00:29, 33.64it/s]
  3%|▎         | 30/1000 [00:01<00:24, 39.58it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.18it/s]
  4%|▍         | 42/1000 [00:01<00:20, 47.76it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.37it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.68it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.81it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.48it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.01it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.58it/s]
  9%|▉         | 90/1000 [00:02<00:15, 56.92it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.33it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.50it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.88it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.87it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.80it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.82it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.78it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.82it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.88it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.92it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.77it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.61it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.63it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.58it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.49it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.45it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.42it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.20it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.09it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.25it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.23it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.46it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.43it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.34it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.41it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.49it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.59it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.63it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.45it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.48it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.63it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.73it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.95it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.92it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.69it/s]
 31%|███       | 312/1000 [00:06<00:13, 49.48it/s]
 32%|███▏      | 318/1000 [00:06<00:13, 51.78it/s]
 32%|███▏      | 324/1000 [00:06<00:12, 53.43it/s]
 33%|███▎      | 330/1000 [00:06<00:12, 54.55it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 55.55it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 56.32it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 56.89it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.03it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.28it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.53it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.73it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.71it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.44it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.55it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.67it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.83it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.91it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.91it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.93it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.97it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.07it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.07it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.87it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.92it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.99it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.95it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.90it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 58.04it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.95it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.93it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.93it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.86it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.91it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.82it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.89it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.94it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.96it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.09it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.10it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.99it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.99it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.05it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.91it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.86it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.98it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.96it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.88it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.99it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.90it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.91it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.02it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.02it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.11it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.93it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.88it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 58.03it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.99it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.06it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.07it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.99it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.05it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.06it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.08it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.97it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.06it/s]
 71%|███████   | 708/1000 [00:13<00:05, 58.04it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.00it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.02it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.98it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.03it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.07it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.06it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.14it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.22it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.04it/s]
 77%|███████▋  | 768/1000 [00:14<00:03, 58.12it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.83it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.93it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.00it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.82it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.84it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.92it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.90it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.89it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.92it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 58.01it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.15it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.19it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.10it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.16it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.13it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.14it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.07it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 58.08it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.95it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.07it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.05it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.99it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.01it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.01it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.00it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.09it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.12it/s]
 94%|█████████▍| 942/1000 [00:17<00:00, 58.13it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.16it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.19it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.15it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.22it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.28it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.03it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.84it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.90it/s]
-2022-12-21,10:44:56 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:09, 90.82it/s]
  1%|          | 512/50176 [00:05<08:55, 92.67it/s]
  2%|▏         | 768/50176 [00:07<08:21, 98.50it/s]
  2%|▏         | 1024/50176 [00:10<08:15, 99.28it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.51it/s]
  3%|▎         | 1536/50176 [00:16<08:40, 93.52it/s]
  4%|▎         | 1792/50176 [00:18<08:33, 94.25it/s]
  4%|▍         | 2048/50176 [00:21<08:14, 97.32it/s]
  5%|▍         | 2304/50176 [00:23<08:02, 99.18it/s]
  5%|▌         | 2560/50176 [00:26<07:54, 100.33it/s]
  6%|▌         | 2816/50176 [00:28<08:02, 98.23it/s] 
  6%|▌         | 3072/50176 [00:31<07:56, 98.93it/s]
  7%|▋         | 3328/50176 [00:33<07:43, 101.11it/s]
  7%|▋         | 3584/50176 [00:36<07:36, 102.03it/s]
  8%|▊         | 3840/50176 [00:38<07:41, 100.48it/s]
  8%|▊         | 4096/50176 [00:42<08:05, 94.89it/s] 
  9%|▊         | 4352/50176 [00:44<07:51, 97.21it/s]
  9%|▉         | 4608/50176 [00:46<07:37, 99.66it/s]
 10%|▉         | 4864/50176 [00:49<07:33, 100.00it/s]
 10%|█         | 5120/50176 [00:51<07:23, 101.57it/s]
 11%|█         | 5376/50176 [00:54<07:22, 101.32it/s]
 11%|█         | 5632/50176 [00:56<07:19, 101.41it/s]
 12%|█▏        | 5888/50176 [00:59<07:17, 101.33it/s]
 12%|█▏        | 6144/50176 [01:02<07:41, 95.31it/s] 
 13%|█▎        | 6400/50176 [01:05<07:32, 96.66it/s]
 13%|█▎        | 6656/50176 [01:07<07:11, 100.78it/s]
 14%|█▍        | 6912/50176 [01:10<07:14, 99.65it/s] 
 14%|█▍        | 7168/50176 [01:12<07:15, 98.83it/s]
 15%|█▍        | 7424/50176 [01:15<07:05, 100.46it/s]
 15%|█▌        | 7680/50176 [01:17<06:55, 102.39it/s]
 16%|█▌        | 7936/50176 [01:19<06:43, 104.81it/s]
 16%|█▋        | 8192/50176 [01:22<06:41, 104.50it/s]
 17%|█▋        | 8448/50176 [01:24<06:38, 104.82it/s]
 17%|█▋        | 8704/50176 [01:27<06:38, 104.14it/s]
 18%|█▊        | 8960/50176 [01:30<06:55, 99.11it/s] 
 18%|█▊        | 9216/50176 [01:32<06:37, 103.03it/s]
 19%|█▉        | 9472/50176 [01:34<06:31, 104.04it/s]
 19%|█▉        | 9728/50176 [01:37<06:36, 101.98it/s]
 20%|█▉        | 9984/50176 [01:40<06:45, 99.22it/s] 
 20%|██        | 10240/50176 [01:42<06:27, 103.12it/s]
 21%|██        | 10496/50176 [01:44<06:22, 103.64it/s]
 21%|██▏       | 10752/50176 [01:47<06:35, 99.72it/s] 
 22%|██▏       | 11008/50176 [01:50<06:28, 100.83it/s]
 22%|██▏       | 11264/50176 [01:52<06:23, 101.38it/s]
 23%|██▎       | 11520/50176 [01:54<06:16, 102.72it/s]
 23%|██▎       | 11776/50176 [01:57<06:19, 101.25it/s]
 24%|██▍       | 12032/50176 [02:00<06:11, 102.67it/s]
 24%|██▍       | 12288/50176 [02:02<06:00, 105.15it/s]
 25%|██▌       | 12544/50176 [02:04<06:04, 103.38it/s]
 26%|██▌       | 12800/50176 [02:07<06:06, 102.07it/s]
 26%|██▌       | 13056/50176 [02:10<06:08, 100.62it/s]
 27%|██▋       | 13312/50176 [02:12<06:01, 101.84it/s]
 27%|██▋       | 13568/50176 [02:15<06:12, 98.21it/s] 
 28%|██▊       | 13824/50176 [02:18<06:19, 95.79it/s]
 28%|██▊       | 14080/50176 [02:20<06:18, 95.31it/s]
 29%|██▊       | 14336/50176 [02:23<06:15, 95.33it/s]
 29%|██▉       | 14592/50176 [02:26<06:03, 97.83it/s]
 30%|██▉       | 14848/50176 [02:29<06:17, 93.67it/s]
 30%|███       | 15104/50176 [02:31<06:09, 94.90it/s]
 31%|███       | 15360/50176 [02:34<06:03, 95.68it/s]
 31%|███       | 15616/50176 [02:36<05:52, 97.94it/s]
 32%|███▏      | 15872/50176 [02:39<05:54, 96.66it/s]
 32%|███▏      | 16128/50176 [02:41<05:46, 98.33it/s]
 33%|███▎      | 16384/50176 [02:44<05:40, 99.33it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.95it/s]
 34%|███▎      | 16896/50176 [02:49<05:37, 98.57it/s]
 34%|███▍      | 17152/50176 [02:52<05:29, 100.12it/s]
 35%|███▍      | 17408/50176 [02:54<05:29, 99.40it/s] 
 35%|███▌      | 17664/50176 [02:57<05:18, 101.92it/s]
 36%|███▌      | 17920/50176 [02:59<05:13, 102.94it/s]
 36%|███▌      | 18176/50176 [03:02<05:11, 102.81it/s]
 37%|███▋      | 18432/50176 [03:04<05:07, 103.29it/s]
 37%|███▋      | 18688/50176 [03:07<05:12, 100.86it/s]
 38%|███▊      | 18944/50176 [03:09<05:10, 100.66it/s]
 38%|███▊      | 19200/50176 [03:12<05:17, 97.69it/s] 
 39%|███▉      | 19456/50176 [03:15<05:24, 94.63it/s]
 39%|███▉      | 19712/50176 [03:18<05:14, 96.96it/s]
 40%|███▉      | 19968/50176 [03:20<05:10, 97.43it/s]
 40%|████      | 20224/50176 [03:22<04:50, 103.28it/s]
 41%|████      | 20480/50176 [03:25<04:51, 102.00it/s]
 41%|████▏     | 20736/50176 [03:27<04:46, 102.93it/s]
 42%|████▏     | 20992/50176 [03:29<04:33, 106.80it/s]
 42%|████▏     | 21248/50176 [03:32<04:41, 102.66it/s]
 43%|████▎     | 21504/50176 [03:35<04:36, 103.55it/s]
 43%|████▎     | 21760/50176 [03:37<04:32, 104.18it/s]
 44%|████▍     | 22016/50176 [03:39<04:19, 108.59it/s]
 44%|████▍     | 22272/50176 [03:41<04:15, 109.14it/s]
 45%|████▍     | 22528/50176 [03:44<04:17, 107.58it/s]
 45%|████▌     | 22784/50176 [03:47<04:25, 103.10it/s]
 46%|████▌     | 23040/50176 [03:49<04:22, 103.55it/s]
 46%|████▋     | 23296/50176 [03:52<04:24, 101.46it/s]
 47%|████▋     | 23552/50176 [03:55<04:31, 98.06it/s] 
 47%|████▋     | 23808/50176 [03:57<04:18, 101.94it/s]
 48%|████▊     | 24064/50176 [03:59<04:09, 104.73it/s]
 48%|████▊     | 24320/50176 [04:02<04:11, 102.98it/s]
 49%|████▉     | 24576/50176 [04:04<04:07, 103.39it/s]
 49%|████▉     | 24832/50176 [04:07<04:01, 104.74it/s]
 50%|█████     | 25088/50176 [04:09<03:56, 106.22it/s]
 51%|█████     | 25344/50176 [04:11<03:49, 108.06it/s]
 51%|█████     | 25600/50176 [04:14<04:16, 95.69it/s] 
 52%|█████▏    | 25856/50176 [04:17<04:15, 95.19it/s]
 52%|█████▏    | 26112/50176 [04:20<04:08, 96.84it/s]
 53%|█████▎    | 26368/50176 [04:22<03:59, 99.26it/s]
 53%|█████▎    | 26624/50176 [04:25<03:52, 101.09it/s]
 54%|█████▎    | 26880/50176 [04:27<03:56, 98.36it/s] 
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.60it/s]
 55%|█████▍    | 27392/50176 [04:32<03:38, 104.18it/s]
 55%|█████▌    | 27648/50176 [04:34<03:27, 108.35it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.27it/s]
 56%|█████▌    | 28160/50176 [04:40<03:41, 99.32it/s] 
 57%|█████▋    | 28416/50176 [04:42<03:33, 101.85it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 101.40it/s]
 58%|█████▊    | 28928/50176 [04:47<03:25, 103.17it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.34it/s]
 59%|█████▊    | 29440/50176 [04:52<03:30, 98.32it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:32, 96.59it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.40it/s]
 60%|██████    | 30208/50176 [05:00<03:20, 99.43it/s]
 61%|██████    | 30464/50176 [05:02<03:06, 105.43it/s]
 61%|██████    | 30720/50176 [05:05<03:04, 105.35it/s]
 62%|██████▏   | 30976/50176 [05:07<03:00, 106.29it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.03it/s]
 63%|██████▎   | 31488/50176 [05:12<02:51, 109.22it/s]
 63%|██████▎   | 31744/50176 [05:14<02:50, 108.13it/s]
 64%|██████▍   | 32000/50176 [05:16<02:46, 109.43it/s]
 64%|██████▍   | 32256/50176 [05:19<02:43, 109.86it/s]
 65%|██████▍   | 32512/50176 [05:21<02:50, 103.77it/s]
 65%|██████▌   | 32768/50176 [05:24<02:42, 106.98it/s]
 66%|██████▌   | 33024/50176 [05:26<02:39, 107.81it/s]
 66%|██████▋   | 33280/50176 [05:28<02:37, 107.32it/s]
 67%|██████▋   | 33536/50176 [05:31<02:43, 101.89it/s]
 67%|██████▋   | 33792/50176 [05:34<02:37, 104.05it/s]
 68%|██████▊   | 34048/50176 [05:36<02:32, 105.99it/s]
 68%|██████▊   | 34304/50176 [05:38<02:26, 108.60it/s]
 69%|██████▉   | 34560/50176 [05:41<02:29, 104.62it/s]
 69%|██████▉   | 34816/50176 [05:43<02:27, 104.16it/s]
 70%|██████▉   | 35072/50176 [05:46<02:25, 103.80it/s]
 70%|███████   | 35328/50176 [05:48<02:18, 107.22it/s]
 71%|███████   | 35584/50176 [05:50<02:16, 106.71it/s]
 71%|███████▏  | 35840/50176 [05:53<02:18, 103.27it/s]
 72%|███████▏  | 36096/50176 [05:55<02:13, 105.13it/s]
 72%|███████▏  | 36352/50176 [05:58<02:12, 104.42it/s]
 73%|███████▎  | 36608/50176 [06:00<02:09, 104.64it/s]
 73%|███████▎  | 36864/50176 [06:03<02:09, 102.78it/s]
 74%|███████▍  | 37120/50176 [06:05<02:06, 103.41it/s]
 74%|███████▍  | 37376/50176 [06:08<01:59, 107.01it/s]
 75%|███████▌  | 37632/50176 [06:10<01:57, 107.16it/s]
 76%|███████▌  | 37888/50176 [06:12<01:55, 106.75it/s]
 76%|███████▌  | 38144/50176 [06:15<01:52, 107.35it/s]
 77%|███████▋  | 38400/50176 [06:17<01:52, 104.79it/s]
 77%|███████▋  | 38656/50176 [06:20<01:50, 104.40it/s]
 78%|███████▊  | 38912/50176 [06:22<01:47, 105.27it/s]
 78%|███████▊  | 39168/50176 [06:25<01:45, 104.44it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 106.13it/s]
 79%|███████▉  | 39680/50176 [06:29<01:38, 106.93it/s]
 80%|███████▉  | 39936/50176 [06:31<01:33, 109.57it/s]
 80%|████████  | 40192/50176 [06:34<01:32, 108.14it/s]
 81%|████████  | 40448/50176 [06:37<01:33, 104.10it/s]
 81%|████████  | 40704/50176 [06:39<01:33, 101.69it/s]
 82%|████████▏ | 40960/50176 [06:41<01:26, 106.44it/s]
 82%|████████▏ | 41216/50176 [06:44<01:25, 104.91it/s]
 83%|████████▎ | 41472/50176 [06:46<01:22, 105.05it/s]
 83%|████████▎ | 41728/50176 [06:49<01:20, 104.69it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.90it/s]
 84%|████████▍ | 42240/50176 [06:54<01:15, 105.49it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 109.30it/s]
 85%|████████▌ | 42752/50176 [06:58<01:08, 107.87it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 107.01it/s]
 86%|████████▌ | 43264/50176 [07:03<01:05, 105.95it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.42it/s]
 87%|████████▋ | 43776/50176 [07:08<01:02, 102.64it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 104.28it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.93it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 102.62it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 103.55it/s]
 90%|████████▉ | 45056/50176 [07:20<00:48, 105.04it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.69it/s]
 91%|█████████ | 45568/50176 [07:25<00:42, 108.09it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.69it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 103.10it/s]
 92%|█████████▏| 46336/50176 [07:33<00:36, 103.91it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 105.30it/s]
 93%|█████████▎| 46848/50176 [07:38<00:32, 103.77it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 99.11it/s] 
 94%|█████████▍| 47360/50176 [07:43<00:28, 100.04it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 98.70it/s] 
 95%|█████████▌| 47872/50176 [07:48<00:22, 100.93it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 102.65it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.74it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.00it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 103.30it/s]
 98%|█████████▊| 49152/50176 [08:00<00:10, 102.16it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 100.47it/s]
 99%|█████████▉| 49664/50176 [08:05<00:05, 101.42it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 96.14it/s] 
-2022-12-21,10:53:05 | INFO | Finished zero-shot imagenet.
-2022-12-21,10:53:05 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6278	imagenet-zeroshot-val-top5: 0.8803

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_46.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt).
-2022-12-21,10:53:30 | INFO | Model:
-2022-12-21,10:53:30 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:53:30 | INFO | Params:
-2022-12-21,10:53:30 | INFO |   batch_size: 256
-2022-12-21,10:53:30 | INFO |   beta1: 0.9
-2022-12-21,10:53:30 | INFO |   beta2: 0.98
-2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/checkpoints
-2022-12-21,10:53:30 | INFO |   copy_codebase: False
-2022-12-21,10:53:30 | INFO |   csv_caption_key: title
-2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
-2022-12-21,10:53:30 | INFO |   csv_separator: 	
-2022-12-21,10:53:30 | INFO |   dataset_resampled: False
-2022-12-21,10:53:30 | INFO |   dataset_type: auto
-2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
-2022-12-21,10:53:30 | INFO |   debug: False
-2022-12-21,10:53:30 | INFO |   device: cuda:0
-2022-12-21,10:53:30 | INFO |   dist_backend: nccl
-2022-12-21,10:53:30 | INFO |   dist_url: env://
-2022-12-21,10:53:30 | INFO |   distributed: False
-2022-12-21,10:53:30 | INFO |   epochs: 32
-2022-12-21,10:53:30 | INFO |   eps: 1e-06
-2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
-2022-12-21,10:53:30 | INFO |   gather_with_grad: False
-2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
-2022-12-21,10:53:30 | INFO |   horovod: False
-2022-12-21,10:53:30 | INFO |   image_mean: None
-2022-12-21,10:53:30 | INFO |   image_std: None
-2022-12-21,10:53:30 | INFO |   imagenet_v2: None
-2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:53:30 | INFO |   local_loss: False
-2022-12-21,10:53:30 | INFO |   local_rank: 0
-2022-12-21,10:53:30 | INFO |   lock_image: False
-2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:53:30 | INFO |   log_level: 20
-2022-12-21,10:53:30 | INFO |   log_local: False
-2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_46.pt/out.log
-2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:53:30 | INFO |   lr: 0.0005
-2022-12-21,10:53:30 | INFO |   model: ViT-B-16
-2022-12-21,10:53:30 | INFO |   name: epoch_46.pt
-2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
-2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
-2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
-2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_46.pt
-2022-12-21,10:53:30 | INFO |   pretrained_image: False
-2022-12-21,10:53:30 | INFO |   rank: 0
-2022-12-21,10:53:30 | INFO |   report_to: 
-2022-12-21,10:53:30 | INFO |   resume: None
-2022-12-21,10:53:30 | INFO |   save_frequency: 1
-2022-12-21,10:53:30 | INFO |   save_most_recent: False
-2022-12-21,10:53:30 | INFO |   seed: 0
-2022-12-21,10:53:30 | INFO |   skip_scheduler: False
-2022-12-21,10:53:30 | INFO |   tensorboard: False
-2022-12-21,10:53:30 | INFO |   tensorboard_path: 
-2022-12-21,10:53:30 | INFO |   torchscript: False
-2022-12-21,10:53:30 | INFO |   trace: False
-2022-12-21,10:53:30 | INFO |   train_data: None
-2022-12-21,10:53:30 | INFO |   train_num_samples: None
-2022-12-21,10:53:30 | INFO |   use_bn_sync: False
-2022-12-21,10:53:30 | INFO |   val_data: None
-2022-12-21,10:53:30 | INFO |   val_frequency: 1
-2022-12-21,10:53:30 | INFO |   val_num_samples: None
-2022-12-21,10:53:30 | INFO |   wandb: False
-2022-12-21,10:53:30 | INFO |   wandb_notes: 
-2022-12-21,10:53:30 | INFO |   warmup: 10000
-2022-12-21,10:53:30 | INFO |   wd: 0.2
-2022-12-21,10:53:30 | INFO |   workers: 1
-2022-12-21,10:53:30 | INFO |   world_size: 1
-2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:53:30 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:43,  1.42it/s]
  1%|          | 6/1000 [00:00<01:43,  9.58it/s]
  1%|          | 12/1000 [00:00<00:52, 18.83it/s]
  2%|▏         | 18/1000 [00:01<00:35, 27.30it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.36it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.12it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.53it/s]
  4%|▍         | 42/1000 [00:01<00:19, 47.92it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.49it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.54it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.58it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.28it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.79it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.12it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.45it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.87it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.06it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.32it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.45it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.43it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.36it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.39it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.35it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.43it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.55it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.57it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.45it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.33it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.33it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.29it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.22it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.17it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.10it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.90it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.75it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.85it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.80it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.01it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.02it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.94it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 57.01it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.11it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.22it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.26it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.06it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.09it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.28it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.39it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.53it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.50it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.47it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.63it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.54it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.44it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.26it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.34it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.46it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.54it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.34it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.34it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.42it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.50it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.39it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.09it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.16it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.26it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.40it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.46it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.44it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.46it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.46it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.58it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.59it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.38it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.43it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.46it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.45it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.43it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 57.60it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.53it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.52it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.51it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.41it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.46it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.37it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.44it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.48it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.56it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.43it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.57it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.58it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.50it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.50it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.57it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.45it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.40it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.55it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.53it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.43it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.55it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.46it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.46it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.59it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.57it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.65it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.47it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.40it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.55it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.50it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.59it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.64it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.58it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.61it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.60it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.62it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.52it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.61it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.59it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.57it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.57it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.52it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.32it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.38it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.40it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.53it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.60it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.45it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.57it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.54it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.58it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.62it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.43it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.42it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.48it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.46it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.43it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.45it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.55it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.68it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.72it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.62it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.67it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.62it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.65it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.60it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.66it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.66it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.53it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.63it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.62it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.58it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.59it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.54it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.54it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.64it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.68it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.68it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.70it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.71it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.65it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.72it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.79it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.57it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.32it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.28it/s]
-2022-12-21,10:53:48 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<08:48, 94.45it/s]
  1%|          | 512/50176 [00:05<08:47, 94.08it/s]
  2%|▏         | 768/50176 [00:07<08:21, 98.48it/s]
  2%|▏         | 1024/50176 [00:10<08:19, 98.49it/s]
  3%|▎         | 1280/50176 [00:13<08:31, 95.52it/s]
  3%|▎         | 1536/50176 [00:16<09:07, 88.79it/s]
  4%|▎         | 1792/50176 [00:19<08:46, 91.82it/s]
  4%|▍         | 2048/50176 [00:21<08:23, 95.64it/s]
  5%|▍         | 2304/50176 [00:24<08:06, 98.31it/s]
  5%|▌         | 2560/50176 [00:26<07:56, 99.84it/s]
  6%|▌         | 2816/50176 [00:29<08:18, 95.09it/s]
  6%|▌         | 3072/50176 [00:32<08:04, 97.15it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.28it/s]
  7%|▋         | 3584/50176 [00:36<07:42, 100.71it/s]
  8%|▊         | 3840/50176 [00:39<07:52, 97.96it/s] 
  8%|▊         | 4096/50176 [00:42<08:04, 95.11it/s]
  9%|▊         | 4352/50176 [00:45<07:49, 97.50it/s]
  9%|▉         | 4608/50176 [00:47<07:34, 100.25it/s]
 10%|▉         | 4864/50176 [00:49<07:32, 100.06it/s]
 10%|█         | 5120/50176 [00:52<07:21, 101.97it/s]
 11%|█         | 5376/50176 [00:54<07:23, 100.93it/s]
 11%|█         | 5632/50176 [00:57<07:32, 98.52it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.18it/s]
 12%|█▏        | 6144/50176 [01:03<07:56, 92.34it/s]
 13%|█▎        | 6400/50176 [01:06<07:41, 94.94it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.35it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.94it/s]
 14%|█▍        | 7168/50176 [01:13<07:16, 98.64it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.64it/s]
 15%|█▌        | 7680/50176 [01:18<07:03, 100.45it/s]
 16%|█▌        | 7936/50176 [01:20<06:50, 102.85it/s]
 16%|█▋        | 8192/50176 [01:23<06:48, 102.74it/s]
 17%|█▋        | 8448/50176 [01:25<06:44, 103.14it/s]
 17%|█▋        | 8704/50176 [01:28<06:54, 99.96it/s] 
 18%|█▊        | 8960/50176 [01:31<06:54, 99.46it/s]
 18%|█▊        | 9216/50176 [01:33<06:47, 100.63it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.88it/s]
 19%|█▉        | 9728/50176 [01:38<06:48, 99.07it/s] 
 20%|█▉        | 9984/50176 [01:41<06:51, 97.71it/s]
 20%|██        | 10240/50176 [01:43<06:30, 102.21it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.46it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.09it/s] 
 22%|██▏       | 11008/50176 [01:51<06:33, 99.48it/s]
 22%|██▏       | 11264/50176 [01:54<06:21, 102.00it/s]
 23%|██▎       | 11520/50176 [01:56<06:21, 101.29it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.79it/s]
 24%|██▍       | 12032/50176 [02:01<06:16, 101.43it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.91it/s]
 25%|██▌       | 12544/50176 [02:06<06:09, 101.82it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.68it/s]
 26%|██▌       | 13056/50176 [02:11<06:01, 102.82it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.09it/s]
 27%|██▋       | 13568/50176 [02:16<06:11, 98.50it/s] 
 28%|██▊       | 13824/50176 [02:19<06:19, 95.70it/s]
 28%|██▊       | 14080/50176 [02:22<06:19, 95.17it/s]
 29%|██▊       | 14336/50176 [02:25<06:21, 93.97it/s]
 29%|██▉       | 14592/50176 [02:27<06:07, 96.91it/s]
 30%|██▉       | 14848/50176 [02:30<06:20, 92.94it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.29it/s]
 31%|███       | 15360/50176 [02:35<05:58, 97.24it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.78it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.07it/s]
 32%|███▏      | 16128/50176 [02:43<05:43, 99.18it/s]
 33%|███▎      | 16384/50176 [02:46<05:38, 99.91it/s]
 33%|███▎      | 16640/50176 [02:48<05:48, 96.25it/s]
 34%|███▎      | 16896/50176 [02:51<05:41, 97.43it/s]
 34%|███▍      | 17152/50176 [02:53<05:30, 99.94it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 101.02it/s]
 35%|███▌      | 17664/50176 [02:58<05:16, 102.59it/s]
 36%|███▌      | 17920/50176 [03:01<05:11, 103.43it/s]
 36%|███▌      | 18176/50176 [03:03<05:09, 103.43it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.34it/s]
 37%|███▋      | 18688/50176 [03:08<05:03, 103.74it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.64it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.12it/s] 
 39%|███▉      | 19456/50176 [03:16<05:20, 95.95it/s]
 39%|███▉      | 19712/50176 [03:19<05:17, 95.82it/s]
 40%|███▉      | 19968/50176 [03:22<05:10, 97.42it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.29it/s]
 41%|████      | 20480/50176 [03:26<04:43, 104.62it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.05it/s]
 42%|████▏     | 20992/50176 [03:31<04:32, 107.28it/s]
 42%|████▏     | 21248/50176 [03:33<04:35, 105.08it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.33it/s]
 43%|████▎     | 21760/50176 [03:38<04:27, 106.38it/s]
 44%|████▍     | 22016/50176 [03:40<04:20, 108.09it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.19it/s]
 45%|████▍     | 22528/50176 [03:45<04:15, 108.10it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.18it/s]
 46%|████▌     | 23040/50176 [03:50<04:26, 101.92it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.59it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.88it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.34it/s]
 48%|████▊     | 24064/50176 [04:00<04:14, 102.53it/s]
 48%|████▊     | 24320/50176 [04:03<04:16, 100.92it/s]
 49%|████▉     | 24576/50176 [04:06<04:12, 101.23it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 105.05it/s]
 50%|█████     | 25088/50176 [04:10<03:56, 106.28it/s]
 51%|█████     | 25344/50176 [04:12<03:51, 107.46it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.17it/s]
 52%|█████▏    | 25856/50176 [04:17<03:46, 107.55it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.20it/s]
 53%|█████▎    | 26368/50176 [04:22<03:44, 105.95it/s]
 53%|█████▎    | 26624/50176 [04:25<03:42, 105.66it/s]
 54%|█████▎    | 26880/50176 [04:27<03:41, 105.06it/s]
 54%|█████▍    | 27136/50176 [04:29<03:40, 104.66it/s]
 55%|█████▍    | 27392/50176 [04:32<03:33, 106.47it/s]
 55%|█████▌    | 27648/50176 [04:34<03:25, 109.52it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.34it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.00it/s]
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.55it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 101.08it/s]
 58%|█████▊    | 28928/50176 [04:47<03:28, 102.15it/s]
 58%|█████▊    | 29184/50176 [04:49<03:23, 103.23it/s]
 59%|█████▊    | 29440/50176 [04:52<03:30, 98.65it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.97it/s]
 60%|█████▉    | 29952/50176 [04:58<03:27, 97.69it/s]
 60%|██████    | 30208/50176 [05:00<03:22, 98.65it/s]
 61%|██████    | 30464/50176 [05:02<03:07, 105.27it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 102.98it/s]
 62%|██████▏   | 30976/50176 [05:07<03:03, 104.85it/s]
 62%|██████▏   | 31232/50176 [05:09<02:58, 105.97it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.41it/s]
 63%|██████▎   | 31744/50176 [05:14<02:49, 108.55it/s]
 64%|██████▍   | 32000/50176 [05:16<02:47, 108.25it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.80it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.31it/s]
 65%|██████▌   | 32768/50176 [05:24<02:46, 104.67it/s]
 66%|██████▌   | 33024/50176 [05:26<02:44, 103.97it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.58it/s]
 67%|██████▋   | 33536/50176 [05:31<02:41, 102.98it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.68it/s]
 68%|██████▊   | 34048/50176 [05:36<02:30, 107.46it/s]
 68%|██████▊   | 34304/50176 [05:38<02:22, 111.43it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.42it/s]
 69%|██████▉   | 34816/50176 [05:43<02:25, 105.91it/s]
 70%|██████▉   | 35072/50176 [05:46<02:23, 105.60it/s]
 70%|███████   | 35328/50176 [05:48<02:20, 106.02it/s]
 71%|███████   | 35584/50176 [05:50<02:16, 106.82it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.14it/s]
 72%|███████▏  | 36096/50176 [05:55<02:12, 106.25it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.59it/s]
 73%|███████▎  | 36608/50176 [06:00<02:11, 103.01it/s]
 73%|███████▎  | 36864/50176 [06:03<02:12, 100.28it/s]
 74%|███████▍  | 37120/50176 [06:05<02:06, 102.83it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.50it/s]
 75%|███████▌  | 37632/50176 [06:10<01:58, 105.85it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.07it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.43it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.70it/s]
 77%|███████▋  | 38656/50176 [06:20<01:49, 105.62it/s]
 78%|███████▊  | 38912/50176 [06:22<01:47, 105.20it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.29it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 105.93it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.36it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.85it/s]
 80%|████████  | 40192/50176 [06:34<01:34, 105.87it/s]
 81%|████████  | 40448/50176 [06:37<01:34, 102.64it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.17it/s]
 82%|████████▏ | 40960/50176 [06:42<01:27, 104.81it/s]
 82%|████████▏ | 41216/50176 [06:44<01:24, 105.59it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 104.07it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 106.69it/s]
 84%|████████▎ | 41984/50176 [06:51<01:16, 106.98it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.86it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 108.82it/s]
 85%|████████▌ | 42752/50176 [06:58<01:07, 109.61it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.14it/s]
 86%|████████▌ | 43264/50176 [07:03<01:04, 107.38it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.44it/s]
 87%|████████▋ | 43776/50176 [07:08<01:01, 103.23it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 105.81it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.88it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 104.04it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.95it/s]
 90%|████████▉ | 45056/50176 [07:20<00:47, 107.92it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.14it/s]
 91%|█████████ | 45568/50176 [07:25<00:43, 106.72it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.97it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 104.84it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.62it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 104.76it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.17it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 101.19it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.10it/s]
 95%|█████████▍| 47616/50176 [07:45<00:25, 100.78it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.73it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 102.51it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.60it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.97it/s]
 97%|█████████▋| 48896/50176 [07:57<00:12, 106.32it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.89it/s]
 98%|█████████▊| 49408/50176 [08:02<00:07, 105.54it/s]
 99%|█████████▉| 49664/50176 [08:05<00:04, 102.62it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.24it/s] 
-2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6270	imagenet-zeroshot-val-top5: 0.8812

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_47.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt).
-2022-12-21,10:53:30 | INFO | Model:
-2022-12-21,10:53:30 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:53:30 | INFO | Params:
-2022-12-21,10:53:30 | INFO |   batch_size: 256
-2022-12-21,10:53:30 | INFO |   beta1: 0.9
-2022-12-21,10:53:30 | INFO |   beta2: 0.98
-2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/checkpoints
-2022-12-21,10:53:30 | INFO |   copy_codebase: False
-2022-12-21,10:53:30 | INFO |   csv_caption_key: title
-2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
-2022-12-21,10:53:30 | INFO |   csv_separator: 	
-2022-12-21,10:53:30 | INFO |   dataset_resampled: False
-2022-12-21,10:53:30 | INFO |   dataset_type: auto
-2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
-2022-12-21,10:53:30 | INFO |   debug: False
-2022-12-21,10:53:30 | INFO |   device: cuda:0
-2022-12-21,10:53:30 | INFO |   dist_backend: nccl
-2022-12-21,10:53:30 | INFO |   dist_url: env://
-2022-12-21,10:53:30 | INFO |   distributed: False
-2022-12-21,10:53:30 | INFO |   epochs: 32
-2022-12-21,10:53:30 | INFO |   eps: 1e-06
-2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
-2022-12-21,10:53:30 | INFO |   gather_with_grad: False
-2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
-2022-12-21,10:53:30 | INFO |   horovod: False
-2022-12-21,10:53:30 | INFO |   image_mean: None
-2022-12-21,10:53:30 | INFO |   image_std: None
-2022-12-21,10:53:30 | INFO |   imagenet_v2: None
-2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:53:30 | INFO |   local_loss: False
-2022-12-21,10:53:30 | INFO |   local_rank: 0
-2022-12-21,10:53:30 | INFO |   lock_image: False
-2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:53:30 | INFO |   log_level: 20
-2022-12-21,10:53:30 | INFO |   log_local: False
-2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_47.pt/out.log
-2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:53:30 | INFO |   lr: 0.0005
-2022-12-21,10:53:30 | INFO |   model: ViT-B-16
-2022-12-21,10:53:30 | INFO |   name: epoch_47.pt
-2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
-2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
-2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
-2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_47.pt
-2022-12-21,10:53:30 | INFO |   pretrained_image: False
-2022-12-21,10:53:30 | INFO |   rank: 0
-2022-12-21,10:53:30 | INFO |   report_to: 
-2022-12-21,10:53:30 | INFO |   resume: None
-2022-12-21,10:53:30 | INFO |   save_frequency: 1
-2022-12-21,10:53:30 | INFO |   save_most_recent: False
-2022-12-21,10:53:30 | INFO |   seed: 0
-2022-12-21,10:53:30 | INFO |   skip_scheduler: False
-2022-12-21,10:53:30 | INFO |   tensorboard: False
-2022-12-21,10:53:30 | INFO |   tensorboard_path: 
-2022-12-21,10:53:30 | INFO |   torchscript: False
-2022-12-21,10:53:30 | INFO |   trace: False
-2022-12-21,10:53:30 | INFO |   train_data: None
-2022-12-21,10:53:30 | INFO |   train_num_samples: None
-2022-12-21,10:53:30 | INFO |   use_bn_sync: False
-2022-12-21,10:53:30 | INFO |   val_data: None
-2022-12-21,10:53:30 | INFO |   val_frequency: 1
-2022-12-21,10:53:30 | INFO |   val_num_samples: None
-2022-12-21,10:53:30 | INFO |   wandb: False
-2022-12-21,10:53:30 | INFO |   wandb_notes: 
-2022-12-21,10:53:30 | INFO |   warmup: 10000
-2022-12-21,10:53:30 | INFO |   wd: 0.2
-2022-12-21,10:53:30 | INFO |   workers: 1
-2022-12-21,10:53:30 | INFO |   world_size: 1
-2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:53:30 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:23,  1.60it/s]
  1%|          | 6/1000 [00:00<01:33, 10.62it/s]
  1%|          | 12/1000 [00:00<00:48, 20.47it/s]
  2%|▏         | 18/1000 [00:00<00:33, 29.23it/s]
  2%|▏         | 24/1000 [00:01<00:26, 36.34it/s]
  3%|▎         | 30/1000 [00:01<00:23, 42.04it/s]
  4%|▎         | 36/1000 [00:01<00:20, 46.16it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.31it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.67it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.46it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.07it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.06it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.81it/s]
  8%|▊         | 78/1000 [00:01<00:16, 56.28it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.78it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.86it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.28it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.49it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.76it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.86it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.81it/s]
 13%|█▎        | 132/1000 [00:02<00:15, 57.85it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.82it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.85it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.91it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.93it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.80it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.66it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.67it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.65it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.47it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.45it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.44it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.26it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.09it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.22it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.23it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.25it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.26it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.22it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.32it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 57.43it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.58it/s]
 26%|██▋       | 264/1000 [00:05<00:15, 47.83it/s]
 27%|██▋       | 270/1000 [00:05<00:14, 50.25it/s]
 28%|██▊       | 276/1000 [00:05<00:13, 52.21it/s]
 28%|██▊       | 282/1000 [00:05<00:13, 53.80it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 54.96it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 55.92it/s]
 30%|███       | 300/1000 [00:05<00:12, 56.46it/s]
 31%|███       | 306/1000 [00:06<00:12, 56.84it/s]
 31%|███       | 312/1000 [00:06<00:12, 57.27it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.44it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.46it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.36it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.54it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.73it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.88it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.70it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.73it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.82it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.93it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.51it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.58it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.70it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.85it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.93it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.90it/s]
 42%|████▏     | 420/1000 [00:07<00:10, 57.92it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 57.94it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.06it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.06it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.84it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.90it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.94it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.91it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.87it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.02it/s]
 48%|████▊     | 480/1000 [00:09<00:08, 57.92it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.91it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.91it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.81it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.84it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.74it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.84it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.89it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.97it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 57.92it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.01it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.99it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.86it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.86it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.91it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.81it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.75it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.87it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.87it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.81it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.92it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.83it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.84it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.95it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.96it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.04it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.88it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.82it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 57.72it/s]
 65%|██████▌   | 654/1000 [00:12<00:05, 57.73it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.81it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.88it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.83it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.90it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.92it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.84it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.94it/s]
 71%|███████   | 708/1000 [00:12<00:05, 57.94it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.92it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.94it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.89it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.89it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.90it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.90it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.98it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.06it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 57.90it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.01it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.95it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.97it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.02it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.80it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.78it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.83it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.79it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.78it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 57.81it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.92it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.02it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.07it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.99it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.03it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.99it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.00it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.94it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 57.99it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 57.98it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.85it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.96it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.97it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.93it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.95it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.94it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.93it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.98it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.05it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.05it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.09it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.11it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.05it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.10it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.17it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.11it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.94it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.82it/s]
-2022-12-21,10:53:48 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:02<09:13, 90.19it/s]
  1%|          | 512/50176 [00:05<08:58, 92.17it/s]
  2%|▏         | 768/50176 [00:08<08:27, 97.31it/s]
  2%|▏         | 1024/50176 [00:10<08:23, 97.69it/s]
  3%|▎         | 1280/50176 [00:13<08:34, 95.07it/s]
  3%|▎         | 1536/50176 [00:16<09:09, 88.50it/s]
  4%|▎         | 1792/50176 [00:19<08:47, 91.67it/s]
  4%|▍         | 2048/50176 [00:21<08:24, 95.43it/s]
  5%|▍         | 2304/50176 [00:24<08:06, 98.33it/s]
  5%|▌         | 2560/50176 [00:26<07:58, 99.61it/s]
  6%|▌         | 2816/50176 [00:29<08:18, 95.07it/s]
  6%|▌         | 3072/50176 [00:32<08:05, 97.00it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.25it/s]
  7%|▋         | 3584/50176 [00:37<07:42, 100.74it/s]
  8%|▊         | 3840/50176 [00:39<07:53, 97.88it/s] 
  8%|▊         | 4096/50176 [00:42<08:04, 95.10it/s]
  9%|▊         | 4352/50176 [00:45<07:49, 97.62it/s]
  9%|▉         | 4608/50176 [00:47<07:34, 100.23it/s]
 10%|▉         | 4864/50176 [00:50<07:33, 99.95it/s] 
 10%|█         | 5120/50176 [00:52<07:21, 101.95it/s]
 11%|█         | 5376/50176 [00:55<07:24, 100.84it/s]
 11%|█         | 5632/50176 [00:57<07:31, 98.64it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.11it/s]
 12%|█▏        | 6144/50176 [01:03<07:56, 92.34it/s]
 13%|█▎        | 6400/50176 [01:06<07:40, 95.10it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.33it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.94it/s]
 14%|█▍        | 7168/50176 [01:13<07:16, 98.49it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.63it/s]
 15%|█▌        | 7680/50176 [01:18<07:03, 100.32it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.99it/s]
 16%|█▋        | 8192/50176 [01:23<06:49, 102.53it/s]
 17%|█▋        | 8448/50176 [01:26<06:44, 103.19it/s]
 17%|█▋        | 8704/50176 [01:28<06:54, 100.05it/s]
 18%|█▊        | 8960/50176 [01:31<06:53, 99.68it/s] 
 18%|█▊        | 9216/50176 [01:33<06:47, 100.50it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.94it/s]
 19%|█▉        | 9728/50176 [01:39<06:48, 99.02it/s] 
 20%|█▉        | 9984/50176 [01:41<06:50, 97.85it/s]
 20%|██        | 10240/50176 [01:44<06:30, 102.17it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.40it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.19it/s] 
 22%|██▏       | 11008/50176 [01:51<06:34, 99.39it/s]
 22%|██▏       | 11264/50176 [01:54<06:20, 102.17it/s]
 23%|██▎       | 11520/50176 [01:56<06:21, 101.34it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.80it/s]
 24%|██▍       | 12032/50176 [02:01<06:16, 101.45it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.80it/s]
 25%|██▌       | 12544/50176 [02:06<06:09, 101.81it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.75it/s]
 26%|██▌       | 13056/50176 [02:11<06:00, 102.97it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.08it/s]
 27%|██▋       | 13568/50176 [02:17<06:11, 98.49it/s] 
 28%|██▊       | 13824/50176 [02:19<06:19, 95.72it/s]
 28%|██▊       | 14080/50176 [02:22<06:19, 95.10it/s]
 29%|██▊       | 14336/50176 [02:25<06:20, 94.09it/s]
 29%|██▉       | 14592/50176 [02:27<06:07, 96.85it/s]
 30%|██▉       | 14848/50176 [02:30<06:20, 92.85it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.30it/s]
 31%|███       | 15360/50176 [02:36<05:58, 97.15it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.84it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.14it/s]
 32%|███▏      | 16128/50176 [02:43<05:43, 99.12it/s]
 33%|███▎      | 16384/50176 [02:46<05:37, 99.98it/s]
 33%|███▎      | 16640/50176 [02:49<05:47, 96.37it/s]
 34%|███▎      | 16896/50176 [02:51<05:41, 97.42it/s]
 34%|███▍      | 17152/50176 [02:54<05:30, 99.83it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 101.01it/s]
 35%|███▌      | 17664/50176 [02:58<05:17, 102.55it/s]
 36%|███▌      | 17920/50176 [03:01<05:11, 103.42it/s]
 36%|███▌      | 18176/50176 [03:03<05:09, 103.45it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.29it/s]
 37%|███▋      | 18688/50176 [03:08<05:03, 103.65it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.73it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.12it/s] 
 39%|███▉      | 19456/50176 [03:16<05:19, 96.00it/s]
 39%|███▉      | 19712/50176 [03:19<05:18, 95.68it/s]
 40%|███▉      | 19968/50176 [03:22<05:09, 97.45it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.35it/s]
 41%|████      | 20480/50176 [03:26<04:44, 104.46it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.07it/s]
 42%|████▏     | 20992/50176 [03:31<04:31, 107.31it/s]
 42%|████▏     | 21248/50176 [03:33<04:35, 104.98it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.36it/s]
 43%|████▎     | 21760/50176 [03:38<04:26, 106.48it/s]
 44%|████▍     | 22016/50176 [03:41<04:20, 107.99it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.20it/s]
 45%|████▍     | 22528/50176 [03:45<04:15, 108.03it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.20it/s]
 46%|████▌     | 23040/50176 [03:51<04:25, 102.08it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.58it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.86it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.33it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.53it/s]
 48%|████▊     | 24320/50176 [04:03<04:16, 100.96it/s]
 49%|████▉     | 24576/50176 [04:06<04:13, 101.12it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 104.95it/s]
 50%|█████     | 25088/50176 [04:10<03:55, 106.33it/s]
 51%|█████     | 25344/50176 [04:13<03:51, 107.41it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.22it/s]
 52%|█████▏    | 25856/50176 [04:17<03:45, 107.65it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.21it/s]
 53%|█████▎    | 26368/50176 [04:22<03:44, 105.90it/s]
 53%|█████▎    | 26624/50176 [04:25<03:43, 105.60it/s]
 54%|█████▎    | 26880/50176 [04:27<03:41, 105.16it/s]
 54%|█████▍    | 27136/50176 [04:30<03:40, 104.66it/s]
 55%|█████▍    | 27392/50176 [04:32<03:34, 106.39it/s]
 55%|█████▌    | 27648/50176 [04:34<03:25, 109.61it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.28it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.06it/s]
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.57it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 100.96it/s]
 58%|█████▊    | 28928/50176 [04:47<03:27, 102.16it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.25it/s]
 59%|█████▊    | 29440/50176 [04:52<03:30, 98.73it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.85it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.81it/s]
 60%|██████    | 30208/50176 [05:00<03:22, 98.58it/s]
 61%|██████    | 30464/50176 [05:02<03:07, 105.30it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 103.03it/s]
 62%|██████▏   | 30976/50176 [05:07<03:03, 104.72it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.14it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.51it/s]
 63%|██████▎   | 31744/50176 [05:14<02:49, 108.55it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.22it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.81it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.17it/s]
 65%|██████▌   | 32768/50176 [05:24<02:46, 104.68it/s]
 66%|██████▌   | 33024/50176 [05:27<02:45, 103.90it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.55it/s]
 67%|██████▋   | 33536/50176 [05:32<02:41, 103.12it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.68it/s]
 68%|██████▊   | 34048/50176 [05:36<02:30, 107.29it/s]
 68%|██████▊   | 34304/50176 [05:38<02:22, 111.59it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.55it/s]
 69%|██████▉   | 34816/50176 [05:43<02:24, 106.02it/s]
 70%|██████▉   | 35072/50176 [05:46<02:23, 105.54it/s]
 70%|███████   | 35328/50176 [05:48<02:19, 106.15it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.71it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.10it/s]
 72%|███████▏  | 36096/50176 [05:55<02:12, 106.30it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.63it/s]
 73%|███████▎  | 36608/50176 [06:01<02:11, 102.85it/s]
 73%|███████▎  | 36864/50176 [06:03<02:12, 100.30it/s]
 74%|███████▍  | 37120/50176 [06:06<02:07, 102.67it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.46it/s]
 75%|███████▌  | 37632/50176 [06:10<01:58, 105.78it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.13it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.29it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.74it/s]
 77%|███████▋  | 38656/50176 [06:20<01:49, 105.67it/s]
 78%|███████▊  | 38912/50176 [06:22<01:46, 105.40it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.35it/s]
 79%|███████▊  | 39424/50176 [06:27<01:41, 105.99it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.34it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.87it/s]
 80%|████████  | 40192/50176 [06:35<01:34, 105.97it/s]
 81%|████████  | 40448/50176 [06:37<01:34, 102.51it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.20it/s]
 82%|████████▏ | 40960/50176 [06:42<01:28, 104.63it/s]
 82%|████████▏ | 41216/50176 [06:44<01:24, 105.78it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 103.94it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 106.73it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.94it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.79it/s]
 85%|████████▍ | 42496/50176 [06:56<01:10, 108.85it/s]
 85%|████████▌ | 42752/50176 [06:59<01:07, 109.73it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.23it/s]
 86%|████████▌ | 43264/50176 [07:03<01:04, 107.43it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.24it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 103.33it/s]
 88%|████████▊ | 44032/50176 [07:11<00:57, 105.94it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.78it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 103.91it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.84it/s]
 90%|████████▉ | 45056/50176 [07:20<00:47, 107.92it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.11it/s]
 91%|█████████ | 45568/50176 [07:25<00:43, 106.82it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.92it/s]
 92%|█████████▏| 46080/50176 [07:30<00:39, 104.80it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.51it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 104.95it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.20it/s]
 94%|█████████▍| 47104/50176 [07:40<00:30, 101.16it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.15it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 100.87it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.79it/s]
 96%|█████████▌| 48128/50176 [07:50<00:19, 102.52it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.62it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 105.00it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 106.13it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.99it/s]
 98%|█████████▊| 49408/50176 [08:02<00:07, 105.52it/s]
 99%|█████████▉| 49664/50176 [08:05<00:04, 102.63it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.23it/s] 
-2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6319	imagenet-zeroshot-val-top5: 0.8823

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_48.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:53:20 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:53:20 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt).
-2022-12-21,10:53:29 | INFO | Model:
-2022-12-21,10:53:29 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:53:29 | INFO | Params:
-2022-12-21,10:53:29 | INFO |   batch_size: 256
-2022-12-21,10:53:29 | INFO |   beta1: 0.9
-2022-12-21,10:53:29 | INFO |   beta2: 0.98
-2022-12-21,10:53:29 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/checkpoints
-2022-12-21,10:53:29 | INFO |   copy_codebase: False
-2022-12-21,10:53:29 | INFO |   csv_caption_key: title
-2022-12-21,10:53:29 | INFO |   csv_img_key: filepath
-2022-12-21,10:53:29 | INFO |   csv_separator: 	
-2022-12-21,10:53:29 | INFO |   dataset_resampled: False
-2022-12-21,10:53:29 | INFO |   dataset_type: auto
-2022-12-21,10:53:29 | INFO |   ddp_static_graph: False
-2022-12-21,10:53:29 | INFO |   debug: False
-2022-12-21,10:53:29 | INFO |   device: cuda:0
-2022-12-21,10:53:29 | INFO |   dist_backend: nccl
-2022-12-21,10:53:29 | INFO |   dist_url: env://
-2022-12-21,10:53:29 | INFO |   distributed: False
-2022-12-21,10:53:29 | INFO |   epochs: 32
-2022-12-21,10:53:29 | INFO |   eps: 1e-06
-2022-12-21,10:53:29 | INFO |   force_quick_gelu: False
-2022-12-21,10:53:29 | INFO |   gather_with_grad: False
-2022-12-21,10:53:29 | INFO |   grad_checkpointing: False
-2022-12-21,10:53:29 | INFO |   horovod: False
-2022-12-21,10:53:29 | INFO |   image_mean: None
-2022-12-21,10:53:29 | INFO |   image_std: None
-2022-12-21,10:53:29 | INFO |   imagenet_v2: None
-2022-12-21,10:53:29 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:53:29 | INFO |   local_loss: False
-2022-12-21,10:53:29 | INFO |   local_rank: 0
-2022-12-21,10:53:29 | INFO |   lock_image: False
-2022-12-21,10:53:29 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:53:29 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:53:29 | INFO |   log_level: 20
-2022-12-21,10:53:29 | INFO |   log_local: False
-2022-12-21,10:53:29 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_48.pt/out.log
-2022-12-21,10:53:29 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:53:29 | INFO |   lr: 0.0005
-2022-12-21,10:53:29 | INFO |   model: ViT-B-16
-2022-12-21,10:53:29 | INFO |   name: epoch_48.pt
-2022-12-21,10:53:29 | INFO |   no_set_device_rank: False
-2022-12-21,10:53:29 | INFO |   norm_gradient_clip: None
-2022-12-21,10:53:29 | INFO |   precision: amp_bfloat16
-2022-12-21,10:53:29 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_48.pt
-2022-12-21,10:53:29 | INFO |   pretrained_image: False
-2022-12-21,10:53:29 | INFO |   rank: 0
-2022-12-21,10:53:29 | INFO |   report_to: 
-2022-12-21,10:53:29 | INFO |   resume: None
-2022-12-21,10:53:29 | INFO |   save_frequency: 1
-2022-12-21,10:53:29 | INFO |   save_most_recent: False
-2022-12-21,10:53:29 | INFO |   seed: 0
-2022-12-21,10:53:29 | INFO |   skip_scheduler: False
-2022-12-21,10:53:29 | INFO |   tensorboard: False
-2022-12-21,10:53:29 | INFO |   tensorboard_path: 
-2022-12-21,10:53:29 | INFO |   torchscript: False
-2022-12-21,10:53:29 | INFO |   trace: False
-2022-12-21,10:53:29 | INFO |   train_data: None
-2022-12-21,10:53:29 | INFO |   train_num_samples: None
-2022-12-21,10:53:29 | INFO |   use_bn_sync: False
-2022-12-21,10:53:29 | INFO |   val_data: None
-2022-12-21,10:53:29 | INFO |   val_frequency: 1
-2022-12-21,10:53:29 | INFO |   val_num_samples: None
-2022-12-21,10:53:29 | INFO |   wandb: False
-2022-12-21,10:53:29 | INFO |   wandb_notes: 
-2022-12-21,10:53:29 | INFO |   warmup: 10000
-2022-12-21,10:53:29 | INFO |   wd: 0.2
-2022-12-21,10:53:29 | INFO |   workers: 1
-2022-12-21,10:53:29 | INFO |   world_size: 1
-2022-12-21,10:53:29 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:53:30 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:01,  1.51it/s]
  1%|          | 6/1000 [00:00<01:39, 10.01it/s]
  1%|          | 11/1000 [00:00<00:54, 18.01it/s]
  2%|▏         | 17/1000 [00:00<00:36, 27.06it/s]
  2%|▏         | 23/1000 [00:01<00:28, 34.60it/s]
  3%|▎         | 29/1000 [00:01<00:23, 40.63it/s]
  4%|▎         | 35/1000 [00:01<00:21, 45.17it/s]
  4%|▍         | 41/1000 [00:01<00:19, 48.72it/s]
  5%|▍         | 47/1000 [00:01<00:18, 51.11it/s]
  5%|▌         | 53/1000 [00:01<00:17, 53.05it/s]
  6%|▌         | 59/1000 [00:01<00:17, 54.27it/s]
  6%|▋         | 65/1000 [00:01<00:16, 55.24it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.78it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.30it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.73it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.15it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.37it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.59it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.80it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.95it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 57.98it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 57.93it/s]
 13%|█▎        | 131/1000 [00:02<00:14, 58.02it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 57.94it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.97it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.01it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.07it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.94it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.82it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.82it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.67it/s]
 19%|█▉        | 191/1000 [00:03<00:14, 57.66it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.61it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.50it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.34it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 221/1000 [00:04<00:15, 50.46it/s]
 23%|██▎       | 227/1000 [00:04<00:14, 52.51it/s]
 23%|██▎       | 233/1000 [00:04<00:14, 54.00it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 54.89it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 55.66it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 56.27it/s]
 26%|██▌       | 257/1000 [00:05<00:13, 56.85it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.16it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.12it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.36it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.61it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.76it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.94it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 57.97it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.02it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.17it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.16it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.12it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.93it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.05it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.17it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 57.98it/s]
 36%|███▌      | 359/1000 [00:06<00:11, 58.12it/s]
 36%|███▋      | 365/1000 [00:07<00:10, 58.05it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 58.19it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 58.03it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.78it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.76it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.86it/s]
 40%|████      | 401/1000 [00:07<00:10, 58.00it/s]
 41%|████      | 407/1000 [00:07<00:10, 58.09it/s]
 41%|████▏     | 413/1000 [00:07<00:10, 58.13it/s]
 42%|████▏     | 419/1000 [00:07<00:09, 58.16it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 58.10it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 58.23it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.23it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 58.03it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.07it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.13it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.11it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 473/1000 [00:08<00:09, 58.22it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.14it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.13it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 58.08it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.98it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.10it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.98it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.04it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.09it/s]
 53%|█████▎    | 527/1000 [00:09<00:08, 58.17it/s]
 53%|█████▎    | 533/1000 [00:09<00:08, 58.08it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.20it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.22it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.09it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.09it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.17it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.04it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.98it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.15it/s]
 59%|█████▊    | 587/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 593/1000 [00:10<00:07, 58.04it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.15it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.06it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.13it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.21it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.24it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.12it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 57.99it/s]
 65%|██████▍   | 647/1000 [00:11<00:06, 58.19it/s]
 65%|██████▌   | 653/1000 [00:11<00:05, 58.11it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.21it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.20it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.25it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.21it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.26it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.13it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.23it/s]
 71%|███████   | 707/1000 [00:12<00:05, 58.24it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.23it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.26it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.19it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.16it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.21it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.17it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.38it/s]
 76%|███████▌  | 761/1000 [00:13<00:04, 58.11it/s]
 77%|███████▋  | 767/1000 [00:13<00:04, 58.16it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.17it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.22it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.26it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.06it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.07it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.09it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.10it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.08it/s]
 82%|████████▏ | 821/1000 [00:14<00:03, 58.16it/s]
 83%|████████▎ | 827/1000 [00:14<00:02, 58.15it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.33it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.41it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.26it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.30it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.24it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.28it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.21it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.23it/s]
 88%|████████▊ | 881/1000 [00:15<00:02, 58.26it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 58.11it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.20it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.23it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.18it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.15it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.20it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.13it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.25it/s]
 94%|█████████▎| 935/1000 [00:16<00:01, 58.33it/s]
 94%|█████████▍| 941/1000 [00:16<00:01, 58.32it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.29it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.32it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.26it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.45it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.38it/s]
-2022-12-21,10:53:47 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:37, 78.24it/s]
  1%|          | 512/50176 [00:05<09:32, 86.79it/s]
  2%|▏         | 768/50176 [00:08<08:44, 94.18it/s]
  2%|▏         | 1024/50176 [00:11<08:33, 95.63it/s]
  3%|▎         | 1280/50176 [00:13<08:40, 93.88it/s]
  3%|▎         | 1536/50176 [00:17<09:13, 87.81it/s]
  4%|▎         | 1792/50176 [00:19<08:50, 91.13it/s]
  4%|▍         | 2048/50176 [00:22<08:25, 95.14it/s]
  5%|▍         | 2304/50176 [00:24<08:09, 97.80it/s]
  5%|▌         | 2560/50176 [00:27<07:58, 99.61it/s]
  6%|▌         | 2816/50176 [00:30<08:18, 94.98it/s]
  6%|▌         | 3072/50176 [00:32<08:05, 97.06it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.22it/s]
  7%|▋         | 3584/50176 [00:37<07:43, 100.60it/s]
  8%|▊         | 3840/50176 [00:40<07:53, 97.79it/s] 
  8%|▊         | 4096/50176 [00:43<08:04, 95.14it/s]
  9%|▊         | 4352/50176 [00:45<07:49, 97.53it/s]
  9%|▉         | 4608/50176 [00:47<07:35, 100.13it/s]
 10%|▉         | 4864/50176 [00:50<07:33, 99.81it/s] 
 10%|█         | 5120/50176 [00:52<07:21, 101.94it/s]
 11%|█         | 5376/50176 [00:55<07:23, 101.13it/s]
 11%|█         | 5632/50176 [00:58<07:31, 98.57it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.16it/s]
 12%|█▏        | 6144/50176 [01:04<07:57, 92.24it/s]
 13%|█▎        | 6400/50176 [01:06<07:40, 94.99it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.39it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.98it/s]
 14%|█▍        | 7168/50176 [01:14<07:16, 98.56it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.72it/s]
 15%|█▌        | 7680/50176 [01:19<07:03, 100.34it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.92it/s]
 16%|█▋        | 8192/50176 [01:24<06:48, 102.72it/s]
 17%|█▋        | 8448/50176 [01:26<06:44, 103.17it/s]
 17%|█▋        | 8704/50176 [01:29<06:54, 100.03it/s]
 18%|█▊        | 8960/50176 [01:31<06:54, 99.52it/s] 
 18%|█▊        | 9216/50176 [01:34<06:47, 100.59it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.84it/s]
 19%|█▉        | 9728/50176 [01:39<06:47, 99.14it/s] 
 20%|█▉        | 9984/50176 [01:42<06:52, 97.54it/s]
 20%|██        | 10240/50176 [01:44<06:30, 102.22it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.50it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.12it/s] 
 22%|██▏       | 11008/50176 [01:52<06:34, 99.41it/s]
 22%|██▏       | 11264/50176 [01:54<06:21, 102.09it/s]
 23%|██▎       | 11520/50176 [01:57<06:21, 101.29it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.84it/s]
 24%|██▍       | 12032/50176 [02:02<06:15, 101.48it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.95it/s]
 25%|██▌       | 12544/50176 [02:07<06:09, 101.76it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.71it/s]
 26%|██▌       | 13056/50176 [02:12<06:00, 102.86it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.05it/s]
 27%|██▋       | 13568/50176 [02:17<06:11, 98.43it/s] 
 28%|██▊       | 13824/50176 [02:20<06:19, 95.73it/s]
 28%|██▊       | 14080/50176 [02:23<06:18, 95.24it/s]
 29%|██▊       | 14336/50176 [02:25<06:21, 94.03it/s]
 29%|██▉       | 14592/50176 [02:28<06:07, 96.90it/s]
 30%|██▉       | 14848/50176 [02:31<06:20, 92.92it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.29it/s]
 31%|███       | 15360/50176 [02:36<05:58, 97.12it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.80it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.07it/s]
 32%|███▏      | 16128/50176 [02:44<05:43, 99.21it/s]
 33%|███▎      | 16384/50176 [02:46<05:38, 99.96it/s]
 33%|███▎      | 16640/50176 [02:49<05:48, 96.31it/s]
 34%|███▎      | 16896/50176 [02:52<05:41, 97.40it/s]
 34%|███▍      | 17152/50176 [02:54<05:30, 99.94it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 101.00it/s]
 35%|███▌      | 17664/50176 [02:59<05:16, 102.61it/s]
 36%|███▌      | 17920/50176 [03:01<05:12, 103.34it/s]
 36%|███▌      | 18176/50176 [03:04<05:09, 103.46it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.36it/s]
 37%|███▋      | 18688/50176 [03:09<05:04, 103.42it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.70it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.22it/s] 
 39%|███▉      | 19456/50176 [03:17<05:20, 95.99it/s]
 39%|███▉      | 19712/50176 [03:20<05:18, 95.74it/s]
 40%|███▉      | 19968/50176 [03:22<05:10, 97.41it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.34it/s]
 41%|████      | 20480/50176 [03:27<04:44, 104.47it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.09it/s]
 42%|████▏     | 20992/50176 [03:31<04:32, 107.29it/s]
 42%|████▏     | 21248/50176 [03:34<04:35, 104.99it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.37it/s]
 43%|████▎     | 21760/50176 [03:39<04:26, 106.46it/s]
 44%|████▍     | 22016/50176 [03:41<04:20, 108.00it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.25it/s]
 45%|████▍     | 22528/50176 [03:46<04:15, 108.09it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.20it/s]
 46%|████▌     | 23040/50176 [03:51<04:25, 102.04it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.52it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.85it/s] 
 47%|████▋     | 23808/50176 [03:59<04:17, 102.39it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.50it/s]
 48%|████▊     | 24320/50176 [04:04<04:16, 100.92it/s]
 49%|████▉     | 24576/50176 [04:06<04:13, 101.14it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 104.97it/s]
 50%|█████     | 25088/50176 [04:11<03:55, 106.38it/s]
 51%|█████     | 25344/50176 [04:13<03:51, 107.44it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.18it/s]
 52%|█████▏    | 25856/50176 [04:18<03:45, 107.72it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.12it/s]
 53%|█████▎    | 26368/50176 [04:23<03:44, 105.96it/s]
 53%|█████▎    | 26624/50176 [04:25<03:43, 105.49it/s]
 54%|█████▎    | 26880/50176 [04:28<03:41, 105.16it/s]
 54%|█████▍    | 27136/50176 [04:30<03:40, 104.68it/s]
 55%|█████▍    | 27392/50176 [04:32<03:34, 106.46it/s]
 55%|█████▌    | 27648/50176 [04:35<03:25, 109.49it/s]
 56%|█████▌    | 27904/50176 [04:37<03:37, 102.21it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.07it/s]
 57%|█████▋    | 28416/50176 [04:43<03:34, 101.57it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 100.99it/s]
 58%|█████▊    | 28928/50176 [04:48<03:27, 102.20it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.29it/s]
 59%|█████▊    | 29440/50176 [04:53<03:30, 98.69it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:31, 96.89it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.80it/s]
 60%|██████    | 30208/50176 [05:01<03:22, 98.62it/s]
 61%|██████    | 30464/50176 [05:03<03:07, 105.29it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 103.04it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.73it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.19it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.33it/s]
 63%|██████▎   | 31744/50176 [05:15<02:49, 108.61it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.24it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.79it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.22it/s]
 65%|██████▌   | 32768/50176 [05:25<02:46, 104.76it/s]
 66%|██████▌   | 33024/50176 [05:27<02:45, 103.91it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.61it/s]
 67%|██████▋   | 33536/50176 [05:32<02:41, 103.08it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.72it/s]
 68%|██████▊   | 34048/50176 [05:37<02:30, 107.32it/s]
 68%|██████▊   | 34304/50176 [05:39<02:22, 111.59it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.48it/s]
 69%|██████▉   | 34816/50176 [05:44<02:25, 105.86it/s]
 70%|██████▉   | 35072/50176 [05:46<02:23, 105.52it/s]
 70%|███████   | 35328/50176 [05:49<02:20, 105.97it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.87it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.02it/s]
 72%|███████▏  | 36096/50176 [05:56<02:12, 106.39it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.65it/s]
 73%|███████▎  | 36608/50176 [06:01<02:11, 102.87it/s]
 73%|███████▎  | 36864/50176 [06:04<02:12, 100.19it/s]
 74%|███████▍  | 37120/50176 [06:06<02:07, 102.66it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.49it/s]
 75%|███████▌  | 37632/50176 [06:11<01:58, 105.83it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.21it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.34it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.68it/s]
 77%|███████▋  | 38656/50176 [06:20<01:48, 105.73it/s]
 78%|███████▊  | 38912/50176 [06:23<01:46, 105.37it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.26it/s]
 79%|███████▊  | 39424/50176 [06:28<01:41, 105.95it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.29it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.89it/s]
 80%|████████  | 40192/50176 [06:35<01:34, 106.02it/s]
 81%|████████  | 40448/50176 [06:38<01:34, 102.54it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.15it/s]
 82%|████████▏ | 40960/50176 [06:42<01:27, 104.75it/s]
 82%|████████▏ | 41216/50176 [06:45<01:24, 105.65it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 104.10it/s]
 83%|████████▎ | 41728/50176 [06:50<01:19, 106.80it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.95it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.80it/s]
 85%|████████▍ | 42496/50176 [06:57<01:10, 108.79it/s]
 85%|████████▌ | 42752/50176 [06:59<01:07, 109.65it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.12it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 107.38it/s]
 87%|████████▋ | 43520/50176 [07:07<01:04, 103.31it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 103.33it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 105.84it/s]
 88%|████████▊ | 44288/50176 [07:14<00:54, 107.80it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 103.90it/s]
 89%|████████▉ | 44800/50176 [07:19<00:51, 104.97it/s]
 90%|████████▉ | 45056/50176 [07:21<00:47, 107.93it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.13it/s]
 91%|█████████ | 45568/50176 [07:26<00:43, 106.74it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 104.94it/s]
 92%|█████████▏| 46080/50176 [07:31<00:39, 104.59it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.60it/s]
 93%|█████████▎| 46592/50176 [07:36<00:34, 105.03it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.32it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 101.16it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.11it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 100.90it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.71it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 102.52it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.59it/s]
 97%|█████████▋| 48640/50176 [07:56<00:14, 105.02it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 106.19it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.97it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 105.56it/s]
 99%|█████████▉| 49664/50176 [08:06<00:04, 102.68it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.17it/s] 
-2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6387	imagenet-zeroshot-val-top5: 0.8844

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_49.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,10:53:19 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,10:53:19 | INFO | Loading ViT-B-16 model config.
-2022-12-21,10:53:21 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt).
-2022-12-21,10:53:30 | INFO | Model:
-2022-12-21,10:53:30 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,10:53:30 | INFO | Params:
-2022-12-21,10:53:30 | INFO |   batch_size: 256
-2022-12-21,10:53:30 | INFO |   beta1: 0.9
-2022-12-21,10:53:30 | INFO |   beta2: 0.98
-2022-12-21,10:53:30 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/checkpoints
-2022-12-21,10:53:30 | INFO |   copy_codebase: False
-2022-12-21,10:53:30 | INFO |   csv_caption_key: title
-2022-12-21,10:53:30 | INFO |   csv_img_key: filepath
-2022-12-21,10:53:30 | INFO |   csv_separator: 	
-2022-12-21,10:53:30 | INFO |   dataset_resampled: False
-2022-12-21,10:53:30 | INFO |   dataset_type: auto
-2022-12-21,10:53:30 | INFO |   ddp_static_graph: False
-2022-12-21,10:53:30 | INFO |   debug: False
-2022-12-21,10:53:30 | INFO |   device: cuda:0
-2022-12-21,10:53:30 | INFO |   dist_backend: nccl
-2022-12-21,10:53:30 | INFO |   dist_url: env://
-2022-12-21,10:53:30 | INFO |   distributed: False
-2022-12-21,10:53:30 | INFO |   epochs: 32
-2022-12-21,10:53:30 | INFO |   eps: 1e-06
-2022-12-21,10:53:30 | INFO |   force_quick_gelu: False
-2022-12-21,10:53:30 | INFO |   gather_with_grad: False
-2022-12-21,10:53:30 | INFO |   grad_checkpointing: False
-2022-12-21,10:53:30 | INFO |   horovod: False
-2022-12-21,10:53:30 | INFO |   image_mean: None
-2022-12-21,10:53:30 | INFO |   image_std: None
-2022-12-21,10:53:30 | INFO |   imagenet_v2: None
-2022-12-21,10:53:30 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,10:53:30 | INFO |   local_loss: False
-2022-12-21,10:53:30 | INFO |   local_rank: 0
-2022-12-21,10:53:30 | INFO |   lock_image: False
-2022-12-21,10:53:30 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,10:53:30 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,10:53:30 | INFO |   log_level: 20
-2022-12-21,10:53:30 | INFO |   log_local: False
-2022-12-21,10:53:30 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_49.pt/out.log
-2022-12-21,10:53:30 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,10:53:30 | INFO |   lr: 0.0005
-2022-12-21,10:53:30 | INFO |   model: ViT-B-16
-2022-12-21,10:53:30 | INFO |   name: epoch_49.pt
-2022-12-21,10:53:30 | INFO |   no_set_device_rank: False
-2022-12-21,10:53:30 | INFO |   norm_gradient_clip: None
-2022-12-21,10:53:30 | INFO |   precision: amp_bfloat16
-2022-12-21,10:53:30 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_49.pt
-2022-12-21,10:53:30 | INFO |   pretrained_image: False
-2022-12-21,10:53:30 | INFO |   rank: 0
-2022-12-21,10:53:30 | INFO |   report_to: 
-2022-12-21,10:53:30 | INFO |   resume: None
-2022-12-21,10:53:30 | INFO |   save_frequency: 1
-2022-12-21,10:53:30 | INFO |   save_most_recent: False
-2022-12-21,10:53:30 | INFO |   seed: 0
-2022-12-21,10:53:30 | INFO |   skip_scheduler: False
-2022-12-21,10:53:30 | INFO |   tensorboard: False
-2022-12-21,10:53:30 | INFO |   tensorboard_path: 
-2022-12-21,10:53:30 | INFO |   torchscript: False
-2022-12-21,10:53:30 | INFO |   trace: False
-2022-12-21,10:53:30 | INFO |   train_data: None
-2022-12-21,10:53:30 | INFO |   train_num_samples: None
-2022-12-21,10:53:30 | INFO |   use_bn_sync: False
-2022-12-21,10:53:30 | INFO |   val_data: None
-2022-12-21,10:53:30 | INFO |   val_frequency: 1
-2022-12-21,10:53:30 | INFO |   val_num_samples: None
-2022-12-21,10:53:30 | INFO |   wandb: False
-2022-12-21,10:53:30 | INFO |   wandb_notes: 
-2022-12-21,10:53:30 | INFO |   warmup: 10000
-2022-12-21,10:53:30 | INFO |   wd: 0.2
-2022-12-21,10:53:30 | INFO |   workers: 1
-2022-12-21,10:53:30 | INFO |   world_size: 1
-2022-12-21,10:53:30 | INFO |   zeroshot_frequency: 2
-2022-12-21,10:53:30 | INFO | Starting zero-shot imagenet.
-2022-12-21,10:53:30 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:02,  1.51it/s]
  1%|          | 6/1000 [00:00<01:38, 10.09it/s]
  1%|          | 12/1000 [00:00<00:49, 19.90it/s]
  2%|▏         | 18/1000 [00:00<00:34, 28.55it/s]
  2%|▏         | 24/1000 [00:01<00:27, 35.72it/s]
  3%|▎         | 30/1000 [00:01<00:23, 41.45it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.80it/s]
  4%|▍         | 42/1000 [00:01<00:19, 49.06it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.48it/s]
  5%|▌         | 54/1000 [00:01<00:17, 53.13it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.42it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.38it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.86it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.31it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.84it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.13it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.53it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.72it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.98it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 58.11it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.98it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.94it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.99it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.96it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.98it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.05it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.08it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.96it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.83it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.85it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.82it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.74it/s]
 19%|█▉        | 192/1000 [00:03<00:14, 57.70it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.65it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.45it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.35it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.45it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.41it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.64it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.64it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.55it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.62it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.71it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.80it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.84it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.59it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.62it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.79it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.91it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.13it/s]
 30%|███       | 300/1000 [00:05<00:12, 58.12it/s]
 31%|███       | 306/1000 [00:05<00:11, 58.05it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.20it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.20it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.10it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.92it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 58.01it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.12it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.21it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.95it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.96it/s]
 37%|███▋      | 366/1000 [00:06<00:10, 58.05it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 58.14it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 58.02it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.71it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.78it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.90it/s]
 40%|████      | 402/1000 [00:07<00:10, 58.05it/s]
 41%|████      | 408/1000 [00:07<00:10, 58.13it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.13it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.16it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.18it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.28it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.26it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.11it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.16it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.13it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 58.09it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.23it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.13it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.11it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.12it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 58.02it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.07it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.06it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.20it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.15it/s]
 54%|█████▍    | 540/1000 [00:09<00:07, 58.21it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.22it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 58.11it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 58.09it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.16it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 58.05it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.99it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.13it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.09it/s]
 59%|█████▉    | 594/1000 [00:10<00:06, 58.01it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.13it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.04it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.04it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.15it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.12it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.21it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.97it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.12it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.07it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.15it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.19it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.13it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.16it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.15it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.20it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.08it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.18it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.16it/s]
 71%|███████▏  | 714/1000 [00:12<00:04, 58.14it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.00it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.98it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.04it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.04it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.15it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.07it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.18it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.14it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.18it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.22it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 58.01it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 58.01it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.06it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 58.03it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.05it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.16it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.29it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.32it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.23it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.26it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.20it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.20it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.15it/s]
 89%|████████▉ | 888/1000 [00:15<00:01, 58.04it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.15it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.15it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.10it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.13it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.09it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.08it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.19it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.24it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.23it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.25it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.29it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.29it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.37it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.38it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.33it/s]
-2022-12-21,10:53:48 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:02, 82.86it/s]
  1%|          | 512/50176 [00:05<09:18, 88.91it/s]
  2%|▏         | 768/50176 [00:08<08:37, 95.49it/s]
  2%|▏         | 1024/50176 [00:10<08:29, 96.41it/s]
  3%|▎         | 1280/50176 [00:13<08:37, 94.40it/s]
  3%|▎         | 1536/50176 [00:16<09:12, 88.07it/s]
  4%|▎         | 1792/50176 [00:19<08:50, 91.27it/s]
  4%|▍         | 2048/50176 [00:22<08:25, 95.19it/s]
  5%|▍         | 2304/50176 [00:24<08:07, 98.21it/s]
  5%|▌         | 2560/50176 [00:26<07:57, 99.65it/s]
  6%|▌         | 2816/50176 [00:29<08:18, 94.97it/s]
  6%|▌         | 3072/50176 [00:32<08:05, 97.01it/s]
  7%|▋         | 3328/50176 [00:34<07:47, 100.24it/s]
  7%|▋         | 3584/50176 [00:37<07:43, 100.62it/s]
  8%|▊         | 3840/50176 [00:40<07:53, 97.89it/s] 
  8%|▊         | 4096/50176 [00:42<08:04, 95.11it/s]
  9%|▊         | 4352/50176 [00:45<07:50, 97.47it/s]
  9%|▉         | 4608/50176 [00:47<07:34, 100.22it/s]
 10%|▉         | 4864/50176 [00:50<07:32, 100.07it/s]
 10%|█         | 5120/50176 [00:52<07:21, 101.98it/s]
 11%|█         | 5376/50176 [00:55<07:23, 100.96it/s]
 11%|█         | 5632/50176 [00:58<07:32, 98.53it/s] 
 12%|█▏        | 5888/50176 [01:00<07:26, 99.22it/s]
 12%|█▏        | 6144/50176 [01:03<07:56, 92.31it/s]
 13%|█▎        | 6400/50176 [01:06<07:41, 94.95it/s]
 13%|█▎        | 6656/50176 [01:08<07:22, 98.34it/s]
 14%|█▍        | 6912/50176 [01:11<07:21, 97.91it/s]
 14%|█▍        | 7168/50176 [01:13<07:16, 98.60it/s]
 15%|█▍        | 7424/50176 [01:16<07:13, 98.68it/s]
 15%|█▌        | 7680/50176 [01:19<07:03, 100.45it/s]
 16%|█▌        | 7936/50176 [01:21<06:50, 102.84it/s]
 16%|█▋        | 8192/50176 [01:23<06:48, 102.69it/s]
 17%|█▋        | 8448/50176 [01:26<06:44, 103.14it/s]
 17%|█▋        | 8704/50176 [01:29<06:54, 100.00it/s]
 18%|█▊        | 8960/50176 [01:31<06:54, 99.49it/s] 
 18%|█▊        | 9216/50176 [01:34<06:46, 100.67it/s]
 19%|█▉        | 9472/50176 [01:36<06:43, 100.85it/s]
 19%|█▉        | 9728/50176 [01:39<06:48, 99.09it/s] 
 20%|█▉        | 9984/50176 [01:42<06:51, 97.76it/s]
 20%|██        | 10240/50176 [01:44<06:30, 102.19it/s]
 21%|██        | 10496/50176 [01:46<06:23, 103.44it/s]
 21%|██▏       | 10752/50176 [01:49<06:37, 99.10it/s] 
 22%|██▏       | 11008/50176 [01:52<06:33, 99.49it/s]
 22%|██▏       | 11264/50176 [01:54<06:21, 102.01it/s]
 23%|██▎       | 11520/50176 [01:57<06:21, 101.28it/s]
 23%|██▎       | 11776/50176 [01:59<06:13, 102.80it/s]
 24%|██▍       | 12032/50176 [02:02<06:16, 101.45it/s]
 24%|██▍       | 12288/50176 [02:04<06:15, 100.93it/s]
 25%|██▌       | 12544/50176 [02:07<06:09, 101.75it/s]
 26%|██▌       | 12800/50176 [02:09<06:07, 101.69it/s]
 26%|██▌       | 13056/50176 [02:11<06:00, 102.85it/s]
 27%|██▋       | 13312/50176 [02:14<06:01, 102.09it/s]
 27%|██▋       | 13568/50176 [02:17<06:11, 98.48it/s] 
 28%|██▊       | 13824/50176 [02:20<06:19, 95.72it/s]
 28%|██▊       | 14080/50176 [02:22<06:19, 95.22it/s]
 29%|██▊       | 14336/50176 [02:25<06:21, 94.02it/s]
 29%|██▉       | 14592/50176 [02:28<06:07, 96.91it/s]
 30%|██▉       | 14848/50176 [02:31<06:20, 92.92it/s]
 30%|███       | 15104/50176 [02:33<06:11, 94.29it/s]
 31%|███       | 15360/50176 [02:36<05:58, 97.21it/s]
 31%|███       | 15616/50176 [02:38<05:49, 98.79it/s]
 32%|███▏      | 15872/50176 [02:41<05:49, 98.07it/s]
 32%|███▏      | 16128/50176 [02:43<05:43, 99.21it/s]
 33%|███▎      | 16384/50176 [02:46<05:38, 99.91it/s]
 33%|███▎      | 16640/50176 [02:49<05:48, 96.28it/s]
 34%|███▎      | 16896/50176 [02:51<05:41, 97.40it/s]
 34%|███▍      | 17152/50176 [02:54<05:30, 99.92it/s]
 35%|███▍      | 17408/50176 [02:56<05:24, 100.99it/s]
 35%|███▌      | 17664/50176 [02:59<05:17, 102.46it/s]
 36%|███▌      | 17920/50176 [03:01<05:11, 103.42it/s]
 36%|███▌      | 18176/50176 [03:04<05:09, 103.50it/s]
 37%|███▋      | 18432/50176 [03:06<05:01, 105.38it/s]
 37%|███▋      | 18688/50176 [03:08<05:03, 103.71it/s]
 38%|███▊      | 18944/50176 [03:11<05:04, 102.63it/s]
 38%|███▊      | 19200/50176 [03:14<05:15, 98.18it/s] 
 39%|███▉      | 19456/50176 [03:17<05:20, 95.96it/s]
 39%|███▉      | 19712/50176 [03:19<05:18, 95.72it/s]
 40%|███▉      | 19968/50176 [03:22<05:10, 97.41it/s]
 40%|████      | 20224/50176 [03:24<04:52, 102.32it/s]
 41%|████      | 20480/50176 [03:26<04:44, 104.49it/s]
 41%|████▏     | 20736/50176 [03:29<04:42, 104.09it/s]
 42%|████▏     | 20992/50176 [03:31<04:31, 107.30it/s]
 42%|████▏     | 21248/50176 [03:34<04:35, 104.98it/s]
 43%|████▎     | 21504/50176 [03:36<04:34, 104.37it/s]
 43%|████▎     | 21760/50176 [03:38<04:26, 106.46it/s]
 44%|████▍     | 22016/50176 [03:41<04:20, 108.00it/s]
 44%|████▍     | 22272/50176 [03:43<04:15, 109.25it/s]
 45%|████▍     | 22528/50176 [03:45<04:15, 108.09it/s]
 45%|████▌     | 22784/50176 [03:48<04:22, 104.20it/s]
 46%|████▌     | 23040/50176 [03:51<04:25, 102.02it/s]
 46%|████▋     | 23296/50176 [03:53<04:24, 101.55it/s]
 47%|████▋     | 23552/50176 [03:56<04:26, 99.86it/s] 
 47%|████▋     | 23808/50176 [03:58<04:17, 102.37it/s]
 48%|████▊     | 24064/50176 [04:01<04:14, 102.52it/s]
 48%|████▊     | 24320/50176 [04:03<04:16, 100.92it/s]
 49%|████▉     | 24576/50176 [04:06<04:12, 101.20it/s]
 49%|████▉     | 24832/50176 [04:08<04:01, 104.96it/s]
 50%|█████     | 25088/50176 [04:11<03:56, 106.29it/s]
 51%|█████     | 25344/50176 [04:13<03:51, 107.39it/s]
 51%|█████     | 25600/50176 [04:15<03:51, 106.21it/s]
 52%|█████▏    | 25856/50176 [04:18<03:45, 107.66it/s]
 52%|█████▏    | 26112/50176 [04:20<03:42, 108.16it/s]
 53%|█████▎    | 26368/50176 [04:22<03:44, 106.02it/s]
 53%|█████▎    | 26624/50176 [04:25<03:43, 105.52it/s]
 54%|█████▎    | 26880/50176 [04:27<03:41, 105.15it/s]
 54%|█████▍    | 27136/50176 [04:30<03:40, 104.67it/s]
 55%|█████▍    | 27392/50176 [04:32<03:33, 106.55it/s]
 55%|█████▌    | 27648/50176 [04:34<03:25, 109.67it/s]
 56%|█████▌    | 27904/50176 [04:37<03:38, 102.15it/s]
 56%|█████▌    | 28160/50176 [04:40<03:40, 100.01it/s]
 57%|█████▋    | 28416/50176 [04:42<03:34, 101.55it/s]
 57%|█████▋    | 28672/50176 [04:45<03:32, 100.98it/s]
 58%|█████▊    | 28928/50176 [04:47<03:27, 102.18it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.27it/s]
 59%|█████▊    | 29440/50176 [04:53<03:30, 98.67it/s] 
 59%|█████▉    | 29696/50176 [04:55<03:31, 96.89it/s]
 60%|█████▉    | 29952/50176 [04:58<03:26, 97.79it/s]
 60%|██████    | 30208/50176 [05:01<03:22, 98.62it/s]
 61%|██████    | 30464/50176 [05:03<03:07, 105.24it/s]
 61%|██████    | 30720/50176 [05:05<03:08, 103.02it/s]
 62%|██████▏   | 30976/50176 [05:08<03:03, 104.74it/s]
 62%|██████▏   | 31232/50176 [05:10<02:58, 106.11it/s]
 63%|██████▎   | 31488/50176 [05:12<02:52, 108.48it/s]
 63%|██████▎   | 31744/50176 [05:14<02:49, 108.54it/s]
 64%|██████▍   | 32000/50176 [05:17<02:47, 108.24it/s]
 64%|██████▍   | 32256/50176 [05:19<02:46, 107.81it/s]
 65%|██████▍   | 32512/50176 [05:22<02:52, 102.20it/s]
 65%|██████▌   | 32768/50176 [05:24<02:46, 104.69it/s]
 66%|██████▌   | 33024/50176 [05:27<02:44, 104.04it/s]
 66%|██████▋   | 33280/50176 [05:29<02:41, 104.65it/s]
 67%|██████▋   | 33536/50176 [05:32<02:41, 102.97it/s]
 67%|██████▋   | 33792/50176 [05:34<02:33, 106.65it/s]
 68%|██████▊   | 34048/50176 [05:36<02:30, 107.32it/s]
 68%|██████▊   | 34304/50176 [05:38<02:22, 111.47it/s]
 69%|██████▉   | 34560/50176 [05:41<02:26, 106.47it/s]
 69%|██████▉   | 34816/50176 [05:44<02:24, 105.99it/s]
 70%|██████▉   | 35072/50176 [05:46<02:22, 105.67it/s]
 70%|███████   | 35328/50176 [05:48<02:20, 106.03it/s]
 71%|███████   | 35584/50176 [05:51<02:16, 106.77it/s]
 71%|███████▏  | 35840/50176 [05:53<02:16, 105.15it/s]
 72%|███████▏  | 36096/50176 [05:56<02:12, 106.22it/s]
 72%|███████▏  | 36352/50176 [05:58<02:13, 103.58it/s]
 73%|███████▎  | 36608/50176 [06:01<02:11, 102.89it/s]
 73%|███████▎  | 36864/50176 [06:03<02:12, 100.31it/s]
 74%|███████▍  | 37120/50176 [06:06<02:07, 102.69it/s]
 74%|███████▍  | 37376/50176 [06:08<02:00, 106.46it/s]
 75%|███████▌  | 37632/50176 [06:10<01:58, 105.84it/s]
 76%|███████▌  | 37888/50176 [06:13<01:56, 105.13it/s]
 76%|███████▌  | 38144/50176 [06:15<01:53, 106.35it/s]
 77%|███████▋  | 38400/50176 [06:18<01:53, 103.76it/s]
 77%|███████▋  | 38656/50176 [06:20<01:49, 105.63it/s]
 78%|███████▊  | 38912/50176 [06:23<01:47, 105.27it/s]
 78%|███████▊  | 39168/50176 [06:25<01:46, 103.26it/s]
 79%|███████▊  | 39424/50176 [06:28<01:41, 105.98it/s]
 79%|███████▉  | 39680/50176 [06:30<01:39, 105.45it/s]
 80%|███████▉  | 39936/50176 [06:32<01:35, 106.84it/s]
 80%|████████  | 40192/50176 [06:35<01:34, 105.88it/s]
 81%|████████  | 40448/50176 [06:37<01:34, 102.69it/s]
 81%|████████  | 40704/50176 [06:40<01:33, 101.23it/s]
 82%|████████▏ | 40960/50176 [06:42<01:27, 104.82it/s]
 82%|████████▏ | 41216/50176 [06:45<01:24, 105.60it/s]
 83%|████████▎ | 41472/50176 [06:47<01:23, 104.00it/s]
 83%|████████▎ | 41728/50176 [06:49<01:19, 106.69it/s]
 84%|████████▎ | 41984/50176 [06:52<01:16, 106.96it/s]
 84%|████████▍ | 42240/50176 [06:54<01:13, 107.91it/s]
 85%|████████▍ | 42496/50176 [06:57<01:10, 108.77it/s]
 85%|████████▌ | 42752/50176 [06:59<01:07, 109.58it/s]
 86%|████████▌ | 43008/50176 [07:01<01:06, 108.19it/s]
 86%|████████▌ | 43264/50176 [07:04<01:04, 107.43it/s]
 87%|████████▋ | 43520/50176 [07:06<01:04, 103.29it/s]
 87%|████████▋ | 43776/50176 [07:09<01:01, 103.26it/s]
 88%|████████▊ | 44032/50176 [07:11<00:58, 105.85it/s]
 88%|████████▊ | 44288/50176 [07:13<00:54, 107.91it/s]
 89%|████████▉ | 44544/50176 [07:16<00:54, 103.91it/s]
 89%|████████▉ | 44800/50176 [07:18<00:51, 104.79it/s]
 90%|████████▉ | 45056/50176 [07:21<00:47, 107.80it/s]
 90%|█████████ | 45312/50176 [07:23<00:46, 104.19it/s]
 91%|█████████ | 45568/50176 [07:26<00:43, 106.76it/s]
 91%|█████████▏| 45824/50176 [07:28<00:41, 105.09it/s]
 92%|█████████▏| 46080/50176 [07:31<00:39, 104.90it/s]
 92%|█████████▏| 46336/50176 [07:33<00:37, 102.51it/s]
 93%|█████████▎| 46592/50176 [07:35<00:34, 104.83it/s]
 93%|█████████▎| 46848/50176 [07:38<00:31, 104.32it/s]
 94%|█████████▍| 47104/50176 [07:41<00:30, 101.15it/s]
 94%|█████████▍| 47360/50176 [07:43<00:27, 101.17it/s]
 95%|█████████▍| 47616/50176 [07:46<00:25, 100.77it/s]
 95%|█████████▌| 47872/50176 [07:48<00:22, 102.71it/s]
 96%|█████████▌| 48128/50176 [07:51<00:19, 102.57it/s]
 96%|█████████▋| 48384/50176 [07:53<00:17, 103.60it/s]
 97%|█████████▋| 48640/50176 [07:55<00:14, 104.95it/s]
 97%|█████████▋| 48896/50176 [07:58<00:12, 106.19it/s]
 98%|█████████▊| 49152/50176 [08:00<00:09, 104.91it/s]
 98%|█████████▊| 49408/50176 [08:03<00:07, 105.56it/s]
 99%|█████████▉| 49664/50176 [08:05<00:04, 102.61it/s]
 99%|█████████▉| 49920/50176 [08:08<00:02, 97.19it/s] 
-2022-12-21,11:01:57 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:01:57 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6437	imagenet-zeroshot-val-top5: 0.8902

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_5.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
-2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt).
-2022-12-21,11:02:51 | INFO | Model:
-2022-12-21,11:02:51 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,11:02:51 | INFO | Params:
-2022-12-21,11:02:51 | INFO |   batch_size: 256
-2022-12-21,11:02:51 | INFO |   beta1: 0.9
-2022-12-21,11:02:51 | INFO |   beta2: 0.98
-2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/checkpoints
-2022-12-21,11:02:51 | INFO |   copy_codebase: False
-2022-12-21,11:02:51 | INFO |   csv_caption_key: title
-2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
-2022-12-21,11:02:51 | INFO |   csv_separator: 	
-2022-12-21,11:02:51 | INFO |   dataset_resampled: False
-2022-12-21,11:02:51 | INFO |   dataset_type: auto
-2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
-2022-12-21,11:02:51 | INFO |   debug: False
-2022-12-21,11:02:51 | INFO |   device: cuda:0
-2022-12-21,11:02:51 | INFO |   dist_backend: nccl
-2022-12-21,11:02:51 | INFO |   dist_url: env://
-2022-12-21,11:02:51 | INFO |   distributed: False
-2022-12-21,11:02:51 | INFO |   epochs: 32
-2022-12-21,11:02:51 | INFO |   eps: 1e-06
-2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
-2022-12-21,11:02:51 | INFO |   gather_with_grad: False
-2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
-2022-12-21,11:02:51 | INFO |   horovod: False
-2022-12-21,11:02:51 | INFO |   image_mean: None
-2022-12-21,11:02:51 | INFO |   image_std: None
-2022-12-21,11:02:51 | INFO |   imagenet_v2: None
-2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,11:02:51 | INFO |   local_loss: False
-2022-12-21,11:02:51 | INFO |   local_rank: 0
-2022-12-21,11:02:51 | INFO |   lock_image: False
-2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,11:02:51 | INFO |   log_level: 20
-2022-12-21,11:02:51 | INFO |   log_local: False
-2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_5.pt/out.log
-2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,11:02:51 | INFO |   lr: 0.0005
-2022-12-21,11:02:51 | INFO |   model: ViT-B-16
-2022-12-21,11:02:51 | INFO |   name: epoch_5.pt
-2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
-2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
-2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
-2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_5.pt
-2022-12-21,11:02:51 | INFO |   pretrained_image: False
-2022-12-21,11:02:51 | INFO |   rank: 0
-2022-12-21,11:02:51 | INFO |   report_to: 
-2022-12-21,11:02:51 | INFO |   resume: None
-2022-12-21,11:02:51 | INFO |   save_frequency: 1
-2022-12-21,11:02:51 | INFO |   save_most_recent: False
-2022-12-21,11:02:51 | INFO |   seed: 0
-2022-12-21,11:02:51 | INFO |   skip_scheduler: False
-2022-12-21,11:02:51 | INFO |   tensorboard: False
-2022-12-21,11:02:51 | INFO |   tensorboard_path: 
-2022-12-21,11:02:51 | INFO |   torchscript: False
-2022-12-21,11:02:51 | INFO |   trace: False
-2022-12-21,11:02:51 | INFO |   train_data: None
-2022-12-21,11:02:51 | INFO |   train_num_samples: None
-2022-12-21,11:02:51 | INFO |   use_bn_sync: False
-2022-12-21,11:02:51 | INFO |   val_data: None
-2022-12-21,11:02:51 | INFO |   val_frequency: 1
-2022-12-21,11:02:51 | INFO |   val_num_samples: None
-2022-12-21,11:02:51 | INFO |   wandb: False
-2022-12-21,11:02:51 | INFO |   wandb_notes: 
-2022-12-21,11:02:51 | INFO |   warmup: 10000
-2022-12-21,11:02:51 | INFO |   wd: 0.2
-2022-12-21,11:02:51 | INFO |   workers: 1
-2022-12-21,11:02:51 | INFO |   world_size: 1
-2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
-2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
-2022-12-21,11:02:51 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:02,  1.51it/s]
  1%|          | 6/1000 [00:00<01:38, 10.05it/s]
  1%|          | 12/1000 [00:00<00:50, 19.40it/s]
  2%|▏         | 18/1000 [00:00<00:35, 27.70it/s]
  2%|▏         | 24/1000 [00:01<00:27, 34.92it/s]
  3%|▎         | 30/1000 [00:01<00:23, 40.77it/s]
  4%|▎         | 36/1000 [00:01<00:21, 45.23it/s]
  4%|▍         | 42/1000 [00:01<00:19, 48.61it/s]
  5%|▍         | 48/1000 [00:01<00:18, 51.10it/s]
  5%|▌         | 54/1000 [00:01<00:17, 52.92it/s]
  6%|▌         | 60/1000 [00:01<00:17, 54.25it/s]
  7%|▋         | 66/1000 [00:01<00:16, 55.24it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.91it/s]
  8%|▊         | 78/1000 [00:02<00:16, 56.37it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.88it/s]
  9%|▉         | 90/1000 [00:02<00:15, 57.17it/s]
 10%|▉         | 96/1000 [00:02<00:15, 57.55it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.67it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.74it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.87it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.88it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.83it/s]
 13%|█▎        | 132/1000 [00:02<00:14, 57.87it/s]
 14%|█▍        | 138/1000 [00:03<00:14, 57.86it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.91it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 58.00it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 58.04it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.89it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.75it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.73it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.72it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.61it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.59it/s]
 20%|█▉        | 198/1000 [00:04<00:13, 57.55it/s]
 20%|██        | 204/1000 [00:04<00:13, 57.37it/s]
 21%|██        | 210/1000 [00:04<00:13, 57.27it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 57.38it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 57.36it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.57it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.57it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 57.48it/s]
 25%|██▍       | 246/1000 [00:04<00:13, 57.55it/s]
 25%|██▌       | 252/1000 [00:05<00:12, 57.64it/s]
 26%|██▌       | 258/1000 [00:05<00:12, 57.74it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.77it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 57.57it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.58it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.68it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.79it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 58.04it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.97it/s]
 31%|███       | 306/1000 [00:05<00:11, 57.97it/s]
 31%|███       | 312/1000 [00:06<00:11, 58.14it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 58.13it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 58.05it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.88it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.94it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 58.05it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 58.14it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.93it/s]
 36%|███▌      | 360/1000 [00:06<00:11, 57.95it/s]
 37%|███▋      | 366/1000 [00:07<00:10, 57.65it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.86it/s]
 38%|███▊      | 378/1000 [00:07<00:10, 57.83it/s]
 38%|███▊      | 384/1000 [00:07<00:10, 57.57it/s]
 39%|███▉      | 390/1000 [00:07<00:10, 57.66it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 57.81it/s]
 40%|████      | 402/1000 [00:07<00:10, 57.88it/s]
 41%|████      | 408/1000 [00:07<00:10, 57.98it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 58.00it/s]
 42%|████▏     | 420/1000 [00:07<00:09, 58.01it/s]
 43%|████▎     | 426/1000 [00:08<00:09, 58.05it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 58.18it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.95it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 58.00it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 58.04it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 58.01it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.97it/s]
 47%|████▋     | 474/1000 [00:08<00:09, 58.13it/s]
 48%|████▊     | 480/1000 [00:08<00:08, 58.04it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 58.02it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 58.02it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.95it/s]
 50%|█████     | 504/1000 [00:09<00:08, 58.01it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.93it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 58.01it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 58.04it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 58.11it/s]
 53%|█████▎    | 534/1000 [00:09<00:08, 58.05it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 58.18it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 58.00it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.93it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.97it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 58.05it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.93it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.89it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 58.04it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 58.03it/s]
 59%|█████▉    | 594/1000 [00:10<00:07, 57.97it/s]
 60%|██████    | 600/1000 [00:11<00:06, 58.10it/s]
 61%|██████    | 606/1000 [00:11<00:06, 58.01it/s]
 61%|██████    | 612/1000 [00:11<00:06, 58.00it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 58.13it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 58.12it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 58.20it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 58.03it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.98it/s]
 65%|██████▍   | 648/1000 [00:11<00:06, 58.12it/s]
 65%|██████▌   | 654/1000 [00:11<00:05, 58.07it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 58.17it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 58.21it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 58.15it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 58.21it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 58.22it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 58.22it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 58.10it/s]
 70%|███████   | 702/1000 [00:12<00:05, 58.19it/s]
 71%|███████   | 708/1000 [00:12<00:05, 58.17it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 58.14it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 58.15it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 58.10it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 58.10it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 58.05it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 58.04it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 58.14it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 58.23it/s]
 76%|███████▌  | 762/1000 [00:13<00:04, 58.05it/s]
 77%|███████▋  | 768/1000 [00:13<00:03, 58.16it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 58.12it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 58.16it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 58.19it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.99it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.99it/s]
 80%|████████  | 804/1000 [00:14<00:03, 58.05it/s]
 81%|████████  | 810/1000 [00:14<00:03, 58.02it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.99it/s]
 82%|████████▏ | 822/1000 [00:14<00:03, 58.02it/s]
 83%|████████▎ | 828/1000 [00:14<00:02, 58.13it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 58.28it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 58.29it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 58.20it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 58.25it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 58.20it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 58.18it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 58.12it/s]
 88%|████████▊ | 876/1000 [00:15<00:02, 58.17it/s]
 88%|████████▊ | 882/1000 [00:15<00:02, 58.16it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 58.02it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 58.13it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 58.12it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 58.09it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 58.11it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 58.12it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 58.11it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 58.21it/s]
 94%|█████████▎| 936/1000 [00:16<00:01, 58.25it/s]
 94%|█████████▍| 942/1000 [00:16<00:00, 58.25it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 58.27it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 58.30it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 58.16it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 58.17it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 58.28it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 58.33it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 58.06it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 58.02it/s]
-2022-12-21,11:03:09 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:39, 78.07it/s]
  1%|          | 512/50176 [00:05<09:15, 89.35it/s]
  2%|▏         | 768/50176 [00:08<08:26, 97.64it/s]
  2%|▏         | 1024/50176 [00:10<08:23, 97.71it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.54it/s]
  3%|▎         | 1536/50176 [00:16<08:53, 91.20it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.21it/s]
  4%|▍         | 2048/50176 [00:21<08:19, 96.27it/s]
  5%|▍         | 2304/50176 [00:24<08:08, 98.00it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.64it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.16it/s] 
  6%|▌         | 3072/50176 [00:31<07:53, 99.50it/s]
  7%|▋         | 3328/50176 [00:34<07:40, 101.81it/s]
  7%|▋         | 3584/50176 [00:36<07:30, 103.46it/s]
  8%|▊         | 3840/50176 [00:39<07:38, 101.12it/s]
  8%|▊         | 4096/50176 [00:42<07:54, 97.17it/s] 
  9%|▊         | 4352/50176 [00:44<07:49, 97.54it/s]
  9%|▉         | 4608/50176 [00:47<07:32, 100.70it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 102.01it/s]
 10%|█         | 5120/50176 [00:51<07:21, 102.14it/s]
 11%|█         | 5376/50176 [00:54<07:26, 100.40it/s]
 11%|█         | 5632/50176 [00:57<07:18, 101.69it/s]
 12%|█▏        | 5888/50176 [00:59<07:20, 100.47it/s]
 12%|█▏        | 6144/50176 [01:03<08:01, 91.53it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.95it/s]
 13%|█▎        | 6656/50176 [01:08<07:26, 97.39it/s]
 14%|█▍        | 6912/50176 [01:10<07:34, 95.15it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.18it/s]
 15%|█▍        | 7424/50176 [01:15<07:03, 100.97it/s]
 15%|█▌        | 7680/50176 [01:18<06:57, 101.77it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.14it/s]
 16%|█▋        | 8192/50176 [01:23<06:50, 102.32it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 103.00it/s]
 17%|█▋        | 8704/50176 [01:28<06:47, 101.83it/s]
 18%|█▊        | 8960/50176 [01:30<06:58, 98.48it/s] 
 18%|█▊        | 9216/50176 [01:33<06:39, 102.66it/s]
 19%|█▉        | 9472/50176 [01:35<06:34, 103.08it/s]
 19%|█▉        | 9728/50176 [01:38<06:43, 100.31it/s]
 20%|█▉        | 9984/50176 [01:40<06:41, 100.12it/s]
 20%|██        | 10240/50176 [01:43<06:27, 103.04it/s]
 21%|██        | 10496/50176 [01:45<06:27, 102.48it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.22it/s]
 22%|██▏       | 11008/50176 [01:50<06:26, 101.30it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.80it/s]
 23%|██▎       | 11520/50176 [01:55<06:12, 103.77it/s]
 23%|██▎       | 11776/50176 [01:58<06:10, 103.54it/s]
 24%|██▍       | 12032/50176 [02:00<06:14, 101.74it/s]
 24%|██▍       | 12288/50176 [02:03<06:08, 102.71it/s]
 25%|██▌       | 12544/50176 [02:05<06:14, 100.61it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 98.88it/s] 
 26%|██▌       | 13056/50176 [02:10<06:03, 102.12it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.80it/s]
 27%|██▋       | 13568/50176 [02:16<06:23, 95.50it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.55it/s]
 28%|██▊       | 14080/50176 [02:22<06:25, 93.73it/s]
 29%|██▊       | 14336/50176 [02:24<06:19, 94.36it/s]
 29%|██▉       | 14592/50176 [02:27<06:01, 98.34it/s]
 30%|██▉       | 14848/50176 [02:29<06:06, 96.39it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.47it/s]
 31%|███       | 15360/50176 [02:35<05:55, 97.89it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.74it/s]
 32%|███▏      | 15872/50176 [02:40<05:47, 98.80it/s]
 32%|███▏      | 16128/50176 [02:42<05:39, 100.32it/s]
 33%|███▎      | 16384/50176 [02:45<05:31, 101.79it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.81it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.90it/s]
 34%|███▍      | 17152/50176 [02:52<05:25, 101.52it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.31it/s]
 35%|███▌      | 17664/50176 [02:57<05:16, 102.70it/s]
 36%|███▌      | 17920/50176 [03:00<05:09, 104.13it/s]
 36%|███▌      | 18176/50176 [03:02<05:13, 102.08it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.90it/s]
 37%|███▋      | 18688/50176 [03:07<05:15, 99.70it/s] 
 38%|███▊      | 18944/50176 [03:10<05:16, 98.67it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.77it/s]
 39%|███▉      | 19456/50176 [03:16<05:18, 96.58it/s]
 39%|███▉      | 19712/50176 [03:18<05:19, 95.41it/s]
 40%|███▉      | 19968/50176 [03:21<05:13, 96.42it/s]
 40%|████      | 20224/50176 [03:23<04:53, 102.17it/s]
 41%|████      | 20480/50176 [03:26<04:49, 102.61it/s]
 41%|████▏     | 20736/50176 [03:28<04:43, 103.93it/s]
 42%|████▏     | 20992/50176 [03:30<04:36, 105.66it/s]
 42%|████▏     | 21248/50176 [03:33<04:40, 103.29it/s]
 43%|████▎     | 21504/50176 [03:35<04:35, 104.03it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.63it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.46it/s]
 44%|████▍     | 22272/50176 [03:42<04:19, 107.38it/s]
 45%|████▍     | 22528/50176 [03:45<04:18, 106.81it/s]
 45%|████▌     | 22784/50176 [03:47<04:23, 103.82it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.58it/s]
 46%|████▋     | 23296/50176 [03:53<04:29, 99.80it/s] 
 47%|████▋     | 23552/50176 [03:56<04:35, 96.81it/s]
 47%|████▋     | 23808/50176 [03:58<04:19, 101.56it/s]
 48%|████▊     | 24064/50176 [04:00<04:19, 100.49it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.03it/s]
 49%|████▉     | 24576/50176 [04:06<04:17, 99.38it/s] 
 49%|████▉     | 24832/50176 [04:08<04:08, 101.95it/s]
 50%|█████     | 25088/50176 [04:10<04:00, 104.21it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.45it/s]
 51%|█████     | 25600/50176 [04:15<03:57, 103.37it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.23it/s]
 52%|█████▏    | 26112/50176 [04:20<03:48, 105.21it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.37it/s]
 53%|█████▎    | 26624/50176 [04:26<03:54, 100.52it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.22it/s]
 54%|█████▍    | 27136/50176 [04:31<03:49, 100.24it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 101.97it/s]
 55%|█████▌    | 27648/50176 [04:35<03:33, 105.38it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.45it/s]
 56%|█████▌    | 28160/50176 [04:41<03:41, 99.18it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:38, 99.80it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.16it/s]
 58%|█████▊    | 28928/50176 [04:48<03:29, 101.57it/s]
 58%|█████▊    | 29184/50176 [04:51<03:23, 103.08it/s]
 59%|█████▊    | 29440/50176 [04:54<03:33, 97.00it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:30, 97.22it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.46it/s]
 60%|██████    | 30208/50176 [05:01<03:23, 98.21it/s]
 61%|██████    | 30464/50176 [05:04<03:12, 102.46it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.81it/s]
 62%|██████▏   | 30976/50176 [05:09<03:05, 103.30it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.13it/s]
 63%|██████▎   | 31488/50176 [05:13<02:55, 106.32it/s]
 63%|██████▎   | 31744/50176 [05:16<02:54, 105.35it/s]
 64%|██████▍   | 32000/50176 [05:18<02:52, 105.39it/s]
 64%|██████▍   | 32256/50176 [05:21<02:49, 105.48it/s]
 65%|██████▍   | 32512/50176 [05:24<02:56, 100.30it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.65it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 101.76it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.70it/s]
 67%|██████▋   | 33536/50176 [05:34<02:46, 100.02it/s]
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.12it/s]
 68%|██████▊   | 34048/50176 [05:38<02:33, 104.87it/s]
 68%|██████▊   | 34304/50176 [05:41<02:27, 107.25it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.84it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 106.06it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.71it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.44it/s]
 71%|███████   | 35584/50176 [05:53<02:16, 106.65it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.47it/s]
 72%|███████▏  | 36096/50176 [05:58<02:14, 104.35it/s]
 72%|███████▏  | 36352/50176 [06:00<02:13, 103.63it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.07it/s]
 73%|███████▎  | 36864/50176 [06:06<02:13, 99.94it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.40it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.19it/s]
 75%|███████▌  | 37632/50176 [06:12<01:56, 107.49it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.68it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 107.06it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.77it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.38it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.05it/s]
 78%|███████▊  | 39168/50176 [06:28<01:49, 100.45it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.85it/s]
 79%|███████▉  | 39680/50176 [06:32<01:42, 101.96it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.20it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.76it/s]
 81%|████████  | 40448/50176 [06:40<01:39, 98.22it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.35it/s]
 82%|████████▏ | 40960/50176 [06:45<01:32, 99.73it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.10it/s]
 83%|████████▎ | 41472/50176 [06:50<01:26, 100.81it/s]
 83%|████████▎ | 41728/50176 [06:53<01:22, 102.93it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.25it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.53it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.26it/s]
 85%|████████▌ | 42752/50176 [07:02<01:09, 107.04it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.98it/s]
 86%|████████▌ | 43264/50176 [07:07<01:05, 105.29it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.30it/s]
 87%|████████▋ | 43776/50176 [07:12<01:02, 102.27it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.84it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.36it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.39it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:25<00:48, 105.97it/s]
 90%|█████████ | 45312/50176 [07:27<00:46, 104.94it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 107.05it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.61it/s]
 92%|█████████▏| 46080/50176 [07:35<00:39, 103.11it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.65it/s]
 93%|█████████▎| 46592/50176 [07:39<00:33, 105.45it/s]
 93%|█████████▎| 46848/50176 [07:42<00:31, 106.76it/s]
 94%|█████████▍| 47104/50176 [07:44<00:30, 101.66it/s]
 94%|█████████▍| 47360/50176 [07:47<00:28, 99.05it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.23it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.88it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 105.67it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.98it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 102.58it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.73it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.29it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.28it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.87it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.79it/s] 
-2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.4672	imagenet-zeroshot-val-top5: 0.7495

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_50.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
-2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt).
-2022-12-21,11:02:51 | INFO | Model:
-2022-12-21,11:02:51 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,11:02:51 | INFO | Params:
-2022-12-21,11:02:51 | INFO |   batch_size: 256
-2022-12-21,11:02:51 | INFO |   beta1: 0.9
-2022-12-21,11:02:51 | INFO |   beta2: 0.98
-2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/checkpoints
-2022-12-21,11:02:51 | INFO |   copy_codebase: False
-2022-12-21,11:02:51 | INFO |   csv_caption_key: title
-2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
-2022-12-21,11:02:51 | INFO |   csv_separator: 	
-2022-12-21,11:02:51 | INFO |   dataset_resampled: False
-2022-12-21,11:02:51 | INFO |   dataset_type: auto
-2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
-2022-12-21,11:02:51 | INFO |   debug: False
-2022-12-21,11:02:51 | INFO |   device: cuda:0
-2022-12-21,11:02:51 | INFO |   dist_backend: nccl
-2022-12-21,11:02:51 | INFO |   dist_url: env://
-2022-12-21,11:02:51 | INFO |   distributed: False
-2022-12-21,11:02:51 | INFO |   epochs: 32
-2022-12-21,11:02:51 | INFO |   eps: 1e-06
-2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
-2022-12-21,11:02:51 | INFO |   gather_with_grad: False
-2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
-2022-12-21,11:02:51 | INFO |   horovod: False
-2022-12-21,11:02:51 | INFO |   image_mean: None
-2022-12-21,11:02:51 | INFO |   image_std: None
-2022-12-21,11:02:51 | INFO |   imagenet_v2: None
-2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,11:02:51 | INFO |   local_loss: False
-2022-12-21,11:02:51 | INFO |   local_rank: 0
-2022-12-21,11:02:51 | INFO |   lock_image: False
-2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,11:02:51 | INFO |   log_level: 20
-2022-12-21,11:02:51 | INFO |   log_local: False
-2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_50.pt/out.log
-2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,11:02:51 | INFO |   lr: 0.0005
-2022-12-21,11:02:51 | INFO |   model: ViT-B-16
-2022-12-21,11:02:51 | INFO |   name: epoch_50.pt
-2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
-2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
-2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
-2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_50.pt
-2022-12-21,11:02:51 | INFO |   pretrained_image: False
-2022-12-21,11:02:51 | INFO |   rank: 0
-2022-12-21,11:02:51 | INFO |   report_to: 
-2022-12-21,11:02:51 | INFO |   resume: None
-2022-12-21,11:02:51 | INFO |   save_frequency: 1
-2022-12-21,11:02:51 | INFO |   save_most_recent: False
-2022-12-21,11:02:51 | INFO |   seed: 0
-2022-12-21,11:02:51 | INFO |   skip_scheduler: False
-2022-12-21,11:02:51 | INFO |   tensorboard: False
-2022-12-21,11:02:51 | INFO |   tensorboard_path: 
-2022-12-21,11:02:51 | INFO |   torchscript: False
-2022-12-21,11:02:51 | INFO |   trace: False
-2022-12-21,11:02:51 | INFO |   train_data: None
-2022-12-21,11:02:51 | INFO |   train_num_samples: None
-2022-12-21,11:02:51 | INFO |   use_bn_sync: False
-2022-12-21,11:02:51 | INFO |   val_data: None
-2022-12-21,11:02:51 | INFO |   val_frequency: 1
-2022-12-21,11:02:51 | INFO |   val_num_samples: None
-2022-12-21,11:02:51 | INFO |   wandb: False
-2022-12-21,11:02:51 | INFO |   wandb_notes: 
-2022-12-21,11:02:51 | INFO |   warmup: 10000
-2022-12-21,11:02:51 | INFO |   wd: 0.2
-2022-12-21,11:02:51 | INFO |   workers: 1
-2022-12-21,11:02:51 | INFO |   world_size: 1
-2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
-2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
-2022-12-21,11:02:51 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:33,  1.44it/s]
  1%|          | 7/1000 [00:00<01:29, 11.07it/s]
  1%|▏         | 13/1000 [00:00<00:49, 20.11it/s]
  2%|▏         | 19/1000 [00:01<00:34, 28.41it/s]
  2%|▎         | 25/1000 [00:01<00:27, 35.36it/s]
  3%|▎         | 31/1000 [00:01<00:23, 41.09it/s]
  4%|▎         | 37/1000 [00:01<00:21, 45.53it/s]
  4%|▍         | 43/1000 [00:01<00:19, 48.88it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.33it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.08it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.51it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.45it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.09it/s]
  8%|▊         | 79/1000 [00:02<00:16, 56.64it/s]
  8%|▊         | 85/1000 [00:02<00:16, 56.94it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.26it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.67it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.83it/s]
 11%|█         | 109/1000 [00:02<00:15, 58.07it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.26it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 58.18it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.18it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.19it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 58.17it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.21it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.25it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.21it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.10it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.90it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.87it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.85it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.85it/s]
 19%|█▉        | 193/1000 [00:04<00:13, 57.81it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.75it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.59it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.41it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.61it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.60it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.81it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.75it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.58it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.70it/s]
 25%|██▌       | 253/1000 [00:05<00:12, 57.77it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.85it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.84it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.68it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.84it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.98it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.12it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.31it/s]
 30%|███       | 301/1000 [00:05<00:11, 58.29it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.23it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.41it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.44it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.27it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 58.14it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 58.27it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.32it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 58.28it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 58.20it/s]
 36%|███▌      | 361/1000 [00:06<00:10, 58.22it/s]
 37%|███▋      | 367/1000 [00:07<00:10, 58.28it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 58.37it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 58.07it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.89it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.95it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 58.03it/s]
 40%|████      | 403/1000 [00:07<00:10, 58.15it/s]
 41%|████      | 409/1000 [00:07<00:10, 58.22it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.30it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.33it/s]
 43%|████▎     | 427/1000 [00:08<00:09, 58.32it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.46it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.46it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.19it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.23it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.27it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.30it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.23it/s]
 48%|████▊     | 475/1000 [00:08<00:08, 58.37it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.18it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.28it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 58.22it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 58.12it/s]
 50%|█████     | 505/1000 [00:09<00:08, 58.18it/s]
 51%|█████     | 511/1000 [00:09<00:08, 58.10it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 58.18it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 58.16it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.19it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.16it/s]
 54%|█████▍    | 541/1000 [00:10<00:07, 58.22it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 58.26it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 58.16it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 58.24it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 58.18it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 58.08it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 58.18it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 58.27it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 58.30it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 58.20it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.30it/s]
 61%|██████    | 607/1000 [00:11<00:06, 58.15it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.21it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.35it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.33it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.31it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 58.23it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 58.22it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 58.30it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.29it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 58.34it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 58.28it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 58.34it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 58.33it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 58.36it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 58.35it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 58.26it/s]
 70%|███████   | 703/1000 [00:12<00:05, 58.33it/s]
 71%|███████   | 709/1000 [00:12<00:04, 58.35it/s]
 72%|███████▏  | 715/1000 [00:13<00:04, 58.32it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.24it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 58.20it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 58.20it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.26it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.28it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.35it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.39it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.27it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.34it/s]
 78%|███████▊  | 775/1000 [00:14<00:03, 58.31it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.36it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.36it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.22it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.14it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.15it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.24it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.23it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.34it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.49it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.52it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.38it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.42it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.38it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.37it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.31it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.34it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.31it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.23it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.32it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.27it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.24it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.18it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.31it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.29it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.39it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.43it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.37it/s]
 95%|█████████▍| 949/1000 [00:17<00:00, 58.27it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.42it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.32it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.44it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.54it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.52it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.14it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 57.94it/s]
-2022-12-21,11:03:09 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<10:36, 78.47it/s]
  1%|          | 512/50176 [00:05<09:16, 89.27it/s]
  2%|▏         | 768/50176 [00:08<08:27, 97.42it/s]
  2%|▏         | 1024/50176 [00:10<08:21, 98.01it/s]
  3%|▎         | 1280/50176 [00:13<08:16, 98.41it/s]
  3%|▎         | 1536/50176 [00:16<08:53, 91.22it/s]
  4%|▎         | 1792/50176 [00:19<08:39, 93.13it/s]
  4%|▍         | 2048/50176 [00:21<08:19, 96.32it/s]
  5%|▍         | 2304/50176 [00:24<08:07, 98.13it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.64it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.09it/s] 
  6%|▌         | 3072/50176 [00:31<07:53, 99.53it/s]
  7%|▋         | 3328/50176 [00:34<07:39, 101.88it/s]
  7%|▋         | 3584/50176 [00:36<07:30, 103.50it/s]
  8%|▊         | 3840/50176 [00:39<07:39, 100.94it/s]
  8%|▊         | 4096/50176 [00:42<07:53, 97.26it/s] 
  9%|▊         | 4352/50176 [00:44<07:49, 97.60it/s]
  9%|▉         | 4608/50176 [00:47<07:32, 100.68it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.86it/s]
 10%|█         | 5120/50176 [00:51<07:20, 102.25it/s]
 11%|█         | 5376/50176 [00:54<07:26, 100.40it/s]
 11%|█         | 5632/50176 [00:57<07:19, 101.46it/s]
 12%|█▏        | 5888/50176 [00:59<07:20, 100.56it/s]
 12%|█▏        | 6144/50176 [01:03<08:00, 91.56it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.84it/s]
 13%|█▎        | 6656/50176 [01:08<07:26, 97.49it/s]
 14%|█▍        | 6912/50176 [01:10<07:35, 95.08it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.23it/s]
 15%|█▍        | 7424/50176 [01:15<07:03, 100.88it/s]
 15%|█▌        | 7680/50176 [01:18<06:56, 101.91it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.27it/s]
 16%|█▋        | 8192/50176 [01:23<06:50, 102.33it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 102.96it/s]
 17%|█▋        | 8704/50176 [01:28<06:47, 101.75it/s]
 18%|█▊        | 8960/50176 [01:30<06:59, 98.34it/s] 
 18%|█▊        | 9216/50176 [01:33<06:38, 102.81it/s]
 19%|█▉        | 9472/50176 [01:35<06:34, 103.23it/s]
 19%|█▉        | 9728/50176 [01:38<06:43, 100.18it/s]
 20%|█▉        | 9984/50176 [01:40<06:41, 100.05it/s]
 20%|██        | 10240/50176 [01:43<06:27, 102.94it/s]
 21%|██        | 10496/50176 [01:45<06:27, 102.35it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.25it/s]
 22%|██▏       | 11008/50176 [01:50<06:26, 101.27it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.85it/s]
 23%|██▎       | 11520/50176 [01:55<06:12, 103.67it/s]
 23%|██▎       | 11776/50176 [01:58<06:10, 103.66it/s]
 24%|██▍       | 12032/50176 [02:00<06:14, 101.72it/s]
 24%|██▍       | 12288/50176 [02:03<06:08, 102.76it/s]
 25%|██▌       | 12544/50176 [02:05<06:14, 100.56it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 98.89it/s] 
 26%|██▌       | 13056/50176 [02:10<06:03, 102.15it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.76it/s]
 27%|██▋       | 13568/50176 [02:16<06:22, 95.61it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.55it/s]
 28%|██▊       | 14080/50176 [02:22<06:25, 93.66it/s]
 29%|██▊       | 14336/50176 [02:24<06:20, 94.31it/s]
 29%|██▉       | 14592/50176 [02:27<06:02, 98.30it/s]
 30%|██▉       | 14848/50176 [02:29<06:06, 96.48it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.45it/s]
 31%|███       | 15360/50176 [02:35<05:55, 97.88it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.79it/s]
 32%|███▏      | 15872/50176 [02:40<05:46, 98.90it/s]
 32%|███▏      | 16128/50176 [02:42<05:40, 100.14it/s]
 33%|███▎      | 16384/50176 [02:45<05:32, 101.71it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.81it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.97it/s]
 34%|███▍      | 17152/50176 [02:52<05:25, 101.48it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.23it/s]
 35%|███▌      | 17664/50176 [02:57<05:16, 102.63it/s]
 36%|███▌      | 17920/50176 [03:00<05:09, 104.24it/s]
 36%|███▌      | 18176/50176 [03:02<05:13, 102.18it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.95it/s]
 37%|███▋      | 18688/50176 [03:07<05:15, 99.79it/s] 
 38%|███▊      | 18944/50176 [03:10<05:17, 98.47it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.84it/s]
 39%|███▉      | 19456/50176 [03:16<05:18, 96.54it/s]
 39%|███▉      | 19712/50176 [03:18<05:19, 95.47it/s]
 40%|███▉      | 19968/50176 [03:21<05:13, 96.48it/s]
 40%|████      | 20224/50176 [03:23<04:53, 102.15it/s]
 41%|████      | 20480/50176 [03:26<04:49, 102.58it/s]
 41%|████▏     | 20736/50176 [03:28<04:43, 104.00it/s]
 42%|████▏     | 20992/50176 [03:30<04:36, 105.50it/s]
 42%|████▏     | 21248/50176 [03:33<04:39, 103.42it/s]
 43%|████▎     | 21504/50176 [03:35<04:35, 104.05it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.55it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.42it/s]
 44%|████▍     | 22272/50176 [03:42<04:20, 107.27it/s]
 45%|████▍     | 22528/50176 [03:45<04:19, 106.73it/s]
 45%|████▌     | 22784/50176 [03:47<04:24, 103.74it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.61it/s]
 46%|██���█▋     | 23296/50176 [03:53<04:28, 99.98it/s] 
 47%|████▋     | 23552/50176 [03:56<04:34, 96.82it/s]
 47%|████▋     | 23808/50176 [03:58<04:19, 101.52it/s]
 48%|████▊     | 24064/50176 [04:00<04:19, 100.52it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.06it/s]
 49%|████▉     | 24576/50176 [04:06<04:17, 99.39it/s] 
 49%|████▉     | 24832/50176 [04:08<04:08, 101.94it/s]
 50%|█████     | 25088/50176 [04:10<04:00, 104.21it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.41it/s]
 51%|█████     | 25600/50176 [04:15<03:57, 103.35it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.24it/s]
 52%|█████▏    | 26112/50176 [04:20<03:48, 105.15it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.28it/s]
 53%|█████▎    | 26624/50176 [04:26<03:54, 100.61it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.22it/s]
 54%|█████▍    | 27136/50176 [04:31<03:49, 100.35it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 102.17it/s]
 55%|█████▌    | 27648/50176 [04:35<03:34, 105.19it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.52it/s]
 56%|█████▌    | 28160/50176 [04:41<03:42, 99.02it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:37, 99.92it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.11it/s]
 58%|█████▊    | 28928/50176 [04:48<03:29, 101.50it/s]
 58%|█████▊    | 29184/50176 [04:51<03:23, 103.06it/s]
 59%|█████▊    | 29440/50176 [04:54<03:33, 96.94it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:30, 97.18it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.49it/s]
 60%|██████    | 30208/50176 [05:01<03:23, 98.33it/s]
 61%|██████    | 30464/50176 [05:04<03:12, 102.40it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.85it/s]
 62%|██████▏   | 30976/50176 [05:09<03:05, 103.29it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.19it/s]
 63%|██████▎   | 31488/50176 [05:13<02:55, 106.35it/s]
 63%|██████▎   | 31744/50176 [05:16<02:54, 105.40it/s]
 64%|██████▍   | 32000/50176 [05:18<02:52, 105.41it/s]
 64%|██████▍   | 32256/50176 [05:21<02:49, 105.48it/s]
 65%|██████▍   | 32512/50176 [05:24<02:56, 100.14it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.78it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 101.81it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.77it/s]
 67%|██████▋   | 33536/50176 [05:34<02:46, 99.96it/s] 
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.18it/s]
 68%|██████▊   | 34048/50176 [05:38<02:33, 104.92it/s]
 68%|██████▊   | 34304/50176 [05:41<02:28, 107.22it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.92it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 106.02it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.62it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.52it/s]
 71%|███████   | 35584/50176 [05:53<02:16, 106.69it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.44it/s]
 72%|███████▏  | 36096/50176 [05:58<02:14, 104.41it/s]
 72%|███████▏  | 36352/50176 [06:00<02:13, 103.67it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.14it/s]
 73%|███████▎  | 36864/50176 [06:06<02:13, 99.86it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.31it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.25it/s]
 75%|███████▌  | 37632/50176 [06:12<01:56, 107.51it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.67it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 107.07it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.77it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.39it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.05it/s]
 78%|███████▊  | 39168/50176 [06:28<01:49, 100.51it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.82it/s]
 79%|███████▉  | 39680/50176 [06:32<01:42, 101.96it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.18it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.71it/s]
 81%|████████  | 40448/50176 [06:40<01:39, 98.22it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.35it/s]
 82%|████████▏ | 40960/50176 [06:45<01:32, 99.77it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.00it/s]
 83%|████████▎ | 41472/50176 [06:50<01:26, 100.88it/s]
 83%|████████▎ | 41728/50176 [06:53<01:22, 102.98it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.23it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.50it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.29it/s]
 85%|████████▌ | 42752/50176 [07:02<01:09, 107.12it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.90it/s]
 86%|████████▌ | 43264/50176 [07:07<01:05, 105.29it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.24it/s]
 87%|████████▋ | 43776/50176 [07:12<01:02, 102.35it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.85it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.38it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.37it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:25<00:48, 105.97it/s]
 90%|█████████ | 45312/50176 [07:27<00:46, 104.97it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 106.94it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.62it/s]
 92%|█████████▏| 46080/50176 [07:35<00:39, 103.15it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.64it/s]
 93%|█████████▎| 46592/50176 [07:39<00:33, 105.44it/s]
 93%|█████████▎| 46848/50176 [07:42<00:31, 106.78it/s]
 94%|█████████▍| 47104/50176 [07:44<00:30, 101.66it/s]
 94%|█████████▍| 47360/50176 [07:47<00:28, 99.05it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.23it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.87it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 105.65it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.96it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 102.47it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.87it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.39it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.09it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.98it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.66it/s] 
-2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6466	imagenet-zeroshot-val-top5: 0.8905

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_51.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
-2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt).
-2022-12-21,11:02:51 | INFO | Model:
-2022-12-21,11:02:51 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,11:02:51 | INFO | Params:
-2022-12-21,11:02:51 | INFO |   batch_size: 256
-2022-12-21,11:02:51 | INFO |   beta1: 0.9
-2022-12-21,11:02:51 | INFO |   beta2: 0.98
-2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/checkpoints
-2022-12-21,11:02:51 | INFO |   copy_codebase: False
-2022-12-21,11:02:51 | INFO |   csv_caption_key: title
-2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
-2022-12-21,11:02:51 | INFO |   csv_separator: 	
-2022-12-21,11:02:51 | INFO |   dataset_resampled: False
-2022-12-21,11:02:51 | INFO |   dataset_type: auto
-2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
-2022-12-21,11:02:51 | INFO |   debug: False
-2022-12-21,11:02:51 | INFO |   device: cuda:0
-2022-12-21,11:02:51 | INFO |   dist_backend: nccl
-2022-12-21,11:02:51 | INFO |   dist_url: env://
-2022-12-21,11:02:51 | INFO |   distributed: False
-2022-12-21,11:02:51 | INFO |   epochs: 32
-2022-12-21,11:02:51 | INFO |   eps: 1e-06
-2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
-2022-12-21,11:02:51 | INFO |   gather_with_grad: False
-2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
-2022-12-21,11:02:51 | INFO |   horovod: False
-2022-12-21,11:02:51 | INFO |   image_mean: None
-2022-12-21,11:02:51 | INFO |   image_std: None
-2022-12-21,11:02:51 | INFO |   imagenet_v2: None
-2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,11:02:51 | INFO |   local_loss: False
-2022-12-21,11:02:51 | INFO |   local_rank: 0
-2022-12-21,11:02:51 | INFO |   lock_image: False
-2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,11:02:51 | INFO |   log_level: 20
-2022-12-21,11:02:51 | INFO |   log_local: False
-2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_51.pt/out.log
-2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,11:02:51 | INFO |   lr: 0.0005
-2022-12-21,11:02:51 | INFO |   model: ViT-B-16
-2022-12-21,11:02:51 | INFO |   name: epoch_51.pt
-2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
-2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
-2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
-2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_51.pt
-2022-12-21,11:02:51 | INFO |   pretrained_image: False
-2022-12-21,11:02:51 | INFO |   rank: 0
-2022-12-21,11:02:51 | INFO |   report_to: 
-2022-12-21,11:02:51 | INFO |   resume: None
-2022-12-21,11:02:51 | INFO |   save_frequency: 1
-2022-12-21,11:02:51 | INFO |   save_most_recent: False
-2022-12-21,11:02:51 | INFO |   seed: 0
-2022-12-21,11:02:51 | INFO |   skip_scheduler: False
-2022-12-21,11:02:51 | INFO |   tensorboard: False
-2022-12-21,11:02:51 | INFO |   tensorboard_path: 
-2022-12-21,11:02:51 | INFO |   torchscript: False
-2022-12-21,11:02:51 | INFO |   trace: False
-2022-12-21,11:02:51 | INFO |   train_data: None
-2022-12-21,11:02:51 | INFO |   train_num_samples: None
-2022-12-21,11:02:51 | INFO |   use_bn_sync: False
-2022-12-21,11:02:51 | INFO |   val_data: None
-2022-12-21,11:02:51 | INFO |   val_frequency: 1
-2022-12-21,11:02:51 | INFO |   val_num_samples: None
-2022-12-21,11:02:51 | INFO |   wandb: False
-2022-12-21,11:02:51 | INFO |   wandb_notes: 
-2022-12-21,11:02:51 | INFO |   warmup: 10000
-2022-12-21,11:02:51 | INFO |   wd: 0.2
-2022-12-21,11:02:51 | INFO |   workers: 1
-2022-12-21,11:02:51 | INFO |   world_size: 1
-2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
-2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
-2022-12-21,11:02:51 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:32,  1.44it/s]
  1%|          | 6/1000 [00:00<01:42,  9.65it/s]
  1%|          | 11/1000 [00:00<00:56, 17.48it/s]
  2%|▏         | 17/1000 [00:01<00:38, 25.76it/s]
  2%|▏         | 23/1000 [00:01<00:29, 32.94it/s]
  3%|▎         | 29/1000 [00:01<00:25, 38.70it/s]
  4%|▎         | 35/1000 [00:01<00:22, 43.53it/s]
  4%|▍         | 41/1000 [00:01<00:20, 47.46it/s]
  5%|▍         | 47/1000 [00:01<00:18, 50.17it/s]
  5%|▌         | 53/1000 [00:01<00:18, 52.37it/s]
  6%|▌         | 59/1000 [00:01<00:17, 53.71it/s]
  6%|▋         | 65/1000 [00:01<00:17, 54.90it/s]
  7%|▋         | 71/1000 [00:01<00:16, 55.61it/s]
  8%|▊         | 77/1000 [00:02<00:16, 56.22it/s]
  8%|▊         | 83/1000 [00:02<00:16, 56.79it/s]
  9%|▉         | 89/1000 [00:02<00:15, 57.25it/s]
 10%|▉         | 95/1000 [00:02<00:15, 57.39it/s]
 10%|█         | 101/1000 [00:02<00:15, 57.65it/s]
 11%|█         | 107/1000 [00:02<00:15, 57.97it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 58.11it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 58.13it/s]
 12%|█▎        | 125/1000 [00:02<00:15, 58.08it/s]
 13%|█▎        | 131/1000 [00:02<00:14, 58.16it/s]
 14%|█▎        | 137/1000 [00:03<00:14, 58.05it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 58.08it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 58.13it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 58.21it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 58.09it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.97it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.93it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.94it/s]
 18%|█▊        | 185/1000 [00:03<00:14, 57.77it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.76it/s]
 20%|█▉        | 197/1000 [00:04<00:13, 57.72it/s]
 20%|██        | 203/1000 [00:04<00:13, 57.60it/s]
 21%|██        | 209/1000 [00:04<00:13, 57.43it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 57.48it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 57.50it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.68it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.54it/s]
 24%|██▍       | 239/1000 [00:04<00:13, 57.45it/s]
 24%|██▍       | 245/1000 [00:04<00:13, 57.50it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.58it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.79it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.84it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.61it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.73it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.67it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.80it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 58.07it/s]
 30%|██▉       | 299/1000 [00:05<00:12, 58.09it/s]
 30%|███       | 305/1000 [00:06<00:11, 58.14it/s]
 31%|███       | 311/1000 [00:06<00:11, 58.26it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 58.25it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 58.19it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 58.00it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 58.08it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 58.13it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 58.23it/s]
 35%|███▌      | 353/1000 [00:06<00:11, 58.05it/s]
 36%|███▌      | 359/1000 [00:07<00:16, 37.99it/s]
 36%|███▋      | 365/1000 [00:07<00:14, 42.37it/s]
 37%|███▋      | 371/1000 [00:07<00:13, 46.16it/s]
 38%|███▊      | 377/1000 [00:07<00:12, 49.11it/s]
 38%|███▊      | 383/1000 [00:07<00:12, 51.33it/s]
 39%|███▉      | 389/1000 [00:07<00:11, 53.12it/s]
 40%|███▉      | 395/1000 [00:07<00:11, 54.55it/s]
 40%|████      | 401/1000 [00:07<00:10, 55.65it/s]
 41%|████      | 407/1000 [00:07<00:10, 56.46it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 57.00it/s]
 42%|████▏     | 419/1000 [00:08<00:10, 57.39it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.58it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.90it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 58.03it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.92it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 58.01it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 58.12it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 58.12it/s]
 47%|████▋     | 467/1000 [00:08<00:09, 58.12it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 58.27it/s]
 48%|████▊     | 479/1000 [00:09<00:08, 58.19it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 58.18it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.88it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.88it/s]
 50%|█████     | 503/1000 [00:09<00:08, 58.07it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.99it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 58.07it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 58.15it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 58.27it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 58.18it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 58.31it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 58.36it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 58.23it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 58.20it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 58.26it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 58.17it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 58.11it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 58.26it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 58.19it/s]
 59%|█████▉    | 593/1000 [00:11<00:06, 58.15it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 58.26it/s]
 60%|██████    | 605/1000 [00:11<00:06, 58.17it/s]
 61%|██████    | 611/1000 [00:11<00:06, 58.16it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 58.27it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 58.29it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 58.35it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 58.22it/s]
 64%|██████▍   | 641/1000 [00:11<00:06, 58.09it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 58.30it/s]
 65%|██████▌   | 653/1000 [00:12<00:05, 58.22it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 58.32it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 58.33it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 58.30it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 58.36it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 58.33it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 58.39it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 58.25it/s]
 70%|███████   | 701/1000 [00:12<00:05, 58.35it/s]
 71%|███████   | 707/1000 [00:13<00:05, 58.33it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 58.34it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 58.38it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 58.31it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 58.28it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 58.33it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 58.28it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 58.26it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 58.41it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 58.23it/s]
 77%|███████▋  | 767/1000 [00:14<00:03, 58.28it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 58.30it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 58.35it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 58.38it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 58.18it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 58.18it/s]
 80%|████████  | 803/1000 [00:14<00:03, 58.20it/s]
 81%|████████  | 809/1000 [00:14<00:03, 58.20it/s]
 82%|████████▏ | 815/1000 [00:14<00:03, 58.21it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 58.02it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 58.08it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 58.30it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 58.41it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 58.29it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 58.16it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 58.17it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 58.26it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 58.24it/s]
 88%|████████▊ | 875/1000 [00:15<00:02, 58.29it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 58.32it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 58.14it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 58.24it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 58.31it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 58.27it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 58.24it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 58.28it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 58.22it/s]
 93%|█████████▎| 929/1000 [00:16<00:01, 58.34it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 58.43it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 58.41it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 58.39it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 58.43it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 58.36it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 58.42it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 58.50it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 58.53it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.55it/s]
 99%|█████████▉| 989/1000 [00:17<00:00, 58.49it/s]
-2022-12-21,11:03:09 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<11:47, 70.57it/s]
  1%|          | 512/50176 [00:06<09:46, 84.62it/s]
  2%|▏         | 768/50176 [00:08<08:41, 94.65it/s]
  2%|▏         | 1024/50176 [00:11<08:31, 96.01it/s]
  3%|▎         | 1280/50176 [00:13<08:22, 97.36it/s]
  3%|▎         | 1536/50176 [00:16<08:57, 90.53it/s]
  4%|▎         | 1792/50176 [00:19<08:42, 92.65it/s]
  4%|▍         | 2048/50176 [00:22<08:21, 96.05it/s]
  5%|▍         | 2304/50176 [00:24<08:10, 97.65it/s]
  5%|▌         | 2560/50176 [00:26<07:53, 100.57it/s]
  6%|▌         | 2816/50176 [00:29<08:02, 98.08it/s] 
  6%|▌         | 3072/50176 [00:32<07:53, 99.40it/s]
  7%|▋         | 3328/50176 [00:34<07:40, 101.83it/s]
  7%|▋         | 3584/50176 [00:36<07:30, 103.35it/s]
  8%|▊         | 3840/50176 [00:39<07:38, 100.99it/s]
  8%|▊         | 4096/50176 [00:42<07:52, 97.45it/s] 
  9%|▊         | 4352/50176 [00:45<07:49, 97.62it/s]
  9%|▉         | 4608/50176 [00:47<07:33, 100.55it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.86it/s]
 10%|█         | 5120/50176 [00:52<07:21, 102.16it/s]
 11%|█         | 5376/50176 [00:55<07:26, 100.30it/s]
 11%|█         | 5632/50176 [00:57<07:18, 101.57it/s]
 12%|█▏        | 5888/50176 [01:00<07:21, 100.37it/s]
 12%|█▏        | 6144/50176 [01:03<08:00, 91.67it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.86it/s]
 13%|█▎        | 6656/50176 [01:08<07:26, 97.54it/s]
 14%|█▍        | 6912/50176 [01:11<07:34, 95.11it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.26it/s]
 15%|█▍        | 7424/50176 [01:16<07:04, 100.78it/s]
 15%|█▌        | 7680/50176 [01:18<06:57, 101.88it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.10it/s]
 16%|█▋        | 8192/50176 [01:23<06:50, 102.33it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 102.96it/s]
 17%|█▋        | 8704/50176 [01:28<06:46, 101.96it/s]
 18%|█▊        | 8960/50176 [01:31<06:58, 98.40it/s] 
 18%|█▊        | 9216/50176 [01:33<06:38, 102.87it/s]
 19%|█▉        | 9472/50176 [01:35<06:35, 102.99it/s]
 19%|█▉        | 9728/50176 [01:38<06:43, 100.30it/s]
 20%|█▉        | 9984/50176 [01:41<06:42, 99.96it/s] 
 20%|██        | 10240/50176 [01:43<06:27, 103.13it/s]
 21%|██        | 10496/50176 [01:46<06:27, 102.36it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.23it/s]
 22%|██▏       | 11008/50176 [01:51<06:26, 101.30it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.82it/s]
 23%|██▎       | 11520/50176 [01:56<06:12, 103.84it/s]
 23%|██▎       | 11776/50176 [01:58<06:10, 103.69it/s]
 24%|██▍       | 12032/50176 [02:01<06:14, 101.72it/s]
 24%|██▍       | 12288/50176 [02:03<06:09, 102.66it/s]
 25%|██▌       | 12544/50176 [02:06<06:14, 100.48it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 98.93it/s] 
 26%|██▌       | 13056/50176 [02:11<06:03, 102.09it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.70it/s]
 27%|██▋       | 13568/50176 [02:16<06:22, 95.59it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.59it/s]
 28%|██▊       | 14080/50176 [02:22<06:25, 93.71it/s]
 29%|██▊       | 14336/50176 [02:25<06:19, 94.40it/s]
 29%|██▉       | 14592/50176 [02:27<06:02, 98.24it/s]
 30%|██▉       | 14848/50176 [02:30<06:06, 96.36it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.56it/s]
 31%|███       | 15360/50176 [02:35<05:55, 98.00it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.70it/s]
 32%|███▏      | 15872/50176 [02:40<05:47, 98.73it/s]
 32%|███▏      | 16128/50176 [02:42<05:39, 100.21it/s]
 33%|███▎      | 16384/50176 [02:45<05:32, 101.77it/s]
 33%|███▎      | 16640/50176 [02:48<05:42, 97.87it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.91it/s]
 34%|███▍      | 17152/50176 [02:53<05:24, 101.63it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.20it/s]
 35%|███▌      | 17664/50176 [02:58<05:16, 102.63it/s]
 36%|███▌      | 17920/50176 [03:00<05:09, 104.12it/s]
 36%|███▌      | 18176/50176 [03:03<05:13, 102.09it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.92it/s]
 37%|███▋      | 18688/50176 [03:08<05:15, 99.69it/s] 
 38%|███▊      | 18944/50176 [03:11<05:16, 98.58it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.92it/s]
 39%|███▉      | 19456/50176 [03:16<05:18, 96.56it/s]
 39%|███▉      | 19712/50176 [03:19<05:19, 95.37it/s]
 40%|███▉      | 19968/50176 [03:21<05:12, 96.57it/s]
 40%|████      | 20224/50176 [03:23<04:52, 102.28it/s]
 41%|████      | 20480/50176 [03:26<04:49, 102.64it/s]
 41%|████▏     | 20736/50176 [03:28<04:43, 103.96it/s]
 42%|████▏     | 20992/50176 [03:31<04:36, 105.47it/s]
 42%|████▏     | 21248/50176 [03:33<04:40, 103.18it/s]
 43%|████▎     | 21504/50176 [03:36<04:35, 103.97it/s]
 43%|████▎     | 21760/50176 [03:38<04:34, 103.68it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.56it/s]
 44%|████▍     | 22272/50176 [03:43<04:20, 107.29it/s]
 45%|████▍     | 22528/50176 [03:45<04:19, 106.70it/s]
 45%|████▌     | 22784/50176 [03:48<04:24, 103.74it/s]
 46%|████▌     | 23040/50176 [03:50<04:27, 101.62it/s]
 46%|████▋     | 23296/50176 [03:53<04:28, 99.95it/s] 
 47%|████▋     | 23552/50176 [03:56<04:34, 96.87it/s]
 47%|████▋     | 23808/50176 [03:58<04:19, 101.60it/s]
 48%|████▊     | 24064/50176 [04:01<04:19, 100.51it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.00it/s]
 49%|████▉     | 24576/50176 [04:06<04:17, 99.35it/s] 
 49%|████▉     | 24832/50176 [04:08<04:08, 101.85it/s]
 50%|█████     | 25088/50176 [04:11<04:00, 104.30it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.35it/s]
 51%|█████     | 25600/50176 [04:16<03:57, 103.39it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.22it/s]
 52%|█████▏    | 26112/50176 [04:21<03:48, 105.12it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.32it/s]
 53%|█████▎    | 26624/50176 [04:26<03:53, 100.66it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.23it/s]
 54%|█████▍    | 27136/50176 [04:31<03:50, 100.15it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 101.96it/s]
 55%|█████▌    | 27648/50176 [04:36<03:33, 105.30it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.62it/s]
 56%|█████▌    | 28160/50176 [04:41<03:42, 99.08it/s] 
 57%|█████▋    | 28416/50176 [04:44<03:37, 99.94it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.26it/s]
 58%|█████▊    | 28928/50176 [04:49<03:29, 101.53it/s]
 58%|█████▊    | 29184/50176 [04:51<03:23, 103.06it/s]
 59%|█████▊    | 29440/50176 [04:54<03:33, 96.92it/s] 
 59%|█████▉    | 29696/50176 [04:57<03:30, 97.15it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.56it/s]
 60%|██████    | 30208/50176 [05:02<03:23, 98.26it/s]
 61%|██████    | 30464/50176 [05:04<03:12, 102.33it/s]
 61%|██████    | 30720/50176 [05:07<03:10, 101.94it/s]
 62%|██████▏   | 30976/50176 [05:09<03:05, 103.37it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.12it/s]
 63%|██████▎   | 31488/50176 [05:14<02:55, 106.43it/s]
 63%|██████▎   | 31744/50176 [05:16<02:55, 105.23it/s]
 64%|██████▍   | 32000/50176 [05:19<02:52, 105.30it/s]
 64%|██████▍   | 32256/50176 [05:21<02:49, 105.42it/s]
 65%|██████▍   | 32512/50176 [05:24<02:56, 100.15it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.67it/s]
 66%|██████▌   | 33024/50176 [05:29<02:48, 101.83it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.61it/s]
 67%|██████▋   | 33536/50176 [05:34<02:46, 99.97it/s] 
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.08it/s]
 68%|██████▊   | 34048/50176 [05:39<02:33, 104.94it/s]
 68%|██████▊   | 34304/50176 [05:41<02:27, 107.35it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.93it/s]
 69%|██████▉   | 34816/50176 [05:46<02:24, 105.94it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.82it/s]
 70%|███████   | 35328/50176 [05:51<02:19, 106.47it/s]
 71%|███████   | 35584/50176 [05:53<02:16, 106.63it/s]
 71%|███████▏  | 35840/50176 [05:56<02:17, 104.50it/s]
 72%|███████▏  | 36096/50176 [05:58<02:14, 104.43it/s]
 72%|███████▏  | 36352/50176 [06:01<02:13, 103.56it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.07it/s]
 73%|███████▎  | 36864/50176 [06:06<02:13, 99.86it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.29it/s]
 74%|███████▍  | 37376/50176 [06:11<02:01, 105.40it/s]
 75%|███████▌  | 37632/50176 [06:13<01:56, 107.46it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.76it/s]
 76%|███████▌  | 38144/50176 [06:18<01:52, 107.13it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.74it/s]
 77%|███████▋  | 38656/50176 [06:23<01:50, 104.43it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 103.98it/s]
 78%|███████▊  | 39168/50176 [06:28<01:49, 100.55it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.66it/s]
 79%|███████▉  | 39680/50176 [06:33<01:42, 101.94it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.34it/s]
 80%|████████  | 40192/50176 [06:38<01:37, 102.60it/s]
 81%|████████  | 40448/50176 [06:41<01:39, 98.25it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.43it/s]
 82%|████████▏ | 40960/50176 [06:46<01:32, 99.67it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.11it/s]
 83%|████████▎ | 41472/50176 [06:51<01:26, 100.83it/s]
 83%|████████▎ | 41728/50176 [06:53<01:21, 103.04it/s]
 84%|████████▎ | 41984/50176 [06:56<01:19, 103.17it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.52it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.30it/s]
 85%|████████▌ | 42752/50176 [07:03<01:09, 107.00it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.90it/s]
 86%|████████▌ | 43264/50176 [07:08<01:05, 105.36it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.28it/s]
 87%|████████▋ | 43776/50176 [07:13<01:02, 102.26it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.98it/s]
 88%|████████▊ | 44288/50176 [07:18<00:56, 104.23it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.34it/s]
 89%|████████▉ | 44800/50176 [07:23<00:52, 102.07it/s]
 90%|████████▉ | 45056/50176 [07:25<00:48, 105.94it/s]
 90%|█████████ | 45312/50176 [07:28<00:46, 104.91it/s]
 91%|█████████ | 45568/50176 [07:30<00:43, 107.02it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.62it/s]
 92%|█████████▏| 46080/50176 [07:35<00:39, 103.17it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.53it/s]
 93%|█████████▎| 46592/50176 [07:40<00:33, 105.57it/s]
 93%|█████████▎| 46848/50176 [07:42<00:31, 106.72it/s]
 94%|█████████▍| 47104/50176 [07:45<00:30, 101.66it/s]
 94%|█████████▍| 47360/50176 [07:48<00:28, 99.12it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.26it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.93it/s]
 96%|█████████▌| 48128/50176 [07:55<00:19, 105.62it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.94it/s]
 97%|█████████▋| 48640/50176 [08:00<00:14, 102.47it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.79it/s]
 98%|█████████▊| 49152/50176 [08:05<00:10, 100.30it/s]
 98%|█████████▊| 49408/50176 [08:08<00:07, 101.20it/s]
 99%|█████████▉| 49664/50176 [08:10<00:05, 100.89it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.70it/s] 
-2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6518	imagenet-zeroshot-val-top5: 0.8922

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_52.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,11:02:41 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,11:02:41 | INFO | Loading ViT-B-16 model config.
-2022-12-21,11:02:42 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt).
-2022-12-21,11:02:51 | INFO | Model:
-2022-12-21,11:02:51 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,11:02:51 | INFO | Params:
-2022-12-21,11:02:51 | INFO |   batch_size: 256
-2022-12-21,11:02:51 | INFO |   beta1: 0.9
-2022-12-21,11:02:51 | INFO |   beta2: 0.98
-2022-12-21,11:02:51 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/checkpoints
-2022-12-21,11:02:51 | INFO |   copy_codebase: False
-2022-12-21,11:02:51 | INFO |   csv_caption_key: title
-2022-12-21,11:02:51 | INFO |   csv_img_key: filepath
-2022-12-21,11:02:51 | INFO |   csv_separator: 	
-2022-12-21,11:02:51 | INFO |   dataset_resampled: False
-2022-12-21,11:02:51 | INFO |   dataset_type: auto
-2022-12-21,11:02:51 | INFO |   ddp_static_graph: False
-2022-12-21,11:02:51 | INFO |   debug: False
-2022-12-21,11:02:51 | INFO |   device: cuda:0
-2022-12-21,11:02:51 | INFO |   dist_backend: nccl
-2022-12-21,11:02:51 | INFO |   dist_url: env://
-2022-12-21,11:02:51 | INFO |   distributed: False
-2022-12-21,11:02:51 | INFO |   epochs: 32
-2022-12-21,11:02:51 | INFO |   eps: 1e-06
-2022-12-21,11:02:51 | INFO |   force_quick_gelu: False
-2022-12-21,11:02:51 | INFO |   gather_with_grad: False
-2022-12-21,11:02:51 | INFO |   grad_checkpointing: False
-2022-12-21,11:02:51 | INFO |   horovod: False
-2022-12-21,11:02:51 | INFO |   image_mean: None
-2022-12-21,11:02:51 | INFO |   image_std: None
-2022-12-21,11:02:51 | INFO |   imagenet_v2: None
-2022-12-21,11:02:51 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,11:02:51 | INFO |   local_loss: False
-2022-12-21,11:02:51 | INFO |   local_rank: 0
-2022-12-21,11:02:51 | INFO |   lock_image: False
-2022-12-21,11:02:51 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,11:02:51 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,11:02:51 | INFO |   log_level: 20
-2022-12-21,11:02:51 | INFO |   log_local: False
-2022-12-21,11:02:51 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_52.pt/out.log
-2022-12-21,11:02:51 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,11:02:51 | INFO |   lr: 0.0005
-2022-12-21,11:02:51 | INFO |   model: ViT-B-16
-2022-12-21,11:02:51 | INFO |   name: epoch_52.pt
-2022-12-21,11:02:51 | INFO |   no_set_device_rank: False
-2022-12-21,11:02:51 | INFO |   norm_gradient_clip: None
-2022-12-21,11:02:51 | INFO |   precision: amp_bfloat16
-2022-12-21,11:02:51 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_52.pt
-2022-12-21,11:02:51 | INFO |   pretrained_image: False
-2022-12-21,11:02:51 | INFO |   rank: 0
-2022-12-21,11:02:51 | INFO |   report_to: 
-2022-12-21,11:02:51 | INFO |   resume: None
-2022-12-21,11:02:51 | INFO |   save_frequency: 1
-2022-12-21,11:02:51 | INFO |   save_most_recent: False
-2022-12-21,11:02:51 | INFO |   seed: 0
-2022-12-21,11:02:51 | INFO |   skip_scheduler: False
-2022-12-21,11:02:51 | INFO |   tensorboard: False
-2022-12-21,11:02:51 | INFO |   tensorboard_path: 
-2022-12-21,11:02:51 | INFO |   torchscript: False
-2022-12-21,11:02:51 | INFO |   trace: False
-2022-12-21,11:02:51 | INFO |   train_data: None
-2022-12-21,11:02:51 | INFO |   train_num_samples: None
-2022-12-21,11:02:51 | INFO |   use_bn_sync: False
-2022-12-21,11:02:51 | INFO |   val_data: None
-2022-12-21,11:02:51 | INFO |   val_frequency: 1
-2022-12-21,11:02:51 | INFO |   val_num_samples: None
-2022-12-21,11:02:51 | INFO |   wandb: False
-2022-12-21,11:02:51 | INFO |   wandb_notes: 
-2022-12-21,11:02:51 | INFO |   warmup: 10000
-2022-12-21,11:02:51 | INFO |   wd: 0.2
-2022-12-21,11:02:51 | INFO |   workers: 1
-2022-12-21,11:02:51 | INFO |   world_size: 1
-2022-12-21,11:02:51 | INFO |   zeroshot_frequency: 2
-2022-12-21,11:02:51 | INFO | Starting zero-shot imagenet.
-2022-12-21,11:02:51 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<11:48,  1.41it/s]
  1%|          | 6/1000 [00:00<01:44,  9.52it/s]
  1%|          | 12/1000 [00:00<00:53, 18.63it/s]
  2%|▏         | 18/1000 [00:01<00:36, 27.06it/s]
  2%|▏         | 24/1000 [00:01<00:28, 34.21it/s]
  3%|▎         | 30/1000 [00:01<00:24, 40.04it/s]
  4%|▎         | 36/1000 [00:01<00:21, 44.53it/s]
  4%|▍         | 42/1000 [00:01<00:19, 47.95it/s]
  5%|▍         | 48/1000 [00:01<00:18, 50.51it/s]
  5%|▌         | 54/1000 [00:01<00:18, 52.36it/s]
  6%|▌         | 60/1000 [00:01<00:17, 53.70it/s]
  7%|▋         | 66/1000 [00:01<00:17, 54.71it/s]
  7%|▋         | 72/1000 [00:01<00:16, 55.40it/s]
  8%|▊         | 78/1000 [00:02<00:16, 55.82it/s]
  8%|▊         | 84/1000 [00:02<00:16, 56.31it/s]
  9%|▉         | 90/1000 [00:02<00:16, 56.61it/s]
 10%|▉         | 96/1000 [00:02<00:15, 56.98it/s]
 10%|█         | 102/1000 [00:02<00:15, 57.16it/s]
 11%|█         | 108/1000 [00:02<00:15, 57.42it/s]
 11%|█▏        | 114/1000 [00:02<00:15, 57.52it/s]
 12%|█▏        | 120/1000 [00:02<00:15, 57.51it/s]
 13%|█▎        | 126/1000 [00:02<00:15, 57.45it/s]
 13%|█▎        | 132/1000 [00:03<00:15, 57.45it/s]
 14%|█▍        | 138/1000 [00:03<00:15, 57.43it/s]
 14%|█▍        | 144/1000 [00:03<00:14, 57.47it/s]
 15%|█▌        | 150/1000 [00:03<00:14, 57.53it/s]
 16%|█▌        | 156/1000 [00:03<00:14, 57.51it/s]
 16%|█▌        | 162/1000 [00:03<00:14, 57.38it/s]
 17%|█▋        | 168/1000 [00:03<00:14, 57.25it/s]
 17%|█▋        | 174/1000 [00:03<00:14, 57.26it/s]
 18%|█▊        | 180/1000 [00:03<00:14, 57.21it/s]
 19%|█▊        | 186/1000 [00:03<00:14, 57.13it/s]
 19%|█▉        | 192/1000 [00:04<00:14, 57.12it/s]
 20%|█▉        | 198/1000 [00:04<00:14, 57.08it/s]
 20%|██        | 204/1000 [00:04<00:13, 56.89it/s]
 21%|██        | 210/1000 [00:04<00:13, 56.78it/s]
 22%|██▏       | 216/1000 [00:04<00:13, 56.89it/s]
 22%|██▏       | 222/1000 [00:04<00:13, 56.84it/s]
 23%|██▎       | 228/1000 [00:04<00:13, 57.05it/s]
 23%|██▎       | 234/1000 [00:04<00:13, 57.03it/s]
 24%|██▍       | 240/1000 [00:04<00:13, 56.95it/s]
 25%|██▍       | 246/1000 [00:05<00:13, 56.87it/s]
 25%|██▌       | 252/1000 [00:05<00:13, 56.97it/s]
 26%|██▌       | 258/1000 [00:05<00:13, 57.06it/s]
 26%|██▋       | 264/1000 [00:05<00:12, 57.14it/s]
 27%|██▋       | 270/1000 [00:05<00:12, 56.97it/s]
 28%|██▊       | 276/1000 [00:05<00:12, 57.02it/s]
 28%|██▊       | 282/1000 [00:05<00:12, 57.21it/s]
 29%|██▉       | 288/1000 [00:05<00:12, 57.33it/s]
 29%|██▉       | 294/1000 [00:05<00:12, 57.53it/s]
 30%|███       | 300/1000 [00:05<00:12, 57.52it/s]
 31%|███       | 306/1000 [00:06<00:12, 57.50it/s]
 31%|███       | 312/1000 [00:06<00:11, 57.61it/s]
 32%|███▏      | 318/1000 [00:06<00:11, 57.58it/s]
 32%|███▏      | 324/1000 [00:06<00:11, 57.50it/s]
 33%|███▎      | 330/1000 [00:06<00:11, 57.32it/s]
 34%|███▎      | 336/1000 [00:06<00:11, 57.38it/s]
 34%|███▍      | 342/1000 [00:06<00:11, 57.47it/s]
 35%|███▍      | 348/1000 [00:06<00:11, 57.57it/s]
 35%|███▌      | 354/1000 [00:06<00:11, 57.36it/s]
 36%|███▌      | 360/1000 [00:07<00:11, 57.37it/s]
 37%|███▋      | 366/1000 [00:07<00:11, 57.23it/s]
 37%|███▋      | 372/1000 [00:07<00:10, 57.37it/s]
 38%|███▊      | 378/1000 [00:07<00:11, 53.80it/s]
 38%|███▊      | 384/1000 [00:07<00:11, 54.52it/s]
 39%|███▉      | 390/1000 [00:07<00:11, 55.34it/s]
 40%|███▉      | 396/1000 [00:07<00:10, 55.97it/s]
 40%|████      | 402/1000 [00:07<00:10, 56.51it/s]
 41%|████      | 408/1000 [00:07<00:10, 56.86it/s]
 41%|████▏     | 414/1000 [00:07<00:10, 57.05it/s]
 42%|████▏     | 420/1000 [00:08<00:10, 57.20it/s]
 43%|████▎     | 426/1000 [00:08<00:10, 57.32it/s]
 43%|████▎     | 432/1000 [00:08<00:09, 57.48it/s]
 44%|████▍     | 438/1000 [00:08<00:09, 57.49it/s]
 44%|████▍     | 444/1000 [00:08<00:09, 57.32it/s]
 45%|████▌     | 450/1000 [00:08<00:09, 57.38it/s]
 46%|████▌     | 456/1000 [00:08<00:09, 57.46it/s]
 46%|████▌     | 462/1000 [00:08<00:09, 57.46it/s]
 47%|████▋     | 468/1000 [00:08<00:09, 57.44it/s]
 47%|████▋     | 474/1000 [00:09<00:09, 57.59it/s]
 48%|████▊     | 480/1000 [00:09<00:09, 57.52it/s]
 49%|████▊     | 486/1000 [00:09<00:08, 57.50it/s]
 49%|████▉     | 492/1000 [00:09<00:08, 57.50it/s]
 50%|████▉     | 498/1000 [00:09<00:08, 57.42it/s]
 50%|█████     | 504/1000 [00:09<00:08, 57.45it/s]
 51%|█████     | 510/1000 [00:09<00:08, 57.37it/s]
 52%|█████▏    | 516/1000 [00:09<00:08, 57.45it/s]
 52%|█████▏    | 522/1000 [00:09<00:08, 57.28it/s]
 53%|█████▎    | 528/1000 [00:09<00:08, 57.42it/s]
 53%|█████▎    | 534/1000 [00:10<00:08, 57.40it/s]
 54%|█████▍    | 540/1000 [00:10<00:07, 57.53it/s]
 55%|█████▍    | 546/1000 [00:10<00:07, 57.58it/s]
 55%|█████▌    | 552/1000 [00:10<00:07, 57.48it/s]
 56%|█████▌    | 558/1000 [00:10<00:07, 57.50it/s]
 56%|█████▋    | 564/1000 [00:10<00:07, 57.55it/s]
 57%|█████▋    | 570/1000 [00:10<00:07, 57.45it/s]
 58%|█████▊    | 576/1000 [00:10<00:07, 57.41it/s]
 58%|█████▊    | 582/1000 [00:10<00:07, 57.55it/s]
 59%|█████▉    | 588/1000 [00:10<00:07, 57.54it/s]
 59%|█████▉    | 594/1000 [00:11<00:07, 57.47it/s]
 60%|██████    | 600/1000 [00:11<00:06, 57.57it/s]
 61%|██████    | 606/1000 [00:11<00:06, 57.49it/s]
 61%|██████    | 612/1000 [00:11<00:06, 57.46it/s]
 62%|██████▏   | 618/1000 [00:11<00:06, 57.58it/s]
 62%|██████▏   | 624/1000 [00:11<00:06, 57.59it/s]
 63%|██████▎   | 630/1000 [00:11<00:06, 57.67it/s]
 64%|██████▎   | 636/1000 [00:11<00:06, 57.49it/s]
 64%|██████▍   | 642/1000 [00:11<00:06, 57.42it/s]
 65%|██████▍   | 648/1000 [00:12<00:06, 57.57it/s]
 65%|██████▌   | 654/1000 [00:12<00:06, 57.52it/s]
 66%|██████▌   | 660/1000 [00:12<00:05, 57.63it/s]
 67%|██████▋   | 666/1000 [00:12<00:05, 57.65it/s]
 67%|██████▋   | 672/1000 [00:12<00:05, 57.60it/s]
 68%|██████▊   | 678/1000 [00:12<00:05, 57.64it/s]
 68%|██████▊   | 684/1000 [00:12<00:05, 57.65it/s]
 69%|██████▉   | 690/1000 [00:12<00:05, 57.68it/s]
 70%|██████▉   | 696/1000 [00:12<00:05, 57.55it/s]
 70%|███████   | 702/1000 [00:12<00:05, 57.32it/s]
 71%|███████   | 708/1000 [00:13<00:05, 57.40it/s]
 71%|███████▏  | 714/1000 [00:13<00:04, 57.46it/s]
 72%|███████▏  | 720/1000 [00:13<00:04, 57.45it/s]
 73%|███████▎  | 726/1000 [00:13<00:04, 57.42it/s]
 73%|███████▎  | 732/1000 [00:13<00:04, 57.46it/s]
 74%|███████▍  | 738/1000 [00:13<00:04, 57.50it/s]
 74%|███████▍  | 744/1000 [00:13<00:04, 57.52it/s]
 75%|███████▌  | 750/1000 [00:13<00:04, 57.59it/s]
 76%|███████▌  | 756/1000 [00:13<00:04, 57.68it/s]
 76%|███████▌  | 762/1000 [00:14<00:04, 57.53it/s]
 77%|███████▋  | 768/1000 [00:14<00:04, 57.65it/s]
 77%|███████▋  | 774/1000 [00:14<00:03, 57.60it/s]
 78%|███████▊  | 780/1000 [00:14<00:03, 57.63it/s]
 79%|███████▊  | 786/1000 [00:14<00:03, 57.67it/s]
 79%|███████▉  | 792/1000 [00:14<00:03, 57.47it/s]
 80%|███████▉  | 798/1000 [00:14<00:03, 57.47it/s]
 80%|████████  | 804/1000 [00:14<00:03, 57.50it/s]
 81%|████████  | 810/1000 [00:14<00:03, 57.49it/s]
 82%|████████▏ | 816/1000 [00:14<00:03, 57.48it/s]
 82%|████████▏ | 822/1000 [00:15<00:03, 57.51it/s]
 83%|████████▎ | 828/1000 [00:15<00:02, 57.58it/s]
 83%|████████▎ | 834/1000 [00:15<00:02, 57.72it/s]
 84%|████████▍ | 840/1000 [00:15<00:02, 57.75it/s]
 85%|████████▍ | 846/1000 [00:15<00:02, 57.67it/s]
 85%|████████▌ | 852/1000 [00:15<00:02, 57.72it/s]
 86%|████████▌ | 858/1000 [00:15<00:02, 57.66it/s]
 86%|████████▋ | 864/1000 [00:15<00:02, 57.67it/s]
 87%|████████▋ | 870/1000 [00:15<00:02, 57.62it/s]
 88%|████████▊ | 876/1000 [00:16<00:02, 57.67it/s]
 88%|████████▊ | 882/1000 [00:16<00:02, 57.64it/s]
 89%|████████▉ | 888/1000 [00:16<00:01, 57.54it/s]
 89%|████████▉ | 894/1000 [00:16<00:01, 57.66it/s]
 90%|█████████ | 900/1000 [00:16<00:01, 57.65it/s]
 91%|█████████ | 906/1000 [00:16<00:01, 57.61it/s]
 91%|█████████ | 912/1000 [00:16<00:01, 57.61it/s]
 92%|█████████▏| 918/1000 [00:16<00:01, 57.59it/s]
 92%|█████████▏| 924/1000 [00:16<00:01, 57.57it/s]
 93%|█████████▎| 930/1000 [00:16<00:01, 57.66it/s]
 94%|█████████▎| 936/1000 [00:17<00:01, 57.71it/s]
 94%|█████████▍| 942/1000 [00:17<00:01, 57.69it/s]
 95%|█████████▍| 948/1000 [00:17<00:00, 57.70it/s]
 95%|█████████▌| 954/1000 [00:17<00:00, 57.73it/s]
 96%|█████████▌| 960/1000 [00:17<00:00, 57.67it/s]
 97%|█████████▋| 966/1000 [00:17<00:00, 57.67it/s]
 97%|█████████▋| 972/1000 [00:17<00:00, 57.51it/s]
 98%|█████████▊| 978/1000 [00:17<00:00, 57.56it/s]
 98%|█████████▊| 984/1000 [00:17<00:00, 57.61it/s]
 99%|█████████▉| 990/1000 [00:17<00:00, 57.36it/s]
-2022-12-21,11:03:10 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:03<09:51, 84.42it/s]
  1%|          | 512/50176 [00:05<08:58, 92.20it/s]
  2%|▏         | 768/50176 [00:07<08:17, 99.26it/s]
  2%|▏         | 1024/50176 [00:10<08:15, 99.21it/s]
  3%|▎         | 1280/50176 [00:13<08:13, 99.12it/s]
  3%|▎         | 1536/50176 [00:16<08:48, 91.99it/s]
  4%|▎         | 1792/50176 [00:18<08:36, 93.74it/s]
  4%|▍         | 2048/50176 [00:21<08:17, 96.76it/s]
  5%|▍         | 2304/50176 [00:23<08:07, 98.26it/s]
  5%|▌         | 2560/50176 [00:26<07:51, 100.97it/s]
  6%|▌         | 2816/50176 [00:29<08:00, 98.49it/s] 
  6%|▌         | 3072/50176 [00:31<07:53, 99.57it/s]
  7%|▋         | 3328/50176 [00:33<07:39, 101.96it/s]
  7%|▋         | 3584/50176 [00:36<07:29, 103.56it/s]
  8%|▊         | 3840/50176 [00:38<07:38, 100.98it/s]
  8%|▊         | 4096/50176 [00:41<07:53, 97.22it/s] 
  9%|▊         | 4352/50176 [00:44<07:49, 97.63it/s]
  9%|▉         | 4608/50176 [00:46<07:32, 100.81it/s]
 10%|▉         | 4864/50176 [00:49<07:24, 101.90it/s]
 10%|█         | 5120/50176 [00:51<07:21, 102.13it/s]
 11%|█         | 5376/50176 [00:54<07:26, 100.29it/s]
 11%|█         | 5632/50176 [00:56<07:18, 101.56it/s]
 12%|█▏        | 5888/50176 [00:59<07:20, 100.52it/s]
 12%|█▏        | 6144/50176 [01:02<08:01, 91.51it/s] 
 13%|█▎        | 6400/50176 [01:05<07:41, 94.82it/s]
 13%|█▎        | 6656/50176 [01:07<07:26, 97.46it/s]
 14%|█▍        | 6912/50176 [01:10<07:34, 95.23it/s]
 14%|█▍        | 7168/50176 [01:13<07:22, 97.18it/s]
 15%|█▍        | 7424/50176 [01:15<07:03, 100.86it/s]
 15%|█▌        | 7680/50176 [01:17<06:56, 101.99it/s]
 16%|█▌        | 7936/50176 [01:20<06:45, 104.17it/s]
 16%|█▋        | 8192/50176 [01:22<06:50, 102.29it/s]
 17%|█▋        | 8448/50176 [01:25<06:45, 102.85it/s]
 17%|█▋        | 8704/50176 [01:27<06:47, 101.76it/s]
 18%|█▊        | 8960/50176 [01:30<06:58, 98.39it/s] 
 18%|█▊        | 9216/50176 [01:32<06:38, 102.84it/s]
 19%|█▉        | 9472/50176 [01:35<06:34, 103.21it/s]
 19%|█▉        | 9728/50176 [01:38<06:42, 100.38it/s]
 20%|█▉        | 9984/50176 [01:40<06:41, 100.09it/s]
 20%|██        | 10240/50176 [01:42<06:27, 102.95it/s]
 21%|██        | 10496/50176 [01:45<06:27, 102.39it/s]
 21%|██▏       | 10752/50176 [01:48<06:29, 101.16it/s]
 22%|██▏       | 11008/50176 [01:50<06:26, 101.27it/s]
 22%|██▏       | 11264/50176 [01:53<06:22, 101.79it/s]
 23%|██▎       | 11520/50176 [01:55<06:12, 103.73it/s]
 23%|██▎       | 11776/50176 [01:57<06:11, 103.50it/s]
 24%|██▍       | 12032/50176 [02:00<06:15, 101.68it/s]
 24%|██▍       | 12288/50176 [02:02<06:08, 102.71it/s]
 25%|██▌       | 12544/50176 [02:05<06:14, 100.50it/s]
 26%|██▌       | 12800/50176 [02:08<06:17, 99.01it/s] 
 26%|██▌       | 13056/50176 [02:10<06:03, 102.07it/s]
 27%|██▋       | 13312/50176 [02:13<05:58, 102.84it/s]
 27%|██▋       | 13568/50176 [02:16<06:23, 95.55it/s] 
 28%|██▊       | 13824/50176 [02:19<06:28, 93.55it/s]
 28%|██▊       | 14080/50176 [02:21<06:24, 93.79it/s]
 29%|██▊       | 14336/50176 [02:24<06:19, 94.32it/s]
 29%|██▉       | 14592/50176 [02:26<06:01, 98.36it/s]
 30%|██▉       | 14848/50176 [02:29<06:06, 96.35it/s]
 30%|███       | 15104/50176 [02:32<06:07, 95.51it/s]
 31%|███       | 15360/50176 [02:34<05:55, 97.83it/s]
 31%|███       | 15616/50176 [02:37<05:46, 99.76it/s]
 32%|███▏      | 15872/50176 [02:39<05:47, 98.83it/s]
 32%|███▏      | 16128/50176 [02:42<05:39, 100.23it/s]
 33%|███▎      | 16384/50176 [02:44<05:31, 101.83it/s]
 33%|███▎      | 16640/50176 [02:47<05:42, 97.91it/s] 
 34%|███▎      | 16896/50176 [02:50<05:36, 98.89it/s]
 34%|███▍      | 17152/50176 [02:52<05:25, 101.48it/s]
 35%|███▍      | 17408/50176 [02:55<05:23, 101.29it/s]
 35%|███▌      | 17664/50176 [02:57<05:16, 102.64it/s]
 36%|███▌      | 17920/50176 [02:59<05:09, 104.11it/s]
 36%|███▌      | 18176/50176 [03:02<05:13, 102.10it/s]
 37%|███▋      | 18432/50176 [03:05<05:14, 100.91it/s]
 37%|███▋      | 18688/50176 [03:07<05:15, 99.73it/s] 
 38%|███▊      | 18944/50176 [03:10<05:16, 98.61it/s]
 38%|███▊      | 19200/50176 [03:13<05:26, 94.84it/s]
 39%|███▉      | 19456/50176 [03:15<05:17, 96.64it/s]
 39%|███▉      | 19712/50176 [03:18<05:19, 95.35it/s]
 40%|███▉      | 19968/50176 [03:21<05:13, 96.45it/s]
 40%|████      | 20224/50176 [03:23<04:52, 102.27it/s]
 41%|████      | 20480/50176 [03:25<04:49, 102.58it/s]
 41%|████▏     | 20736/50176 [03:28<04:42, 104.03it/s]
 42%|████▏     | 20992/50176 [03:30<04:36, 105.56it/s]
 42%|████▏     | 21248/50176 [03:33<04:40, 103.27it/s]
 43%|████▎     | 21504/50176 [03:35<04:35, 103.95it/s]
 43%|████▎     | 21760/50176 [03:38<04:33, 103.72it/s]
 44%|████▍     | 22016/50176 [03:40<04:24, 106.42it/s]
 44%|████▍     | 22272/50176 [03:42<04:19, 107.35it/s]
 45%|████▍     | 22528/50176 [03:45<04:18, 106.78it/s]
 45%|████▌     | 22784/50176 [03:47<04:23, 103.81it/s]
 46%|████▌     | 23040/50176 [03:50<04:26, 101.67it/s]
 46%|████▋     | 23296/50176 [03:53<04:28, 100.01it/s]
 47%|████▋     | 23552/50176 [03:55<04:35, 96.77it/s] 
 47%|████▋     | 23808/50176 [03:58<04:19, 101.47it/s]
 48%|████▊     | 24064/50176 [04:00<04:19, 100.49it/s]
 48%|████▊     | 24320/50176 [04:03<04:18, 100.12it/s]
 49%|████▉     | 24576/50176 [04:05<04:17, 99.43it/s] 
 49%|████▉     | 24832/50176 [04:08<04:09, 101.77it/s]
 50%|█████     | 25088/50176 [04:10<04:00, 104.18it/s]
 51%|█████     | 25344/50176 [04:13<04:00, 103.33it/s]
 51%|█████     | 25600/50176 [04:15<03:57, 103.48it/s]
 52%|█████▏    | 25856/50176 [04:18<03:53, 104.32it/s]
 52%|█████▏    | 26112/50176 [04:20<03:48, 105.10it/s]
 53%|█████▎    | 26368/50176 [04:23<03:57, 100.29it/s]
 53%|█████▎    | 26624/50176 [04:25<03:54, 100.58it/s]
 54%|█████▎    | 26880/50176 [04:28<03:52, 100.29it/s]
 54%|█████▍    | 27136/50176 [04:30<03:49, 100.21it/s]
 55%|█████▍    | 27392/50176 [04:33<03:43, 102.09it/s]
 55%|█████▌    | 27648/50176 [04:35<03:34, 105.26it/s]
 56%|█████▌    | 27904/50176 [04:38<03:39, 101.53it/s]
 56%|█████▌    | 28160/50176 [04:40<03:42, 99.10it/s] 
 57%|█████▋    | 28416/50176 [04:43<03:37, 99.90it/s]
 57%|█████▋    | 28672/50176 [04:46<03:36, 99.10it/s]
 58%|█████▊    | 28928/50176 [04:48<03:29, 101.57it/s]
 58%|█████▊    | 29184/50176 [04:50<03:23, 103.14it/s]
 59%|█████▊    | 29440/50176 [04:53<03:33, 96.98it/s] 
 59%|█████▉    | 29696/50176 [04:56<03:30, 97.20it/s]
 60%|█████▉    | 29952/50176 [04:59<03:29, 96.49it/s]
 60%|██████    | 30208/50176 [05:01<03:23, 98.16it/s]
 61%|██████    | 30464/50176 [05:03<03:12, 102.41it/s]
 61%|██████    | 30720/50176 [05:06<03:11, 101.81it/s]
 62%|██████▏   | 30976/50176 [05:08<03:05, 103.30it/s]
 62%|██████▏   | 31232/50176 [05:11<02:58, 106.13it/s]
 63%|██████▎   | 31488/50176 [05:13<02:55, 106.36it/s]
 63%|██████▎   | 31744/50176 [05:16<02:55, 105.28it/s]
 64%|██████▍   | 32000/50176 [05:18<02:52, 105.36it/s]
 64%|██████▍   | 32256/50176 [05:20<02:49, 105.45it/s]
 65%|██████▍   | 32512/50176 [05:23<02:56, 100.28it/s]
 65%|██████▌   | 32768/50176 [05:26<02:49, 102.76it/s]
 66%|██████▌   | 33024/50176 [05:28<02:48, 101.83it/s]
 66%|██████▋   | 33280/50176 [05:31<02:46, 101.70it/s]
 67%|██████▋   | 33536/50176 [05:33<02:46, 100.10it/s]
 67%|██████▋   | 33792/50176 [05:36<02:40, 102.08it/s]
 68%|██████▊   | 34048/50176 [05:38<02:33, 105.08it/s]
 68%|██████▊   | 34304/50176 [05:40<02:28, 107.18it/s]
 69%|██████▉   | 34560/50176 [05:43<02:27, 105.87it/s]
 69%|██████▉   | 34816/50176 [05:45<02:24, 105.99it/s]
 70%|██████▉   | 35072/50176 [05:48<02:25, 103.66it/s]
 70%|███████   | 35328/50176 [05:50<02:19, 106.49it/s]
 71%|███████   | 35584/50176 [05:52<02:16, 106.68it/s]
 71%|███████▏  | 35840/50176 [05:55<02:17, 104.52it/s]
 72%|███████▏  | 36096/50176 [05:57<02:14, 104.38it/s]
 72%|███████▏  | 36352/50176 [06:00<02:13, 103.58it/s]
 73%|███████▎  | 36608/50176 [06:03<02:12, 102.10it/s]
 73%|███████▎  | 36864/50176 [06:05<02:13, 99.96it/s] 
 74%|███████▍  | 37120/50176 [06:08<02:11, 99.37it/s]
 74%|███████▍  | 37376/50176 [06:10<02:01, 105.23it/s]
 75%|███████▌  | 37632/50176 [06:12<01:56, 107.51it/s]
 76%|███████▌  | 37888/50176 [06:15<01:54, 107.64it/s]
 76%|███████▌  | 38144/50176 [06:17<01:52, 107.07it/s]
 77%|███████▋  | 38400/50176 [06:20<01:52, 104.84it/s]
 77%|███████▋  | 38656/50176 [06:22<01:50, 104.38it/s]
 78%|███████▊  | 38912/50176 [06:25<01:48, 104.02it/s]
 78%|███████▊  | 39168/50176 [06:27<01:49, 100.64it/s]
 79%|███████▊  | 39424/50176 [06:30<01:44, 102.70it/s]
 79%|███████▉  | 39680/50176 [06:32<01:43, 101.85it/s]
 80%|███████▉  | 39936/50176 [06:35<01:38, 104.31it/s]
 80%|████████  | 40192/50176 [06:37<01:37, 102.69it/s]
 81%|████████  | 40448/50176 [06:40<01:38, 98.31it/s] 
 81%|████████  | 40704/50176 [06:43<01:39, 95.33it/s]
 82%|████████▏ | 40960/50176 [06:45<01:32, 99.69it/s]
 82%|████████▏ | 41216/50176 [06:48<01:29, 100.00it/s]
 83%|████████▎ | 41472/50176 [06:50<01:26, 100.84it/s]
 83%|████████▎ | 41728/50176 [06:53<01:22, 102.99it/s]
 84%|████████▎ | 41984/50176 [06:55<01:19, 103.21it/s]
 84%|████████▍ | 42240/50176 [06:58<01:17, 102.63it/s]
 85%|████████▍ | 42496/50176 [07:00<01:12, 106.34it/s]
 85%|████████▌ | 42752/50176 [07:02<01:09, 107.01it/s]
 86%|████████▌ | 43008/50176 [07:05<01:07, 105.93it/s]
 86%|████████▌ | 43264/50176 [07:07<01:05, 105.36it/s]
 87%|████████▋ | 43520/50176 [07:10<01:04, 103.27it/s]
 87%|████████▋ | 43776/50176 [07:12<01:02, 102.26it/s]
 88%|████████▊ | 44032/50176 [07:15<01:00, 101.81it/s]
 88%|████████▊ | 44288/50176 [07:17<00:56, 104.28it/s]
 89%|████████▉ | 44544/50176 [07:20<00:56, 100.37it/s]
 89%|████████▉ | 44800/50176 [07:22<00:52, 102.11it/s]
 90%|████████▉ | 45056/50176 [07:24<00:48, 105.97it/s]
 90%|█████████ | 45312/50176 [07:27<00:46, 104.93it/s]
 91%|█████████ | 45568/50176 [07:29<00:43, 107.05it/s]
 91%|█████████▏| 45824/50176 [07:32<00:41, 105.61it/s]
 92%|█████████▏| 46080/50176 [07:34<00:39, 103.14it/s]
 92%|█████████▏| 46336/50176 [07:37<00:37, 102.58it/s]
 93%|█████████▎| 46592/50176 [07:39<00:33, 105.57it/s]
 93%|█████████▎| 46848/50176 [07:41<00:31, 106.69it/s]
 94%|█████████▍| 47104/50176 [07:44<00:30, 101.65it/s]
 94%|█████████▍| 47360/50176 [07:47<00:28, 99.14it/s] 
 95%|█████████▍| 47616/50176 [07:50<00:25, 99.14it/s]
 95%|█████████▌| 47872/50176 [07:52<00:22, 102.98it/s]
 96%|█████████▌| 48128/50176 [07:54<00:19, 105.61it/s]
 96%|█████████▋| 48384/50176 [07:57<00:17, 102.99it/s]
 97%|█████████▋| 48640/50176 [07:59<00:14, 102.46it/s]
 97%|█████████▋| 48896/50176 [08:02<00:12, 101.79it/s]
 98%|█████████▊| 49152/50176 [08:04<00:10, 100.30it/s]
 98%|█████████▊| 49408/50176 [08:07<00:07, 101.26it/s]
 99%|█████████▉| 49664/50176 [08:09<00:05, 100.90it/s]
 99%|█████████▉| 49920/50176 [08:13<00:02, 94.72it/s] 
-2022-12-21,11:11:23 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:11:23 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6474	imagenet-zeroshot-val-top5: 0.8909

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_53.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
-2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt).
-2022-12-21,11:11:47 | INFO | Model:
-2022-12-21,11:11:47 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,11:11:47 | INFO | Params:
-2022-12-21,11:11:47 | INFO |   batch_size: 256
-2022-12-21,11:11:47 | INFO |   beta1: 0.9
-2022-12-21,11:11:47 | INFO |   beta2: 0.98
-2022-12-21,11:11:47 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/checkpoints
-2022-12-21,11:11:47 | INFO |   copy_codebase: False
-2022-12-21,11:11:47 | INFO |   csv_caption_key: title
-2022-12-21,11:11:47 | INFO |   csv_img_key: filepath
-2022-12-21,11:11:47 | INFO |   csv_separator: 	
-2022-12-21,11:11:47 | INFO |   dataset_resampled: False
-2022-12-21,11:11:47 | INFO |   dataset_type: auto
-2022-12-21,11:11:47 | INFO |   ddp_static_graph: False
-2022-12-21,11:11:47 | INFO |   debug: False
-2022-12-21,11:11:47 | INFO |   device: cuda:0
-2022-12-21,11:11:47 | INFO |   dist_backend: nccl
-2022-12-21,11:11:47 | INFO |   dist_url: env://
-2022-12-21,11:11:47 | INFO |   distributed: False
-2022-12-21,11:11:47 | INFO |   epochs: 32
-2022-12-21,11:11:47 | INFO |   eps: 1e-06
-2022-12-21,11:11:47 | INFO |   force_quick_gelu: False
-2022-12-21,11:11:47 | INFO |   gather_with_grad: False
-2022-12-21,11:11:47 | INFO |   grad_checkpointing: False
-2022-12-21,11:11:47 | INFO |   horovod: False
-2022-12-21,11:11:47 | INFO |   image_mean: None
-2022-12-21,11:11:47 | INFO |   image_std: None
-2022-12-21,11:11:47 | INFO |   imagenet_v2: None
-2022-12-21,11:11:47 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,11:11:47 | INFO |   local_loss: False
-2022-12-21,11:11:47 | INFO |   local_rank: 0
-2022-12-21,11:11:47 | INFO |   lock_image: False
-2022-12-21,11:11:47 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,11:11:47 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,11:11:47 | INFO |   log_level: 20
-2022-12-21,11:11:47 | INFO |   log_local: False
-2022-12-21,11:11:47 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_53.pt/out.log
-2022-12-21,11:11:47 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,11:11:47 | INFO |   lr: 0.0005
-2022-12-21,11:11:47 | INFO |   model: ViT-B-16
-2022-12-21,11:11:47 | INFO |   name: epoch_53.pt
-2022-12-21,11:11:47 | INFO |   no_set_device_rank: False
-2022-12-21,11:11:47 | INFO |   norm_gradient_clip: None
-2022-12-21,11:11:47 | INFO |   precision: amp_bfloat16
-2022-12-21,11:11:47 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_53.pt
-2022-12-21,11:11:47 | INFO |   pretrained_image: False
-2022-12-21,11:11:47 | INFO |   rank: 0
-2022-12-21,11:11:47 | INFO |   report_to: 
-2022-12-21,11:11:47 | INFO |   resume: None
-2022-12-21,11:11:47 | INFO |   save_frequency: 1
-2022-12-21,11:11:47 | INFO |   save_most_recent: False
-2022-12-21,11:11:47 | INFO |   seed: 0
-2022-12-21,11:11:47 | INFO |   skip_scheduler: False
-2022-12-21,11:11:47 | INFO |   tensorboard: False
-2022-12-21,11:11:47 | INFO |   tensorboard_path: 
-2022-12-21,11:11:47 | INFO |   torchscript: False
-2022-12-21,11:11:47 | INFO |   trace: False
-2022-12-21,11:11:47 | INFO |   train_data: None
-2022-12-21,11:11:47 | INFO |   train_num_samples: None
-2022-12-21,11:11:47 | INFO |   use_bn_sync: False
-2022-12-21,11:11:47 | INFO |   val_data: None
-2022-12-21,11:11:47 | INFO |   val_frequency: 1
-2022-12-21,11:11:47 | INFO |   val_num_samples: None
-2022-12-21,11:11:47 | INFO |   wandb: False
-2022-12-21,11:11:47 | INFO |   wandb_notes: 
-2022-12-21,11:11:47 | INFO |   warmup: 10000
-2022-12-21,11:11:47 | INFO |   wd: 0.2
-2022-12-21,11:11:47 | INFO |   workers: 1
-2022-12-21,11:11:47 | INFO |   world_size: 1
-2022-12-21,11:11:47 | INFO |   zeroshot_frequency: 2
-2022-12-21,11:11:47 | INFO | Starting zero-shot imagenet.
-2022-12-21,11:11:47 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<10:07,  1.64it/s]
  1%|          | 7/1000 [00:00<01:20, 12.27it/s]
  1%|▏         | 13/1000 [00:00<00:44, 21.95it/s]
  2%|▏         | 19/1000 [00:00<00:32, 30.29it/s]
  2%|▎         | 25/1000 [00:01<00:26, 37.11it/s]
  3%|▎         | 31/1000 [00:01<00:22, 42.58it/s]
  4%|▎         | 37/1000 [00:01<00:20, 46.72it/s]
  4%|▍         | 43/1000 [00:01<00:19, 49.76it/s]
  5%|▍         | 49/1000 [00:01<00:18, 51.96it/s]
  6%|▌         | 55/1000 [00:01<00:17, 53.42it/s]
  6%|▌         | 61/1000 [00:01<00:17, 54.75it/s]
  7%|▋         | 67/1000 [00:01<00:16, 55.60it/s]
  7%|▋         | 73/1000 [00:01<00:16, 56.12it/s]
  8%|▊         | 79/1000 [00:01<00:16, 56.65it/s]
  8%|▊         | 85/1000 [00:02<00:16, 57.11it/s]
  9%|▉         | 91/1000 [00:02<00:15, 57.35it/s]
 10%|▉         | 97/1000 [00:02<00:15, 57.70it/s]
 10%|█         | 103/1000 [00:02<00:15, 57.84it/s]
 11%|█         | 109/1000 [00:02<00:15, 58.05it/s]
 12%|█▏        | 115/1000 [00:02<00:15, 58.20it/s]
 12%|█▏        | 121/1000 [00:02<00:15, 58.11it/s]
 13%|█▎        | 127/1000 [00:02<00:15, 58.07it/s]
 13%|█▎        | 133/1000 [00:02<00:14, 58.09it/s]
 14%|█▍        | 139/1000 [00:03<00:14, 58.04it/s]
 14%|█▍        | 145/1000 [00:03<00:14, 58.08it/s]
 15%|█▌        | 151/1000 [00:03<00:14, 58.12it/s]
 16%|█▌        | 157/1000 [00:03<00:14, 58.08it/s]
 16%|█▋        | 163/1000 [00:03<00:14, 58.06it/s]
 17%|█▋        | 169/1000 [00:03<00:14, 57.83it/s]
 18%|█▊        | 175/1000 [00:03<00:14, 57.84it/s]
 18%|█▊        | 181/1000 [00:03<00:14, 57.80it/s]
 19%|█▊        | 187/1000 [00:03<00:14, 57.78it/s]
 19%|█▉        | 193/1000 [00:03<00:13, 57.73it/s]
 20%|█▉        | 199/1000 [00:04<00:13, 57.65it/s]
 20%|██        | 205/1000 [00:04<00:13, 57.49it/s]
 21%|██        | 211/1000 [00:04<00:13, 57.32it/s]
 22%|██▏       | 217/1000 [00:04<00:13, 57.53it/s]
 22%|██▏       | 223/1000 [00:04<00:13, 57.49it/s]
 23%|██▎       | 229/1000 [00:04<00:13, 57.65it/s]
 24%|██▎       | 235/1000 [00:04<00:13, 57.64it/s]
 24%|██▍       | 241/1000 [00:04<00:13, 57.47it/s]
 25%|██▍       | 247/1000 [00:04<00:13, 57.61it/s]
 25%|██▌       | 253/1000 [00:04<00:12, 57.70it/s]
 26%|██▌       | 259/1000 [00:05<00:12, 57.81it/s]
 26%|██▋       | 265/1000 [00:05<00:12, 57.78it/s]
 27%|██▋       | 271/1000 [00:05<00:12, 57.63it/s]
 28%|██▊       | 277/1000 [00:05<00:12, 57.77it/s]
 28%|██▊       | 283/1000 [00:05<00:12, 57.90it/s]
 29%|██▉       | 289/1000 [00:05<00:12, 58.02it/s]
 30%|██▉       | 295/1000 [00:05<00:12, 58.19it/s]
 30%|███       | 301/1000 [00:05<00:12, 58.18it/s]
 31%|███       | 307/1000 [00:05<00:11, 58.14it/s]
 31%|███▏      | 313/1000 [00:06<00:11, 58.31it/s]
 32%|███▏      | 319/1000 [00:06<00:11, 58.33it/s]
 32%|███▎      | 325/1000 [00:06<00:11, 58.15it/s]
 33%|███▎      | 331/1000 [00:06<00:11, 57.67it/s]
 34%|███▎      | 337/1000 [00:06<00:11, 57.89it/s]
 34%|███▍      | 343/1000 [00:06<00:11, 58.02it/s]
 35%|███▍      | 349/1000 [00:06<00:11, 55.81it/s]
 36%|███▌      | 355/1000 [00:06<00:11, 56.44it/s]
 36%|███▌      | 361/1000 [00:06<00:11, 56.97it/s]
 37%|███▋      | 367/1000 [00:06<00:11, 57.36it/s]
 37%|███▋      | 373/1000 [00:07<00:10, 57.70it/s]
 38%|███▊      | 379/1000 [00:07<00:10, 57.78it/s]
 38%|███▊      | 385/1000 [00:07<00:10, 57.82it/s]
 39%|███▉      | 391/1000 [00:07<00:10, 57.60it/s]
 40%|███▉      | 397/1000 [00:07<00:10, 57.73it/s]
 40%|████      | 403/1000 [00:07<00:10, 57.86it/s]
 41%|████      | 409/1000 [00:07<00:10, 57.98it/s]
 42%|████▏     | 415/1000 [00:07<00:10, 58.08it/s]
 42%|████▏     | 421/1000 [00:07<00:09, 58.13it/s]
 43%|████▎     | 427/1000 [00:07<00:09, 58.16it/s]
 43%|████▎     | 433/1000 [00:08<00:09, 58.29it/s]
 44%|████▍     | 439/1000 [00:08<00:09, 58.29it/s]
 44%|████▍     | 445/1000 [00:08<00:09, 58.04it/s]
 45%|████▌     | 451/1000 [00:08<00:09, 58.13it/s]
 46%|████▌     | 457/1000 [00:08<00:09, 58.16it/s]
 46%|████▋     | 463/1000 [00:08<00:09, 58.19it/s]
 47%|████▋     | 469/1000 [00:08<00:09, 58.11it/s]
 48%|████▊     | 475/1000 [00:08<00:09, 58.27it/s]
 48%|████▊     | 481/1000 [00:08<00:08, 58.09it/s]
 49%|████▊     | 487/1000 [00:09<00:08, 58.18it/s]
 49%|████▉     | 493/1000 [00:09<00:08, 57.87it/s]
 50%|████▉     | 499/1000 [00:09<00:08, 57.62it/s]
 50%|█████     | 505/1000 [00:09<00:08, 57.79it/s]
 51%|█████     | 511/1000 [00:09<00:08, 57.77it/s]
 52%|█████▏    | 517/1000 [00:09<00:08, 57.95it/s]
 52%|█████▏    | 523/1000 [00:09<00:08, 57.97it/s]
 53%|█████▎    | 529/1000 [00:09<00:08, 58.06it/s]
 54%|█████▎    | 535/1000 [00:09<00:07, 58.14it/s]
 54%|█████▍    | 541/1000 [00:09<00:08, 57.37it/s]
 55%|█████▍    | 547/1000 [00:10<00:07, 57.57it/s]
 55%|█████▌    | 553/1000 [00:10<00:07, 57.66it/s]
 56%|█████▌    | 559/1000 [00:10<00:07, 57.82it/s]
 56%|█████▋    | 565/1000 [00:10<00:07, 57.59it/s]
 57%|█████▋    | 571/1000 [00:10<00:07, 57.62it/s]
 58%|█████▊    | 577/1000 [00:10<00:07, 57.82it/s]
 58%|█████▊    | 583/1000 [00:10<00:07, 57.99it/s]
 59%|█████▉    | 589/1000 [00:10<00:07, 57.98it/s]
 60%|█████▉    | 595/1000 [00:10<00:06, 57.96it/s]
 60%|██████    | 601/1000 [00:11<00:06, 58.10it/s]
 61%|██████    | 607/1000 [00:11<00:06, 57.98it/s]
 61%|██████▏   | 613/1000 [00:11<00:06, 58.11it/s]
 62%|██████▏   | 619/1000 [00:11<00:06, 58.32it/s]
 62%|██████▎   | 625/1000 [00:11<00:06, 58.28it/s]
 63%|██████▎   | 631/1000 [00:11<00:06, 58.25it/s]
 64%|██████▎   | 637/1000 [00:11<00:06, 57.69it/s]
 64%|██████▍   | 643/1000 [00:11<00:06, 57.79it/s]
 65%|██████▍   | 649/1000 [00:11<00:06, 57.99it/s]
 66%|██████▌   | 655/1000 [00:11<00:05, 58.06it/s]
 66%|██████▌   | 661/1000 [00:12<00:05, 57.64it/s]
 67%|██████▋   | 667/1000 [00:12<00:05, 57.78it/s]
 67%|██████▋   | 673/1000 [00:12<00:05, 57.95it/s]
 68%|██████▊   | 679/1000 [00:12<00:05, 57.82it/s]
 68%|██████▊   | 685/1000 [00:12<00:05, 57.83it/s]
 69%|██████▉   | 691/1000 [00:12<00:05, 57.95it/s]
 70%|██████▉   | 697/1000 [00:12<00:05, 57.96it/s]
 70%|███████   | 703/1000 [00:12<00:05, 57.87it/s]
 71%|███████   | 709/1000 [00:12<00:05, 57.97it/s]
 72%|███████▏  | 715/1000 [00:12<00:04, 58.03it/s]
 72%|███████▏  | 721/1000 [00:13<00:04, 58.08it/s]
 73%|███████▎  | 727/1000 [00:13<00:04, 57.91it/s]
 73%|███████▎  | 733/1000 [00:13<00:04, 57.97it/s]
 74%|███████▍  | 739/1000 [00:13<00:04, 58.08it/s]
 74%|███████▍  | 745/1000 [00:13<00:04, 58.15it/s]
 75%|███████▌  | 751/1000 [00:13<00:04, 58.24it/s]
 76%|███████▌  | 757/1000 [00:13<00:04, 58.30it/s]
 76%|███████▋  | 763/1000 [00:13<00:04, 58.17it/s]
 77%|███████▋  | 769/1000 [00:13<00:03, 58.24it/s]
 78%|███████▊  | 775/1000 [00:13<00:03, 58.21it/s]
 78%|███████▊  | 781/1000 [00:14<00:03, 58.23it/s]
 79%|███████▊  | 787/1000 [00:14<00:03, 58.24it/s]
 79%|███████▉  | 793/1000 [00:14<00:03, 58.11it/s]
 80%|███████▉  | 799/1000 [00:14<00:03, 58.08it/s]
 80%|████████  | 805/1000 [00:14<00:03, 58.12it/s]
 81%|████████  | 811/1000 [00:14<00:03, 58.05it/s]
 82%|████████▏ | 817/1000 [00:14<00:03, 58.13it/s]
 82%|████████▏ | 823/1000 [00:14<00:03, 58.09it/s]
 83%|████████▎ | 829/1000 [00:14<00:02, 58.21it/s]
 84%|████████▎ | 835/1000 [00:15<00:02, 58.37it/s]
 84%|████████▍ | 841/1000 [00:15<00:02, 58.42it/s]
 85%|████████▍ | 847/1000 [00:15<00:02, 58.30it/s]
 85%|████████▌ | 853/1000 [00:15<00:02, 58.35it/s]
 86%|████████▌ | 859/1000 [00:15<00:02, 58.34it/s]
 86%|████████▋ | 865/1000 [00:15<00:02, 58.34it/s]
 87%|████████▋ | 871/1000 [00:15<00:02, 58.25it/s]
 88%|████████▊ | 877/1000 [00:15<00:02, 58.30it/s]
 88%|████████▊ | 883/1000 [00:15<00:02, 58.26it/s]
 89%|████████▉ | 889/1000 [00:15<00:01, 58.18it/s]
 90%|████████▉ | 895/1000 [00:16<00:01, 58.26it/s]
 90%|█████████ | 901/1000 [00:16<00:01, 58.24it/s]
 91%|█████████ | 907/1000 [00:16<00:01, 58.17it/s]
 91%|█████████▏| 913/1000 [00:16<00:01, 58.13it/s]
 92%|█████████▏| 919/1000 [00:16<00:01, 58.23it/s]
 92%|█████████▎| 925/1000 [00:16<00:01, 58.18it/s]
 93%|█████████▎| 931/1000 [00:16<00:01, 58.27it/s]
 94%|█████████▎| 937/1000 [00:16<00:01, 58.31it/s]
 94%|█████████▍| 943/1000 [00:16<00:00, 58.21it/s]
 95%|█████████▍| 949/1000 [00:16<00:00, 58.00it/s]
 96%|█████████▌| 955/1000 [00:17<00:00, 58.19it/s]
 96%|█████████▌| 961/1000 [00:17<00:00, 58.12it/s]
 97%|█████████▋| 967/1000 [00:17<00:00, 58.24it/s]
 97%|█████████▋| 973/1000 [00:17<00:00, 58.39it/s]
 98%|█████████▊| 979/1000 [00:17<00:00, 58.42it/s]
 98%|█████████▊| 985/1000 [00:17<00:00, 58.37it/s]
 99%|█████████▉| 991/1000 [00:17<00:00, 58.27it/s]
-2022-12-21,11:12:05 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:04<14:36, 56.97it/s]
  1%|          | 512/50176 [00:07<11:03, 74.86it/s]
  2%|▏         | 768/50176 [00:09<09:47, 84.07it/s]
  2%|▏         | 1024/50176 [00:12<09:17, 88.18it/s]
  3%|▎         | 1280/50176 [00:15<09:13, 88.34it/s]
  3%|▎         | 1536/50176 [00:18<09:30, 85.31it/s]
  4%|▎         | 1792/50176 [00:21<09:16, 86.95it/s]
  4%|▍         | 2048/50176 [00:23<08:48, 91.09it/s]
  5%|▍         | 2304/50176 [00:26<08:48, 90.59it/s]
  5%|▌         | 2560/50176 [00:29<08:23, 94.63it/s]
  6%|▌         | 2816/50176 [00:32<08:32, 92.47it/s]
  6%|▌         | 3072/50176 [00:34<08:23, 93.60it/s]
  7%|▋         | 3328/50176 [00:37<08:10, 95.42it/s]
  7%|▋         | 3584/50176 [00:39<07:57, 97.53it/s]
  8%|▊         | 3840/50176 [00:42<08:06, 95.16it/s]
  8%|▊         | 4096/50176 [00:45<08:25, 91.14it/s]
  9%|▊         | 4352/50176 [00:48<08:06, 94.12it/s]
  9%|▉         | 4608/50176 [00:50<07:58, 95.23it/s]
 10%|▉         | 4864/50176 [00:53<07:45, 97.38it/s]
 10%|█         | 5120/50176 [00:55<07:35, 98.91it/s]
 11%|█         | 5376/50176 [00:58<07:40, 97.29it/s]
 11%|█         | 5632/50176 [01:01<07:53, 94.13it/s]
 12%|█▏        | 5888/50176 [01:04<07:42, 95.82it/s]
 12%|█▏        | 6144/50176 [01:07<08:05, 90.73it/s]
 13%|█▎        | 6400/50176 [01:09<07:51, 92.89it/s]
 13%|█▎        | 6656/50176 [01:12<07:33, 95.98it/s]
 14%|█▍        | 6912/50176 [01:15<07:36, 94.75it/s]
 14%|█▍        | 7168/50176 [01:17<07:23, 96.87it/s]
 15%|█▍        | 7424/50176 [01:20<07:16, 97.95it/s]
 15%|█▌        | 7680/50176 [01:22<07:10, 98.81it/s]
 16%|█▌        | 7936/50176 [01:25<07:03, 99.78it/s]
 16%|█▋        | 8192/50176 [01:27<07:06, 98.54it/s]
 17%|█▋        | 8448/50176 [01:30<06:54, 100.61it/s]
 17%|█▋        | 8704/50176 [01:32<06:58, 99.19it/s] 
 18%|█▊        | 8960/50176 [01:35<07:03, 97.31it/s]
 18%|█▊        | 9216/50176 [01:38<06:49, 99.93it/s]
 19%|█▉        | 9472/50176 [01:40<06:46, 100.04it/s]
 19%|█▉        | 9728/50176 [01:43<06:51, 98.38it/s] 
 20%|█▉        | 9984/50176 [01:48<08:44, 76.69it/s]
 20%|██        | 10240/50176 [01:52<09:32, 69.74it/s]
 21%|██        | 10496/50176 [01:55<08:29, 77.85it/s]
 21%|██▏       | 10752/50176 [01:58<08:04, 81.37it/s]
 22%|██▏       | 11008/50176 [02:00<07:28, 87.35it/s]
 22%|██▏       | 11264/50176 [02:03<07:07, 91.10it/s]
 23%|██▎       | 11520/50176 [02:05<06:50, 94.25it/s]
 23%|██▎       | 11776/50176 [02:07<06:33, 97.57it/s]
 24%|██▍       | 12032/50176 [02:10<06:32, 97.12it/s]
 24%|██▍       | 12288/50176 [02:12<06:19, 99.84it/s]
 25%|██▌       | 12544/50176 [02:15<06:11, 101.27it/s]
 26%|██▌       | 12800/50176 [02:17<06:10, 100.80it/s]
 26%|██▌       | 13056/50176 [02:20<06:06, 101.41it/s]
 27%|██▋       | 13312/50176 [02:22<06:03, 101.46it/s]
 27%|██▋       | 13568/50176 [02:25<06:12, 98.23it/s] 
 28%|██▊       | 13824/50176 [02:28<06:24, 94.44it/s]
 28%|██▊       | 14080/50176 [02:31<06:22, 94.40it/s]
 29%|██▊       | 14336/50176 [02:34<06:23, 93.48it/s]
 29%|██▉       | 14592/50176 [02:36<06:08, 96.56it/s]
 30%|██▉       | 14848/50176 [02:39<06:31, 90.25it/s]
 30%|███       | 15104/50176 [02:42<06:17, 92.90it/s]
 31%|███       | 15360/50176 [02:45<06:08, 94.38it/s]
 31%|███       | 15616/50176 [02:47<05:55, 97.16it/s]
 32%|███▏      | 15872/50176 [02:52<07:27, 76.65it/s]
 32%|███▏      | 16128/50176 [02:55<06:49, 83.05it/s]
 33%|███▎      | 16384/50176 [02:57<06:22, 88.28it/s]
 33%|███▎      | 16640/50176 [03:00<06:23, 87.40it/s]
 34%|███▎      | 16896/50176 [03:03<06:05, 90.96it/s]
 34%|███▍      | 17152/50176 [03:05<05:50, 94.12it/s]
 35%|███▍      | 17408/50176 [03:08<05:45, 94.83it/s]
 35%|███▌      | 17664/50176 [03:10<05:30, 98.44it/s]
 36%|███▌      | 17920/50176 [03:13<05:24, 99.46it/s]
 36%|███▌      | 18176/50176 [03:15<05:22, 99.29it/s]
 37%|███▋      | 18432/50176 [03:18<05:14, 100.81it/s]
 37%|███▋      | 18688/50176 [03:21<05:26, 96.52it/s] 
 38%|███▊      | 18944/50176 [03:23<05:24, 96.17it/s]
 38%|███▊      | 19200/50176 [03:26<05:27, 94.56it/s]
 39%|███▉      | 19456/50176 [03:29<05:25, 94.26it/s]
 39%|███▉      | 19712/50176 [03:34<06:46, 74.86it/s]
 40%|███▉      | 19968/50176 [03:39<07:29, 67.17it/s]
 40%|████      | 20224/50176 [03:41<06:29, 76.87it/s]
 41%|████      | 20480/50176 [03:46<07:22, 67.13it/s]
 41%|████▏     | 20736/50176 [03:48<06:32, 75.08it/s]
 42%|████▏     | 20992/50176 [03:51<05:50, 83.31it/s]
 42%|████▏     | 21248/50176 [03:53<05:34, 86.42it/s]
 43%|████▎     | 21504/50176 [03:58<06:36, 72.27it/s]
 43%|████▎     | 21760/50176 [04:01<05:56, 79.72it/s]
 44%|████▍     | 22016/50176 [04:03<05:21, 87.51it/s]
 44%|████▍     | 22272/50176 [04:05<05:01, 92.49it/s]
 45%|████▍     | 22528/50176 [04:08<04:44, 97.29it/s]
 45%|████▌     | 22784/50176 [04:10<04:41, 97.25it/s]
 46%|████▌     | 23040/50176 [04:13<04:41, 96.41it/s]
 46%|████▋     | 23296/50176 [04:16<04:41, 95.56it/s]
 47%|████▋     | 23552/50176 [04:18<04:38, 95.50it/s]
 47%|████▋     | 23808/50176 [04:21<04:23, 100.11it/s]
 48%|████▊     | 24064/50176 [04:23<04:15, 102.04it/s]
 48%|████▊     | 24320/50176 [04:25<04:14, 101.51it/s]
 49%|████▉     | 24576/50176 [04:28<04:14, 100.71it/s]
 49%|████▉     | 24832/50176 [04:31<04:08, 101.90it/s]
 50%|█████     | 25088/50176 [04:33<04:01, 104.01it/s]
 51%|█████     | 25344/50176 [04:35<03:53, 106.30it/s]
 51%|█████     | 25600/50176 [04:38<03:55, 104.26it/s]
 52%|█████▏    | 25856/50176 [04:40<03:51, 105.02it/s]
 52%|█████▏    | 26112/50176 [04:42<03:46, 106.17it/s]
 53%|█████▎    | 26368/50176 [04:45<03:45, 105.67it/s]
 53%|█████▎    | 26624/50176 [04:47<03:43, 105.45it/s]
 54%|█████▎    | 26880/50176 [04:50<03:42, 104.83it/s]
 54%|█████▍    | 27136/50176 [04:52<03:43, 103.30it/s]
 55%|█████▍    | 27392/50176 [04:55<03:32, 106.97it/s]
 55%|█████▌    | 27648/50176 [04:57<03:27, 108.71it/s]
 56%|█████▌    | 27904/50176 [05:00<03:35, 103.22it/s]
 56%|█████▌    | 28160/50176 [05:02<03:37, 101.09it/s]
 57%|█████▋    | 28416/50176 [05:05<03:34, 101.33it/s]
 57%|█████▋    | 28672/50176 [05:07<03:31, 101.58it/s]
 58%|█████▊    | 28928/50176 [05:10<03:22, 104.86it/s]
 58%|█████▊    | 29184/50176 [05:12<03:22, 103.71it/s]
 59%|█████▊    | 29440/50176 [05:15<03:32, 97.48it/s] 
 59%|█████▉    | 29696/50176 [05:18<03:35, 95.13it/s]
 60%|█████▉    | 29952/50176 [05:20<03:28, 97.17it/s]
 60%|██████    | 30208/50176 [05:23<03:21, 99.16it/s]
 61%|██████    | 30464/50176 [05:25<03:09, 104.27it/s]
 61%|██████    | 30720/50176 [05:28<03:08, 103.46it/s]
 62%|██████▏   | 30976/50176 [05:30<03:00, 106.18it/s]
 62%|██████▏   | 31232/50176 [05:32<02:58, 105.83it/s]
 63%|██████▎   | 31488/50176 [05:35<02:53, 107.62it/s]
 63%|██████▎   | 31744/50176 [05:37<02:49, 108.51it/s]
 64%|██████▍   | 32000/50176 [05:39<02:48, 108.14it/s]
 64%|██████▍   | 32256/50176 [05:42<02:48, 106.48it/s]
 65%|██████▍   | 32512/50176 [05:44<02:53, 102.00it/s]
 65%|██████▌   | 32768/50176 [05:47<02:44, 105.53it/s]
 66%|██████▌   | 33024/50176 [05:49<02:42, 105.69it/s]
 66%|██████▋   | 33280/50176 [05:51<02:36, 107.90it/s]
 67%|██████▋   | 33536/50176 [05:54<02:37, 105.62it/s]
 67%|██████▋   | 33792/50176 [05:56<02:34, 105.97it/s]
 68%|██████▊   | 34048/50176 [05:59<02:29, 108.20it/s]
 68%|██████▊   | 34304/50176 [06:01<02:24, 109.46it/s]
 69%|██████▉   | 34560/50176 [06:03<02:26, 106.57it/s]
 69%|██████▉   | 34816/50176 [06:06<02:25, 105.41it/s]
 70%|██████▉   | 35072/50176 [06:08<02:25, 103.70it/s]
 70%|███████   | 35328/50176 [06:11<02:19, 106.59it/s]
 71%|███████   | 35584/50176 [06:13<02:16, 106.64it/s]
 71%|███████▏  | 35840/50176 [06:16<02:17, 103.94it/s]
 72%|███████▏  | 36096/50176 [06:18<02:12, 106.58it/s]
 72%|███████▏  | 36352/50176 [06:20<02:10, 105.54it/s]
 73%|███████▎  | 36608/50176 [06:23<02:10, 103.80it/s]
 73%|███████▎  | 36864/50176 [06:26<02:12, 100.78it/s]
 74%|███████▍  | 37120/50176 [06:28<02:08, 101.72it/s]
 74%|███████▍  | 37376/50176 [06:30<02:00, 105.95it/s]
 75%|███████▌  | 37632/50176 [06:33<01:57, 106.44it/s]
 76%|███████▌  | 37888/50176 [06:35<01:55, 106.46it/s]
 76%|███████▌  | 38144/50176 [06:38<01:52, 107.16it/s]
 77%|███████▋  | 38400/50176 [06:40<01:51, 105.55it/s]
 77%|███████▋  | 38656/50176 [06:43<01:50, 104.48it/s]
 78%|███████▊  | 38912/50176 [06:45<01:47, 104.30it/s]
 78%|███████▊  | 39168/50176 [06:48<01:47, 102.64it/s]
 79%|███████▊  | 39424/50176 [06:50<01:42, 104.70it/s]
 79%|███████▉  | 39680/50176 [06:52<01:40, 104.63it/s]
 80%|███████▉  | 39936/50176 [06:55<01:35, 107.37it/s]
 80%|████████  | 40192/50176 [06:57<01:33, 106.49it/s]
 81%|████████  | 40448/50176 [07:00<01:34, 102.44it/s]
 81%|████████  | 40704/50176 [07:03<01:36, 98.23it/s] 
 82%|████████▏ | 40960/50176 [07:05<01:29, 102.80it/s]
 82%|████████▏ | 41216/50176 [07:07<01:25, 104.31it/s]
 83%|████████▎ | 41472/50176 [07:10<01:22, 105.09it/s]
 83%|████████▎ | 41728/50176 [07:12<01:19, 106.36it/s]
 84%|████████▎ | 41984/50176 [07:14<01:16, 106.51it/s]
 84%|████████▍ | 42240/50176 [07:17<01:13, 108.17it/s]
 85%|███████���▍ | 42496/50176 [07:19<01:10, 109.34it/s]
 85%|████████▌ | 42752/50176 [07:24<01:29, 82.52it/s] 
 86%|████████▌ | 43008/50176 [07:26<01:21, 88.29it/s]
 86%|████████▌ | 43264/50176 [07:29<01:14, 93.30it/s]
 87%|████████▋ | 43520/50176 [07:36<01:45, 63.23it/s]
 87%|████████▋ | 43776/50176 [07:40<01:46, 59.86it/s]
 88%|████████▊ | 44032/50176 [07:43<01:28, 69.57it/s]
 88%|████████▊ | 44288/50176 [07:45<01:16, 77.47it/s]
 89%|████████▉ | 44544/50176 [07:48<01:08, 82.46it/s]
 89%|████████▉ | 44800/50176 [07:50<01:01, 87.90it/s]
 90%|████████▉ | 45056/50176 [07:53<00:54, 93.97it/s]
 90%|█████████ | 45312/50176 [07:55<00:50, 95.43it/s]
 91%|█████████ | 45568/50176 [07:57<00:45, 101.35it/s]
 91%|█████████▏| 45824/50176 [08:00<00:43, 100.94it/s]
 92%|█████████▏| 46080/50176 [08:03<00:40, 100.08it/s]
 92%|█████████▏| 46336/50176 [08:05<00:38, 100.74it/s]
 93%|█████████▎| 46592/50176 [08:07<00:34, 103.87it/s]
 93%|█████████▎| 46848/50176 [08:10<00:32, 103.70it/s]
 94%|█████████▍| 47104/50176 [08:13<00:30, 99.11it/s] 
 94%|█████████▍| 47360/50176 [08:15<00:28, 99.06it/s]
 95%|█████████▍| 47616/50176 [08:18<00:25, 98.79it/s]
 95%|█████████▌| 47872/50176 [08:20<00:22, 101.25it/s]
 96%|█████████▌| 48128/50176 [08:23<00:19, 103.61it/s]
 96%|█████████▋| 48384/50176 [08:25<00:17, 103.34it/s]
 97%|█████████▋| 48640/50176 [08:27<00:14, 104.21it/s]
 97%|█████████▋| 48896/50176 [08:30<00:12, 104.13it/s]
 98%|█████████▊| 49152/50176 [08:32<00:09, 102.64it/s]
 98%|█████████▊| 49408/50176 [08:35<00:07, 102.13it/s]
 99%|█████████▉| 49664/50176 [08:38<00:05, 101.28it/s]
 99%|█████████▉| 49920/50176 [08:41<00:02, 95.67it/s] 
-2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6545	imagenet-zeroshot-val-top5: 0.8941

full_checkpoints/Model-B-16_Data-2B_Samples-13B_lr-1e-3_bs-88k/epoch_54.pt.txt

@@ -1,432 +0,0 @@
-2022-12-21,11:11:38 | INFO | Running with a single process. Device cuda:0.
-2022-12-21,11:11:38 | INFO | Loading ViT-B-16 model config.
-2022-12-21,11:11:39 | INFO | Loading pretrained ViT-B-16 weights (logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt).
-2022-12-21,11:11:45 | INFO | Model:
-2022-12-21,11:11:45 | INFO | CLIP(
-  (visual): VisualTransformer(
-    (conv1): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16), bias=False)
-    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-    (transformer): Transformer(
-      (resblocks): ModuleList(
-        (0): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (1): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (2): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (3): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (4): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (5): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (6): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (7): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (8): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (9): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (10): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-        (11): ResidualAttentionBlock(
-          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (attn): MultiheadAttention(
-            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
-          )
-          (ln_attn): Identity()
-          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-          (mlp): Sequential(
-            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
-            (ln): Identity()
-            (gelu): GELU(approximate=none)
-            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
-          )
-        )
-      )
-    )
-    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
-  )
-  (transformer): Transformer(
-    (resblocks): ModuleList(
-      (0): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (1): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (2): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (3): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (4): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (5): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (6): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (7): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (8): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (9): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (10): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-      (11): ResidualAttentionBlock(
-        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (attn): MultiheadAttention(
-          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
-        )
-        (ln_attn): Identity()
-        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-        (mlp): Sequential(
-          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
-          (ln): Identity()
-          (gelu): GELU(approximate=none)
-          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
-        )
-      )
-    )
-  )
-  (token_embedding): Embedding(49408, 512)
-  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
-)
-2022-12-21,11:11:45 | INFO | Params:
-2022-12-21,11:11:45 | INFO |   batch_size: 256
-2022-12-21,11:11:45 | INFO |   beta1: 0.9
-2022-12-21,11:11:45 | INFO |   beta2: 0.98
-2022-12-21,11:11:45 | INFO |   checkpoint_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/checkpoints
-2022-12-21,11:11:45 | INFO |   copy_codebase: False
-2022-12-21,11:11:45 | INFO |   csv_caption_key: title
-2022-12-21,11:11:45 | INFO |   csv_img_key: filepath
-2022-12-21,11:11:45 | INFO |   csv_separator: 	
-2022-12-21,11:11:45 | INFO |   dataset_resampled: False
-2022-12-21,11:11:45 | INFO |   dataset_type: auto
-2022-12-21,11:11:45 | INFO |   ddp_static_graph: False
-2022-12-21,11:11:45 | INFO |   debug: False
-2022-12-21,11:11:45 | INFO |   device: cuda:0
-2022-12-21,11:11:45 | INFO |   dist_backend: nccl
-2022-12-21,11:11:45 | INFO |   dist_url: env://
-2022-12-21,11:11:45 | INFO |   distributed: False
-2022-12-21,11:11:45 | INFO |   epochs: 32
-2022-12-21,11:11:45 | INFO |   eps: 1e-06
-2022-12-21,11:11:45 | INFO |   force_quick_gelu: False
-2022-12-21,11:11:45 | INFO |   gather_with_grad: False
-2022-12-21,11:11:45 | INFO |   grad_checkpointing: False
-2022-12-21,11:11:45 | INFO |   horovod: False
-2022-12-21,11:11:45 | INFO |   image_mean: None
-2022-12-21,11:11:45 | INFO |   image_std: None
-2022-12-21,11:11:45 | INFO |   imagenet_v2: None
-2022-12-21,11:11:45 | INFO |   imagenet_val: /p/fastdata/mmlaion/imagenet_val
-2022-12-21,11:11:45 | INFO |   local_loss: False
-2022-12-21,11:11:45 | INFO |   local_rank: 0
-2022-12-21,11:11:45 | INFO |   lock_image: False
-2022-12-21,11:11:45 | INFO |   lock_image_freeze_bn_stats: False
-2022-12-21,11:11:45 | INFO |   lock_image_unlocked_groups: 0
-2022-12-21,11:11:45 | INFO |   log_level: 20
-2022-12-21,11:11:45 | INFO |   log_local: False
-2022-12-21,11:11:45 | INFO |   log_path: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval/epoch_54.pt/out.log
-2022-12-21,11:11:45 | INFO |   logs: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/eval
-2022-12-21,11:11:45 | INFO |   lr: 0.0005
-2022-12-21,11:11:45 | INFO |   model: ViT-B-16
-2022-12-21,11:11:45 | INFO |   name: epoch_54.pt
-2022-12-21,11:11:45 | INFO |   no_set_device_rank: False
-2022-12-21,11:11:45 | INFO |   norm_gradient_clip: None
-2022-12-21,11:11:45 | INFO |   precision: amp_bfloat16
-2022-12-21,11:11:45 | INFO |   pretrained: logs/cvpr_stability/ViT-B-16_2B_13b/checkpoints/epoch_54.pt
-2022-12-21,11:11:45 | INFO |   pretrained_image: False
-2022-12-21,11:11:45 | INFO |   rank: 0
-2022-12-21,11:11:45 | INFO |   report_to: 
-2022-12-21,11:11:45 | INFO |   resume: None
-2022-12-21,11:11:45 | INFO |   save_frequency: 1
-2022-12-21,11:11:45 | INFO |   save_most_recent: False
-2022-12-21,11:11:45 | INFO |   seed: 0
-2022-12-21,11:11:45 | INFO |   skip_scheduler: False
-2022-12-21,11:11:45 | INFO |   tensorboard: False
-2022-12-21,11:11:45 | INFO |   tensorboard_path: 
-2022-12-21,11:11:45 | INFO |   torchscript: False
-2022-12-21,11:11:45 | INFO |   trace: False
-2022-12-21,11:11:45 | INFO |   train_data: None
-2022-12-21,11:11:45 | INFO |   train_num_samples: None
-2022-12-21,11:11:45 | INFO |   use_bn_sync: False
-2022-12-21,11:11:45 | INFO |   val_data: None
-2022-12-21,11:11:45 | INFO |   val_frequency: 1
-2022-12-21,11:11:45 | INFO |   val_num_samples: None
-2022-12-21,11:11:45 | INFO |   wandb: False
-2022-12-21,11:11:45 | INFO |   wandb_notes: 
-2022-12-21,11:11:45 | INFO |   warmup: 10000
-2022-12-21,11:11:45 | INFO |   wd: 0.2
-2022-12-21,11:11:45 | INFO |   workers: 1
-2022-12-21,11:11:45 | INFO |   world_size: 1
-2022-12-21,11:11:45 | INFO |   zeroshot_frequency: 2
-2022-12-21,11:11:46 | INFO | Starting zero-shot imagenet.
-2022-12-21,11:11:46 | INFO | Building zero-shot classifier
-
  0%|          | 0/1000 [00:00<?, ?it/s]
  0%|          | 1/1000 [00:00<13:23,  1.24it/s]
  1%|          | 6/1000 [00:00<01:56,  8.52it/s]
  1%|          | 11/1000 [00:01<01:02, 15.72it/s]
  2%|▏         | 17/1000 [00:01<00:41, 23.89it/s]
  2%|▏         | 23/1000 [00:01<00:31, 30.98it/s]
  3%|▎         | 29/1000 [00:01<00:26, 37.04it/s]
  4%|▎         | 35/1000 [00:01<00:22, 42.14it/s]
  4%|▍         | 41/1000 [00:01<00:20, 45.82it/s]
  5%|▍         | 47/1000 [00:01<00:19, 48.10it/s]
  5%|▌         | 53/1000 [00:01<00:18, 49.95it/s]
  6%|▌         | 59/1000 [00:01<00:18, 51.19it/s]
  6%|▋         | 65/1000 [00:02<00:17, 52.22it/s]
  7%|▋         | 71/1000 [00:02<00:17, 52.90it/s]
  8%|▊         | 77/1000 [00:02<00:17, 54.08it/s]
  8%|▊         | 83/1000 [00:02<00:16, 55.04it/s]
  9%|▉         | 89/1000 [00:02<00:16, 55.84it/s]
 10%|▉         | 95/1000 [00:02<00:16, 56.33it/s]
 10%|█         | 101/1000 [00:02<00:15, 56.71it/s]
 11%|█         | 107/1000 [00:02<00:15, 56.93it/s]
 11%|█▏        | 113/1000 [00:02<00:15, 57.21it/s]
 12%|█▏        | 119/1000 [00:02<00:15, 57.32it/s]
 12%|█▎        | 125/1000 [00:03<00:15, 57.34it/s]
 13%|█▎        | 131/1000 [00:03<00:15, 57.47it/s]
 14%|█▎        | 137/1000 [00:03<00:15, 57.42it/s]
 14%|█▍        | 143/1000 [00:03<00:14, 57.37it/s]
 15%|█▍        | 149/1000 [00:03<00:14, 57.48it/s]
 16%|█▌        | 155/1000 [00:03<00:14, 57.57it/s]
 16%|█▌        | 161/1000 [00:03<00:14, 57.47it/s]
 17%|█▋        | 167/1000 [00:03<00:14, 57.37it/s]
 17%|█▋        | 173/1000 [00:03<00:14, 57.38it/s]
 18%|█▊        | 179/1000 [00:03<00:14, 57.17it/s]
 18%|█▊        | 185/1000 [00:04<00:14, 57.06it/s]
 19%|█▉        | 191/1000 [00:04<00:14, 57.09it/s]
 20%|█▉        | 197/1000 [00:04<00:14, 57.09it/s]
 20%|██        | 203/1000 [00:04<00:13, 56.94it/s]
 21%|██        | 209/1000 [00:04<00:13, 56.79it/s]
 22%|██▏       | 215/1000 [00:04<00:13, 56.87it/s]
 22%|██▏       | 221/1000 [00:04<00:13, 56.89it/s]
 23%|██▎       | 227/1000 [00:04<00:13, 57.03it/s]
 23%|██▎       | 233/1000 [00:04<00:13, 57.12it/s]
 24%|██▍       | 239/1000 [00:05<00:13, 56.99it/s]
 24%|██▍       | 245/1000 [00:05<00:13, 57.05it/s]
 25%|██▌       | 251/1000 [00:05<00:13, 57.13it/s]
 26%|██▌       | 257/1000 [00:05<00:12, 57.32it/s]
 26%|██▋       | 263/1000 [00:05<00:12, 57.35it/s]
 27%|██▋       | 269/1000 [00:05<00:12, 57.10it/s]
 28%|██▊       | 275/1000 [00:05<00:12, 57.19it/s]
 28%|██▊       | 281/1000 [00:05<00:12, 57.34it/s]
 29%|██▊       | 287/1000 [00:05<00:12, 57.43it/s]
 29%|██▉       | 293/1000 [00:05<00:12, 57.66it/s]
 30%|██▉       | 299/1000 [00:06<00:12, 57.66it/s]
 30%|███       | 305/1000 [00:06<00:12, 57.68it/s]
 31%|███       | 311/1000 [00:06<00:11, 57.80it/s]
 32%|███▏      | 317/1000 [00:06<00:11, 57.77it/s]
 32%|███▏      | 323/1000 [00:06<00:11, 57.73it/s]
 33%|███▎      | 329/1000 [00:06<00:11, 57.53it/s]
 34%|███▎      | 335/1000 [00:06<00:11, 57.59it/s]
 34%|███▍      | 341/1000 [00:06<00:11, 57.61it/s]
 35%|███▍      | 347/1000 [00:06<00:11, 57.73it/s]
 35%|███▌      | 353/1000 [00:07<00:11, 57.52it/s]
 36%|███▌      | 359/1000 [00:07<00:11, 57.66it/s]
 36%|███▋      | 365/1000 [00:07<00:11, 57.57it/s]
 37%|███▋      | 371/1000 [00:07<00:10, 57.72it/s]
 38%|███▊      | 377/1000 [00:07<00:10, 57.58it/s]
 38%|███▊      | 383/1000 [00:07<00:10, 57.36it/s]
 39%|███▉      | 389/1000 [00:07<00:10, 57.28it/s]
 40%|███▉      | 395/1000 [00:07<00:10, 57.35it/s]
 40%|████      | 401/1000 [00:07<00:10, 57.50it/s]
 41%|████      | 407/1000 [00:07<00:10, 57.61it/s]
 41%|████▏     | 413/1000 [00:08<00:10, 57.68it/s]
 42%|████▏     | 419/1000 [00:08<00:10, 57.74it/s]
 42%|████▎     | 425/1000 [00:08<00:09, 57.68it/s]
 43%|████▎     | 431/1000 [00:08<00:09, 57.84it/s]
 44%|████▎     | 437/1000 [00:08<00:09, 57.86it/s]
 44%|████▍     | 443/1000 [00:08<00:09, 57.66it/s]
 45%|████▍     | 449/1000 [00:08<00:09, 57.70it/s]
 46%|████▌     | 455/1000 [00:08<00:09, 57.76it/s]
 46%|████▌     | 461/1000 [00:08<00:09, 57.74it/s]
 47%|████▋     | 467/1000 [00:09<00:09, 57.72it/s]
 47%|████▋     | 473/1000 [00:09<00:09, 57.86it/s]
 48%|████▊     | 479/1000 [00:09<00:09, 57.77it/s]
 48%|████▊     | 485/1000 [00:09<00:08, 57.74it/s]
 49%|████▉     | 491/1000 [00:09<00:08, 57.71it/s]
 50%|████▉     | 497/1000 [00:09<00:08, 57.62it/s]
 50%|█████     | 503/1000 [00:09<00:08, 57.74it/s]
 51%|█████     | 509/1000 [00:09<00:08, 57.62it/s]
 52%|█████▏    | 515/1000 [00:09<00:08, 57.65it/s]
 52%|█████▏    | 521/1000 [00:09<00:08, 57.70it/s]
 53%|█████▎    | 527/1000 [00:10<00:08, 57.78it/s]
 53%|█████▎    | 533/1000 [00:10<00:08, 57.68it/s]
 54%|█████▍    | 539/1000 [00:10<00:07, 57.78it/s]
 55%|█████▍    | 545/1000 [00:10<00:07, 57.82it/s]
 55%|█████▌    | 551/1000 [00:10<00:07, 57.73it/s]
 56%|█████▌    | 557/1000 [00:10<00:07, 57.71it/s]
 56%|█████▋    | 563/1000 [00:10<00:07, 57.80it/s]
 57%|█████▋    | 569/1000 [00:10<00:07, 57.69it/s]
 57%|█████▊    | 575/1000 [00:10<00:07, 57.62it/s]
 58%|█████▊    | 581/1000 [00:10<00:07, 57.77it/s]
 59%|█████▊    | 587/1000 [00:11<00:07, 57.71it/s]
 59%|█████▉    | 593/1000 [00:11<00:07, 57.64it/s]
 60%|█████▉    | 599/1000 [00:11<00:06, 57.75it/s]
 60%|██████    | 605/1000 [00:11<00:06, 57.65it/s]
 61%|██████    | 611/1000 [00:11<00:06, 57.71it/s]
 62%|██████▏   | 617/1000 [00:11<00:06, 57.78it/s]
 62%|██████▏   | 623/1000 [00:11<00:06, 57.79it/s]
 63%|██████▎   | 629/1000 [00:11<00:06, 57.84it/s]
 64%|██████▎   | 635/1000 [00:11<00:06, 57.71it/s]
 64%|██████▍   | 641/1000 [00:12<00:06, 57.59it/s]
 65%|██████▍   | 647/1000 [00:12<00:06, 57.79it/s]
 65%|██████▌   | 653/1000 [00:12<00:06, 57.72it/s]
 66%|██████▌   | 659/1000 [00:12<00:05, 57.82it/s]
 66%|██████▋   | 665/1000 [00:12<00:05, 57.80it/s]
 67%|██████▋   | 671/1000 [00:12<00:05, 57.78it/s]
 68%|██████▊   | 677/1000 [00:12<00:05, 57.81it/s]
 68%|██████▊   | 683/1000 [00:12<00:05, 57.78it/s]
 69%|██████▉   | 689/1000 [00:12<00:05, 57.82it/s]
 70%|██████▉   | 695/1000 [00:12<00:05, 57.71it/s]
 70%|███████   | 701/1000 [00:13<00:05, 57.80it/s]
 71%|███████   | 707/1000 [00:13<00:05, 57.78it/s]
 71%|███████▏  | 713/1000 [00:13<00:04, 57.75it/s]
 72%|███████▏  | 719/1000 [00:13<00:04, 57.78it/s]
 72%|███████▎  | 725/1000 [00:13<00:04, 57.73it/s]
 73%|███████▎  | 731/1000 [00:13<00:04, 57.71it/s]
 74%|███████▎  | 737/1000 [00:13<00:04, 57.78it/s]
 74%|███████▍  | 743/1000 [00:13<00:04, 57.74it/s]
 75%|███████▍  | 749/1000 [00:13<00:04, 57.81it/s]
 76%|███████▌  | 755/1000 [00:13<00:04, 57.95it/s]
 76%|███████▌  | 761/1000 [00:14<00:04, 57.77it/s]
 77%|███████▋  | 767/1000 [00:14<00:04, 57.83it/s]
 77%|███████▋  | 773/1000 [00:14<00:03, 57.83it/s]
 78%|███████▊  | 779/1000 [00:14<00:03, 57.86it/s]
 78%|███████▊  | 785/1000 [00:14<00:03, 57.87it/s]
 79%|███████▉  | 791/1000 [00:14<00:03, 57.66it/s]
 80%|███████▉  | 797/1000 [00:14<00:03, 57.66it/s]
 80%|████████  | 803/1000 [00:14<00:03, 57.68it/s]
 81%|████████  | 809/1000 [00:14<00:03, 57.66it/s]
 82%|████████▏ | 815/1000 [00:15<00:03, 57.66it/s]
 82%|████████▏ | 821/1000 [00:15<00:03, 57.73it/s]
 83%|████████▎ | 827/1000 [00:15<00:02, 57.72it/s]
 83%|████████▎ | 833/1000 [00:15<00:02, 57.89it/s]
 84%|████████▍ | 839/1000 [00:15<00:02, 57.97it/s]
 84%|████████▍ | 845/1000 [00:15<00:02, 57.82it/s]
 85%|████████▌ | 851/1000 [00:15<00:02, 57.84it/s]
 86%|████████▌ | 857/1000 [00:15<00:02, 57.77it/s]
 86%|████████▋ | 863/1000 [00:15<00:02, 57.81it/s]
 87%|████████▋ | 869/1000 [00:15<00:02, 57.75it/s]
 88%|████████▊ | 875/1000 [00:16<00:02, 57.79it/s]
 88%|████████▊ | 881/1000 [00:16<00:02, 57.81it/s]
 89%|████████▊ | 887/1000 [00:16<00:01, 57.64it/s]
 89%|████████▉ | 893/1000 [00:16<00:01, 57.74it/s]
 90%|████████▉ | 899/1000 [00:16<00:01, 57.75it/s]
 90%|█████████ | 905/1000 [00:16<00:01, 57.72it/s]
 91%|█████████ | 911/1000 [00:16<00:01, 57.70it/s]
 92%|█████████▏| 917/1000 [00:16<00:01, 57.76it/s]
 92%|█████████▏| 923/1000 [00:16<00:01, 57.68it/s]
 93%|█████████▎| 929/1000 [00:17<00:01, 57.81it/s]
 94%|█████████▎| 935/1000 [00:17<00:01, 57.88it/s]
 94%|█████████▍| 941/1000 [00:17<00:01, 57.86it/s]
 95%|█████████▍| 947/1000 [00:17<00:00, 57.83it/s]
 95%|█████████▌| 953/1000 [00:17<00:00, 57.87it/s]
 96%|█████████▌| 959/1000 [00:17<00:00, 57.83it/s]
 96%|█████████▋| 965/1000 [00:17<00:00, 57.88it/s]
 97%|█████████▋| 971/1000 [00:17<00:00, 57.96it/s]
 98%|█████████▊| 977/1000 [00:17<00:00, 57.99it/s]
 98%|█████████▊| 983/1000 [00:17<00:00, 58.00it/s]
 99%|█████████▉| 989/1000 [00:18<00:00, 57.93it/s]
-2022-12-21,11:12:04 | INFO | Using classifier
-
  0%|          | 0/50176 [00:00<?, ?it/s]
  1%|          | 256/50176 [00:05<17:57, 46.35it/s]
  1%|          | 512/50176 [00:08<12:29, 66.24it/s]
  2%|▏         | 768/50176 [00:10<10:32, 78.15it/s]
  2%|▏         | 1024/50176 [00:13<09:44, 84.10it/s]
  3%|▎         | 1280/50176 [00:16<09:30, 85.75it/s]
  3%|▎         | 1536/50176 [00:19<09:42, 83.52it/s]
  4%|▎         | 1792/50176 [00:22<09:24, 85.76it/s]
  4%|▍         | 2048/50176 [00:24<08:52, 90.33it/s]
  5%|▍         | 2304/50176 [00:27<08:52, 89.86it/s]
  5%|▌         | 2560/50176 [00:30<08:25, 94.19it/s]
  6%|▌         | 2816/50176 [00:33<08:33, 92.19it/s]
  6%|▌         | 3072/50176 [00:35<08:25, 93.25it/s]
  7%|▋         | 3328/50176 [00:38<08:10, 95.41it/s]
  7%|▋         | 3584/50176 [00:40<07:59, 97.26it/s]
  8%|▊         | 3840/50176 [00:43<08:08, 94.85it/s]
  8%|▊         | 4096/50176 [00:46<08:24, 91.34it/s]
  9%|▊         | 4352/50176 [00:49<08:05, 94.31it/s]
  9%|▉         | 4608/50176 [00:51<07:59, 95.09it/s]
 10%|▉         | 4864/50176 [00:54<07:44, 97.46it/s]
 10%|█         | 5120/50176 [00:56<07:34, 99.05it/s]
 11%|█         | 5376/50176 [00:59<07:41, 97.11it/s]
 11%|█         | 5632/50176 [01:02<07:53, 94.15it/s]
 12%|█▏        | 5888/50176 [01:05<07:42, 95.73it/s]
 12%|█▏        | 6144/50176 [01:08<08:06, 90.58it/s]
 13%|█▎        | 6400/50176 [01:10<07:50, 93.09it/s]
 13%|█▎        | 6656/50176 [01:13<07:34, 95.83it/s]
 14%|█▍        | 6912/50176 [01:16<07:37, 94.66it/s]
 14%|█▍        | 7168/50176 [01:18<07:23, 96.92it/s]
 15%|█▍        | 7424/50176 [01:21<07:15, 98.14it/s]
 15%|█▌        | 7680/50176 [01:23<07:10, 98.81it/s]
 16%|█▌        | 7936/50176 [01:26<07:03, 99.68it/s]
 16%|█▋        | 8192/50176 [01:28<07:06, 98.35it/s]
 17%|█▋        | 8448/50176 [01:31<06:55, 100.53it/s]
 17%|█▋        | 8704/50176 [01:33<06:58, 99.20it/s] 
 18%|█▊        | 8960/50176 [01:36<07:02, 97.50it/s]
 18%|█▊        | 9216/50176 [01:39<06:50, 99.84it/s]
 19%|█▉        | 9472/50176 [01:41<06:45, 100.31it/s]
 19%|█▉        | 9728/50176 [01:44<06:51, 98.20it/s] 
 20%|█▉        | 9984/50176 [01:49<08:44, 76.70it/s]
 20%|██        | 10240/50176 [01:53<09:32, 69.79it/s]
 21%|██        | 10496/50176 [01:56<08:29, 77.81it/s]
 21%|██▏       | 10752/50176 [01:59<08:04, 81.35it/s]
 22%|██▏       | 11008/50176 [02:01<07:28, 87.32it/s]
 22%|██▏       | 11264/50176 [02:04<07:07, 91.05it/s]
 23%|██▎       | 11520/50176 [02:06<06:49, 94.38it/s]
 23%|██▎       | 11776/50176 [02:08<06:33, 97.65it/s]
 24%|██▍       | 12032/50176 [02:11<06:32, 97.22it/s]
 24%|██▍       | 12288/50176 [02:14<06:19, 99.73it/s]
 25%|██▌       | 12544/50176 [02:16<06:12, 101.12it/s]
 26%|██▌       | 12800/50176 [02:19<06:10, 100.89it/s]
 26%|██▌       | 13056/50176 [02:21<06:06, 101.23it/s]
 27%|██▋       | 13312/50176 [02:24<06:03, 101.52it/s]
 27%|██▋       | 13568/50176 [02:26<06:11, 98.41it/s] 
 28%|██▊       | 13824/50176 [02:29<06:25, 94.35it/s]
 28%|██▊       | 14080/50176 [02:32<06:22, 94.32it/s]
 29%|██▊       | 14336/50176 [02:35<06:23, 93.45it/s]
 29%|██▉       | 14592/50176 [02:37<06:08, 96.55it/s]
 30%|██▉       | 14848/50176 [02:41<06:30, 90.47it/s]
 30%|███       | 15104/50176 [02:43<06:17, 92.82it/s]
 31%|███       | 15360/50176 [02:46<06:09, 94.32it/s]
 31%|███       | 15616/50176 [02:48<05:54, 97.35it/s]
 32%|███▏      | 15872/50176 [02:53<07:28, 76.56it/s]
 32%|███▏      | 16128/50176 [02:56<06:49, 83.05it/s]
 33%|███▎      | 16384/50176 [02:58<06:21, 88.48it/s]
 33%|███▎      | 16640/50176 [03:01<06:23, 87.34it/s]
 34%|███▎      | 16896/50176 [03:04<06:05, 91.06it/s]
 34%|███▍      | 17152/50176 [03:06<05:50, 94.20it/s]
 35%|███▍      | 17408/50176 [03:09<05:45, 94.76it/s]
 35%|███▌      | 17664/50176 [03:11<05:30, 98.35it/s]
 36%|███▌      | 17920/50176 [03:14<05:24, 99.48it/s]
 36%|███▌      | 18176/50176 [03:16<05:22, 99.34it/s]
 37%|███▋      | 18432/50176 [03:19<05:15, 100.61it/s]
 37%|███▋      | 18688/50176 [03:22<05:26, 96.55it/s] 
 38%|███▊      | 18944/50176 [03:24<05:24, 96.24it/s]
 38%|███▊      | 19200/50176 [03:27<05:27, 94.67it/s]
 39%|███▉      | 19456/50176 [03:30<05:26, 94.13it/s]
 39%|███▉      | 19712/50176 [03:35<06:47, 74.82it/s]
 40%|███▉      | 19968/50176 [03:40<07:29, 67.19it/s]
 40%|████      | 20224/50176 [03:42<06:29, 76.96it/s]
 41%|████      | 20480/50176 [03:47<07:22, 67.04it/s]
 41%|████▏     | 20736/50176 [03:49<06:32, 75.04it/s]
 42%|████▏     | 20992/50176 [03:52<05:50, 83.35it/s]
 42%|████▏     | 21248/50176 [03:54<05:35, 86.22it/s]
 43%|████▎     | 21504/50176 [03:59<06:36, 72.39it/s]
 43%|████▎     | 21760/50176 [04:02<05:56, 79.69it/s]
 44%|████▍     | 22016/50176 [04:04<05:21, 87.71it/s]
 44%|████▍     | 22272/50176 [04:06<05:01, 92.50it/s]
 45%|████▍     | 22528/50176 [04:09<04:44, 97.34it/s]
 45%|████▌     | 22784/50176 [04:11<04:41, 97.29it/s]
 46%|████▌     | 23040/50176 [04:14<04:41, 96.42it/s]
 46%|████▋     | 23296/50176 [04:17<04:41, 95.54it/s]
 47%|████▋     | 23552/50176 [04:19<04:38, 95.56it/s]
 47%|████▋     | 23808/50176 [04:22<04:23, 100.16it/s]
 48%|████▊     | 24064/50176 [04:24<04:15, 102.00it/s]
 48%|████▊     | 24320/50176 [04:27<04:15, 101.33it/s]
 49%|████▉     | 24576/50176 [04:29<04:14, 100.62it/s]
 49%|████▉     | 24832/50176 [04:32<04:08, 101.90it/s]
 50%|█████     | 25088/50176 [04:34<04:01, 104.09it/s]
 51%|█████     | 25344/50176 [04:36<03:53, 106.23it/s]
 51%|█████     | 25600/50176 [04:39<03:55, 104.45it/s]
 52%|█████▏    | 25856/50176 [04:41<03:51, 105.12it/s]
 52%|█████▏    | 26112/50176 [04:44<03:46, 106.14it/s]
 53%|█████▎    | 26368/50176 [04:46<03:45, 105.55it/s]
 53%|█████▎    | 26624/50176 [04:48<03:43, 105.51it/s]
 54%|█████▎    | 26880/50176 [04:51<03:41, 104.96it/s]
 54%|█████▍    | 27136/50176 [04:53<03:43, 103.19it/s]
 55%|█████▍    | 27392/50176 [04:56<03:33, 106.87it/s]
 55%|█████▌    | 27648/50176 [04:58<03:27, 108.76it/s]
 56%|█████▌    | 27904/50176 [05:01<03:35, 103.21it/s]
 56%|█████▌    | 28160/50176 [05:03<03:37, 101.03it/s]
 57%|█████▋    | 28416/50176 [05:06<03:34, 101.23it/s]
 57%|█████▋    | 28672/50176 [05:08<03:31, 101.53it/s]
 58%|█████▊    | 28928/50176 [05:11<03:22, 104.90it/s]
 58%|█████▊    | 29184/50176 [05:13<03:22, 103.56it/s]
 59%|█████▊    | 29440/50176 [05:16<03:32, 97.58it/s] 
 59%|█████▉    | 29696/50176 [05:19<03:35, 95.00it/s]
 60%|█████▉    | 29952/50176 [05:21<03:27, 97.28it/s]
 60%|██████    | 30208/50176 [05:24<03:21, 99.20it/s]
 61%|██████    | 30464/50176 [05:26<03:08, 104.39it/s]
 61%|██████    | 30720/50176 [05:29<03:07, 103.51it/s]
 62%|██████▏   | 30976/50176 [05:31<03:01, 106.03it/s]
 62%|██████▏   | 31232/50176 [05:33<02:59, 105.69it/s]
 63%|██████▎   | 31488/50176 [05:36<02:53, 107.85it/s]
 63%|██████▎   | 31744/50176 [05:38<02:49, 108.49it/s]
 64%|██████▍   | 32000/50176 [05:40<02:47, 108.31it/s]
 64%|██████▍   | 32256/50176 [05:43<02:48, 106.44it/s]
 65%|██████▍   | 32512/50176 [05:46<02:53, 102.07it/s]
 65%|██████▌   | 32768/50176 [05:48<02:44, 105.55it/s]
 66%|██████▌   | 33024/50176 [05:50<02:42, 105.70it/s]
 66%|██████▋   | 33280/50176 [05:52<02:36, 108.16it/s]
 67%|██████▋   | 33536/50176 [05:55<02:37, 105.49it/s]
 67%|██████▋   | 33792/50176 [05:57<02:34, 105.89it/s]
 68%|██████▊   | 34048/50176 [06:00<02:29, 108.18it/s]
 68%|██████▊   | 34304/50176 [06:02<02:24, 109.57it/s]
 69%|██████▉   | 34560/50176 [06:04<02:26, 106.50it/s]
 69%|██████▉   | 34816/50176 [06:07<02:25, 105.45it/s]
 70%|██████▉   | 35072/50176 [06:09<02:25, 103.71it/s]
 70%|███████   | 35328/50176 [06:12<02:19, 106.76it/s]
 71%|███████   | 35584/50176 [06:14<02:16, 106.52it/s]
 71%|███████▏  | 35840/50176 [06:17<02:17, 103.90it/s]
 72%|███████▏  | 36096/50176 [06:19<02:12, 106.50it/s]
 72%|███████▏  | 36352/50176 [06:22<02:11, 105.35it/s]
 73%|███████▎  | 36608/50176 [06:24<02:10, 103.85it/s]
 73%|███████▎  | 36864/50176 [06:27<02:12, 100.62it/s]
 74%|███████▍  | 37120/50176 [06:29<02:08, 101.72it/s]
 74%|███████▍  | 37376/50176 [06:31<02:00, 105.85it/s]
 75%|███████▌  | 37632/50176 [06:34<01:57, 106.54it/s]
 76%|███████▌  | 37888/50176 [06:36<01:55, 106.68it/s]
 76%|███████▌  | 38144/50176 [06:39<01:52, 107.11it/s]
 77%|███████▋  | 38400/50176 [06:41<01:51, 105.47it/s]
 77%|███████▋  | 38656/50176 [06:44<01:50, 104.46it/s]
 78%|███████▊  | 38912/50176 [06:46<01:47, 104.33it/s]
 78%|███████▊  | 39168/50176 [06:49<01:47, 102.65it/s]
 79%|███████▊  | 39424/50176 [06:51<01:42, 104.65it/s]
 79%|███████▉  | 39680/50176 [06:53<01:40, 104.59it/s]
 80%|███████▉  | 39936/50176 [06:56<01:35, 107.29it/s]
 80%|████████  | 40192/50176 [06:58<01:33, 106.51it/s]
 81%|████████  | 40448/50176 [07:01<01:34, 102.57it/s]
 81%|████████  | 40704/50176 [07:04<01:36, 98.17it/s] 
 82%|████████▏ | 40960/50176 [07:06<01:29, 102.83it/s]
 82%|████████▏ | 41216/50176 [07:08<01:25, 104.37it/s]
 83%|████████▎ | 41472/50176 [07:11<01:22, 105.07it/s]
 83%|████████▎ | 41728/50176 [07:13<01:19, 106.24it/s]
 84%|████████▎ | 41984/50176 [07:15<01:16, 106.44it/s]
 84%|████████▍ | 42240/50176 [07:18<01:13, 108.25it/s]
 85%|████████▍ | 42496/50176 [07:20<01:10, 109.42it/s]
 85%|████████▌ | 42752/50176 [07:25<01:29, 82.59it/s] 
 86%|████████▌ | 43008/50176 [07:27<01:21, 88.36it/s]
 86%|████████▌ | 43264/50176 [07:30<01:14, 93.28it/s]
 87%|████████▋ | 43520/50176 [07:37<01:45, 63.23it/s]
 87%|████████▋ | 43776/50176 [07:42<01:46, 59.85it/s]
 88%|████████▊ | 44032/50176 [07:44<01:28, 69.55it/s]
 88%|████████▊ | 44288/50176 [07:46<01:16, 77.39it/s]
 89%|████████▉ | 44544/50176 [07:49<01:08, 82.40it/s]
 89%|████████▉ | 44800/50176 [07:51<01:01, 87.93it/s]
 90%|████████▉ | 45056/50176 [07:54<00:54, 93.98it/s]
 90%|█████████ | 45312/50176 [07:56<00:50, 95.40it/s]
 91%|█████████ | 45568/50176 [07:58<00:45, 101.30it/s]
 91%|█████████▏| 45824/50176 [08:01<00:43, 101.05it/s]
 92%|█████████▏| 46080/50176 [08:04<00:40, 100.20it/s]
 92%|█████████▏| 46336/50176 [08:06<00:38, 100.63it/s]
 93%|█████████▎| 46592/50176 [08:08<00:34, 103.89it/s]
 93%|█████████▎| 46848/50176 [08:11<00:32, 103.75it/s]
 94%|█████████▍| 47104/50176 [08:14<00:30, 99.20it/s] 
 94%|█████████▍| 47360/50176 [08:16<00:28, 99.06it/s]
 95%|█████████▍| 47616/50176 [08:19<00:25, 98.80it/s]
 95%|█████████▌| 47872/50176 [08:21<00:22, 101.20it/s]
 96%|█████████▌| 48128/50176 [08:24<00:19, 103.58it/s]
 96%|█████████▋| 48384/50176 [08:26<00:17, 103.38it/s]
 97%|█████████▋| 48640/50176 [08:28<00:14, 104.19it/s]
 97%|█████████▋| 48896/50176 [08:31<00:12, 104.29it/s]
 98%|█████████▊| 49152/50176 [08:34<00:09, 102.61it/s]
 98%|█████████▊| 49408/50176 [08:36<00:07, 102.14it/s]
 99%|█████████▉| 49664/50176 [08:39<00:05, 101.22it/s]
 99%|█████████▉| 49920/50176 [08:42<00:02, 95.69it/s] 
-2022-12-21,11:20:47 | INFO | Finished zero-shot imagenet.
-2022-12-21,11:20:47 | INFO | Eval Epoch: 0 imagenet-zeroshot-val-top1: 0.6589	imagenet-zeroshot-val-top5: 0.8959