diff --git "a/baseline/baseline/out.log" "b/baseline/baseline/out.log"
new file mode 100644--- /dev/null
+++ "b/baseline/baseline/out.log"
@@ -0,0 +1,2053 @@
+2024-09-09,16:35:04 | INFO | No latest resume checkpoint found in /home/baseline/baseline/checkpoints.
+2024-09-09,16:35:05 | INFO | Running in distributed mode with multiple processes. Device: cuda:0.Process (global: 0, local 0), total 2.
+2024-09-09,16:35:05 | INFO | Loaded ViT-B-32 model config.
+2024-09-09,16:35:07 | INFO | Model:
+2024-09-09,16:35:07 | INFO | CLIP(
+  (visual): VisionTransformer(
+    (patchnorm_pre_ln): Identity()
+    (conv1): Conv2d(3, 768, kernel_size=(32, 32), stride=(32, 32), bias=False)
+    (patch_dropout): Identity()
+    (ln_pre): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+    (transformer): Transformer(
+      (resblocks): ModuleList(
+        (0): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (1): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (2): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (3): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (4): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (5): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (6): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (7): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (8): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (9): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (10): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+        (11): ResidualAttentionBlock(
+          (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (attn): MultiheadAttention(
+            (out_proj): NonDynamicallyQuantizableLinear(in_features=768, out_features=768, bias=True)
+          )
+          (ls_1): Identity()
+          (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+          (mlp): Sequential(
+            (c_fc): Linear(in_features=768, out_features=3072, bias=True)
+            (gelu): GELU(approximate='none')
+            (c_proj): Linear(in_features=3072, out_features=768, bias=True)
+          )
+          (ls_2): Identity()
+        )
+      )
+    )
+    (ln_post): LayerNorm((768,), eps=1e-05, elementwise_affine=True)
+  )
+  (transformer): Transformer(
+    (resblocks): ModuleList(
+      (0): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (1): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (2): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (3): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (4): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (5): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (6): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (7): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (8): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (9): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (10): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+      (11): ResidualAttentionBlock(
+        (ln_1): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (attn): MultiheadAttention(
+          (out_proj): NonDynamicallyQuantizableLinear(in_features=512, out_features=512, bias=True)
+        )
+        (ls_1): Identity()
+        (ln_2): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+        (mlp): Sequential(
+          (c_fc): Linear(in_features=512, out_features=2048, bias=True)
+          (gelu): GELU(approximate='none')
+          (c_proj): Linear(in_features=2048, out_features=512, bias=True)
+        )
+        (ls_2): Identity()
+      )
+    )
+  )
+  (token_embedding): Embedding(49408, 512)
+  (ln_final): LayerNorm((512,), eps=1e-05, elementwise_affine=True)
+)
+2024-09-09,16:35:07 | INFO | Params:
+2024-09-09,16:35:07 | INFO |   accum_freq: 1
+2024-09-09,16:35:07 | INFO |   aug_cfg: {}
+2024-09-09,16:35:07 | INFO |   batch_size: 2048
+2024-09-09,16:35:07 | INFO |   beta1: 0.9
+2024-09-09,16:35:07 | INFO |   beta2: 0.98
+2024-09-09,16:35:07 | INFO |   checkpoint_path: /home/baseline/baseline/checkpoints
+2024-09-09,16:35:07 | INFO |   coca_caption_loss_weight: 2.0
+2024-09-09,16:35:07 | INFO |   coca_contrastive_loss_weight: 1.0
+2024-09-09,16:35:07 | INFO |   copy_codebase: False
+2024-09-09,16:35:07 | INFO |   csv_caption_key: title
+2024-09-09,16:35:07 | INFO |   csv_img_key: filepath
+2024-09-09,16:35:07 | INFO |   csv_separator: 	
+2024-09-09,16:35:07 | INFO |   dataset_resampled: True
+2024-09-09,16:35:07 | INFO |   dataset_type: webdataset
+2024-09-09,16:35:07 | INFO |   ddp_static_graph: True
+2024-09-09,16:35:07 | INFO |   debug: False
+2024-09-09,16:35:07 | INFO |   delete_previous_checkpoint: False
+2024-09-09,16:35:07 | INFO |   device: cuda:0
+2024-09-09,16:35:07 | INFO |   dist_backend: nccl
+2024-09-09,16:35:07 | INFO |   dist_url: env://
+2024-09-09,16:35:07 | INFO |   distill: False
+2024-09-09,16:35:07 | INFO |   distill_model: None
+2024-09-09,16:35:07 | INFO |   distill_pretrained: None
+2024-09-09,16:35:07 | INFO |   distributed: True
+2024-09-09,16:35:07 | INFO |   epochs: 5
+2024-09-09,16:35:07 | INFO |   epochs_cooldown: None
+2024-09-09,16:35:07 | INFO |   eps: 1e-06
+2024-09-09,16:35:07 | INFO |   force_custom_text: False
+2024-09-09,16:35:07 | INFO |   force_image_size: None
+2024-09-09,16:35:07 | INFO |   force_patch_dropout: None
+2024-09-09,16:35:07 | INFO |   force_quick_gelu: False
+2024-09-09,16:35:07 | INFO |   gather_with_grad: True
+2024-09-09,16:35:07 | INFO |   grad_checkpointing: True
+2024-09-09,16:35:07 | INFO |   grad_clip_norm: None
+2024-09-09,16:35:07 | INFO |   horovod: False
+2024-09-09,16:35:07 | INFO |   image_mean: None
+2024-09-09,16:35:07 | INFO |   image_std: None
+2024-09-09,16:35:07 | INFO |   imagenet_v2: None
+2024-09-09,16:35:07 | INFO |   imagenet_val: None
+2024-09-09,16:35:07 | INFO |   local_loss: True
+2024-09-09,16:35:07 | INFO |   local_rank: 0
+2024-09-09,16:35:07 | INFO |   lock_image: False
+2024-09-09,16:35:07 | INFO |   lock_image_freeze_bn_stats: False
+2024-09-09,16:35:07 | INFO |   lock_image_unlocked_groups: 0
+2024-09-09,16:35:07 | INFO |   lock_text: False
+2024-09-09,16:35:07 | INFO |   lock_text_freeze_layer_norm: False
+2024-09-09,16:35:07 | INFO |   lock_text_unlocked_layers: 0
+2024-09-09,16:35:07 | INFO |   log_every_n_steps: 100
+2024-09-09,16:35:07 | INFO |   log_level: 20
+2024-09-09,16:35:07 | INFO |   log_local: False
+2024-09-09,16:35:07 | INFO |   log_path: /home/baseline/baseline/out.log
+2024-09-09,16:35:07 | INFO |   logs: /home/baseline
+2024-09-09,16:35:07 | INFO |   lr: 0.0005
+2024-09-09,16:35:07 | INFO |   lr_cooldown_end: 0.0
+2024-09-09,16:35:07 | INFO |   lr_cooldown_power: 1.0
+2024-09-09,16:35:07 | INFO |   lr_scheduler: cosine
+2024-09-09,16:35:07 | INFO |   model: ViT-B-32
+2024-09-09,16:35:07 | INFO |   name: baseline
+2024-09-09,16:35:07 | INFO |   no_set_device_rank: False
+2024-09-09,16:35:07 | INFO |   precision: amp
+2024-09-09,16:35:07 | INFO |   pretrained: 
+2024-09-09,16:35:07 | INFO |   pretrained_image: False
+2024-09-09,16:35:07 | INFO |   rank: 0
+2024-09-09,16:35:07 | INFO |   remote_sync: None
+2024-09-09,16:35:07 | INFO |   remote_sync_frequency: 300
+2024-09-09,16:35:07 | INFO |   remote_sync_protocol: s3
+2024-09-09,16:35:07 | INFO |   report_to: wandb
+2024-09-09,16:35:07 | INFO |   resume: None
+2024-09-09,16:35:07 | INFO |   save_frequency: 0
+2024-09-09,16:35:07 | INFO |   save_most_recent: True
+2024-09-09,16:35:07 | INFO |   seed: 0
+2024-09-09,16:35:07 | INFO |   skip_scheduler: False
+2024-09-09,16:35:07 | INFO |   tensorboard: False
+2024-09-09,16:35:07 | INFO |   tensorboard_path: 
+2024-09-09,16:35:07 | INFO |   torchscript: False
+2024-09-09,16:35:07 | INFO |   trace: False
+2024-09-09,16:35:07 | INFO |   train_data: ::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::/home/small_datacomp_all/{00000000..00001287}.tar
+2024-09-09,16:35:07 | INFO |   train_data_upsampling_factors: None
+2024-09-09,16:35:07 | INFO |   train_num_samples: 2560000
+2024-09-09,16:35:07 | INFO |   use_bn_sync: False
+2024-09-09,16:35:07 | INFO |   val_data: None
+2024-09-09,16:35:07 | INFO |   val_frequency: 1
+2024-09-09,16:35:07 | INFO |   val_num_samples: None
+2024-09-09,16:35:07 | INFO |   wandb: True
+2024-09-09,16:35:07 | INFO |   wandb_notes: 
+2024-09-09,16:35:07 | INFO |   wandb_project_name: clip_text_hq_clusters
+2024-09-09,16:35:07 | INFO |   warmup: 500
+2024-09-09,16:35:07 | INFO |   wd: 0.2
+2024-09-09,16:35:07 | INFO |   workers: 4
+2024-09-09,16:35:07 | INFO |   world_size: 2
+2024-09-09,16:35:07 | INFO |   zeroshot_frequency: 2
+2024-09-09,16:35:13 | INFO | Start epoch 0
+2024-09-09,16:35:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:30 | INFO | Train Epoch: 0 [   4096/2572288 (0%)] Data (t): 13.176 Batch (t): 17.350, 236.075/s, 118.038/s/gpu LR: 0.000001 Logit Scale: 14.286 Contrastive_loss: 8.3749 (8.3749) Loss: 8.3749 (8.3749)
+2024-09-09,16:35:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:35:33 | INFO | Reducer buckets have been rebuilt in this iteration.
+2024-09-09,16:36:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:36:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:37:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:37:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:37:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:37:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:38:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:35 | INFO | Train Epoch: 0 [ 413696/2572288 (16%)] Data (t): 0.404 Batch (t): 2.447, 1738.16/s, 869.081/s/gpu LR: 0.000101 Logit Scale: 14.268 Contrastive_loss: 8.1801 (8.2775) Loss: 8.1801 (8.2775)
+2024-09-09,16:39:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:39:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:41:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:41:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:41:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:41:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:42:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:32 | INFO | Train Epoch: 0 [ 823296/2572288 (32%)] Data (t): 0.360 Batch (t): 2.371, 1738.16/s, 869.078/s/gpu LR: 0.000201 Logit Scale: 14.235 Contrastive_loss: 8.0512 (8.2020) Loss: 8.0512 (8.2020)
+2024-09-09,16:43:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:43:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:44:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:44:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:44:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:44:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:44:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:44:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:45:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:46:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:46:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:46:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:46:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:46:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:29 | INFO | Train Epoch: 0 [1232896/2572288 (48%)] Data (t): 0.360 Batch (t): 2.370, 1758.90/s, 879.452/s/gpu LR: 0.000301 Logit Scale: 14.204 Contrastive_loss: 7.8912 (8.1243) Loss: 7.8912 (8.1243)
+2024-09-09,16:47:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:47:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:48:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:49:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:49:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:49:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:50:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:50:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:50:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:50:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:26 | INFO | Train Epoch: 0 [1642496/2572288 (64%)] Data (t): 0.361 Batch (t): 2.370, 1760.66/s, 880.328/s/gpu LR: 0.000401 Logit Scale: 14.169 Contrastive_loss: 7.8317 (8.0658) Loss: 7.8317 (8.0658)
+2024-09-09,16:51:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:51:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:52:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:53:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:54:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:55:23 | INFO | Train Epoch: 0 [2052096/2572288 (80%)] Data (t): 0.364 Batch (t): 2.373, 1738.32/s, 869.159/s/gpu LR: 0.000500 Logit Scale: 14.147 Contrastive_loss: 7.7018 (8.0051) Loss: 7.7018 (8.0051)
+2024-09-09,16:55:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:55:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:55:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:55:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:55:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:55:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:56:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:57:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:58:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:22 | INFO | Train Epoch: 0 [2461696/2572288 (96%)] Data (t): 0.371 Batch (t): 2.384, 1718.60/s, 859.302/s/gpu LR: 0.000498 Logit Scale: 14.145 Contrastive_loss: 7.6058 (7.9481) Loss: 7.6058 (7.9481)
+2024-09-09,16:59:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,16:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:00:26 | INFO | Train Epoch: 0 [2572288/2572288 (100%)] Data (t): 0.372 Batch (t): 2.390, 1697.93/s, 848.964/s/gpu LR: 0.000497 Logit Scale: 14.147 Contrastive_loss: 7.5590 (7.8994) Loss: 7.5590 (7.8994)
+2024-09-09,17:00:28 | INFO | Start epoch 1
+2024-09-09,17:00:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:00:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:00:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:00:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:00:40 | INFO | Train Epoch: 1 [   4096/2572288 (0%)] Data (t): 9.524 Batch (t): 11.534, 355.109/s, 177.555/s/gpu LR: 0.000497 Logit Scale: 14.149 Contrastive_loss: 7.5932 (7.5932) Loss: 7.5932 (7.5932)
+2024-09-09,17:00:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:00:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:01:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:02:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:03:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:04:37 | INFO | Train Epoch: 1 [ 413696/2572288 (16%)] Data (t): 0.347 Batch (t): 2.370, 1710.36/s, 855.181/s/gpu LR: 0.000491 Logit Scale: 14.191 Contrastive_loss: 7.3800 (7.4866) Loss: 7.3800 (7.4866)
+2024-09-09,17:04:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:05:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:52 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:52 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:52 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:06:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:07:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:35 | INFO | Train Epoch: 1 [ 823296/2572288 (32%)] Data (t): 0.372 Batch (t): 2.383, 1728.16/s, 864.081/s/gpu LR: 0.000481 Logit Scale: 14.245 Contrastive_loss: 7.4128 (7.4620) Loss: 7.4128 (7.4620)
+2024-09-09,17:08:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:08:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:09:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:10:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:11:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:33 | INFO | Train Epoch: 1 [1232896/2572288 (48%)] Data (t): 0.374 Batch (t): 2.385, 1664.80/s, 832.398/s/gpu LR: 0.000468 Logit Scale: 14.320 Contrastive_loss: 7.3493 (7.4338) Loss: 7.3493 (7.4338)
+2024-09-09,17:12:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:12:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:13:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:14:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:15:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:32 | INFO | Train Epoch: 1 [1642496/2572288 (64%)] Data (t): 0.379 Batch (t): 2.390, 1750.28/s, 875.139/s/gpu LR: 0.000452 Logit Scale: 14.426 Contrastive_loss: 7.1557 (7.3782) Loss: 7.1557 (7.3782)
+2024-09-09,17:16:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:16:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:17:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:18:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:19:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:32 | INFO | Train Epoch: 1 [2052096/2572288 (80%)] Data (t): 0.381 Batch (t): 2.393, 1722.75/s, 861.377/s/gpu LR: 0.000433 Logit Scale: 14.555 Contrastive_loss: 7.2304 (7.3536) Loss: 7.2304 (7.3536)
+2024-09-09,17:20:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:20:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:21:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:22:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:23:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:24:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:24:30 | INFO | Train Epoch: 1 [2461696/2572288 (96%)] Data (t): 0.376 Batch (t): 2.386, 1720.03/s, 860.014/s/gpu LR: 0.000412 Logit Scale: 14.715 Contrastive_loss: 7.1434 (7.3235) Loss: 7.1434 (7.3235)
+2024-09-09,17:25:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:35 | INFO | Train Epoch: 1 [2572288/2572288 (100%)] Data (t): 0.377 Batch (t): 2.389, 1738.34/s, 869.170/s/gpu LR: 0.000406 Logit Scale: 14.762 Contrastive_loss: 7.2076 (7.3091) Loss: 7.2076 (7.3091)
+2024-09-09,17:25:36 | INFO | Start epoch 2
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:48 | INFO | Train Epoch: 2 [   4096/2572288 (0%)] Data (t): 9.136 Batch (t): 11.139, 367.720/s, 183.860/s/gpu LR: 0.000405 Logit Scale: 14.762 Contrastive_loss: 7.0886 (7.0886) Loss: 7.0886 (7.0886)
+2024-09-09,17:25:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:25:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:26:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:27:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:28:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:29:46 | INFO | Train Epoch: 2 [ 413696/2572288 (16%)] Data (t): 0.362 Batch (t): 2.388, 1747.12/s, 873.561/s/gpu LR: 0.000381 Logit Scale: 14.917 Contrastive_loss: 6.9718 (7.0302) Loss: 6.9718 (7.0302)
+2024-09-09,17:30:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:08 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:30:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:31:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:32:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:46 | INFO | Train Epoch: 2 [ 823296/2572288 (32%)] Data (t): 0.382 Batch (t): 2.392, 1717.22/s, 858.611/s/gpu LR: 0.000355 Logit Scale: 15.078 Contrastive_loss: 7.0147 (7.0250) Loss: 7.0147 (7.0250)
+2024-09-09,17:33:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:33:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:34:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:35:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:35:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:35:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:35:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:35:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:35:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:36:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:37:46 | INFO | Train Epoch: 2 [1232896/2572288 (48%)] Data (t): 0.388 Batch (t): 2.399, 1755.41/s, 877.704/s/gpu LR: 0.000327 Logit Scale: 15.239 Contrastive_loss: 6.9100 (6.9963) Loss: 6.9100 (6.9963)
+2024-09-09,17:38:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:38:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:39:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:40:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:40:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:40:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:41:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:41:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:41:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:41:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:41:45 | INFO | Train Epoch: 2 [1642496/2572288 (64%)] Data (t): 0.385 Batch (t): 2.396, 1715.24/s, 857.618/s/gpu LR: 0.000298 Logit Scale: 15.432 Contrastive_loss: 6.7076 (6.9385) Loss: 6.7076 (6.9385)
+2024-09-09,17:41:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:42:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:42:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:42:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:42:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:42:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:43:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:43:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:43:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:43:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:43:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:43:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:44:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:44 | INFO | Train Epoch: 2 [2052096/2572288 (80%)] Data (t): 0.378 Batch (t): 2.390, 1728.95/s, 864.477/s/gpu LR: 0.000269 Logit Scale: 15.629 Contrastive_loss: 6.8026 (6.9159) Loss: 6.8026 (6.9159)
+2024-09-09,17:45:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:45:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:46:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:47:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:48:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:48:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:48:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:48:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:48:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:48:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:42 | INFO | Train Epoch: 2 [2461696/2572288 (96%)] Data (t): 0.370 Batch (t): 2.375, 1732.47/s, 866.233/s/gpu LR: 0.000239 Logit Scale: 15.771 Contrastive_loss: 6.7551 (6.8929) Loss: 6.7551 (6.8929)
+2024-09-09,17:49:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:49:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:46 | INFO | Train Epoch: 2 [2572288/2572288 (100%)] Data (t): 0.360 Batch (t): 2.367, 1733.88/s, 866.942/s/gpu LR: 0.000231 Logit Scale: 15.818 Contrastive_loss: 6.6716 (6.8652) Loss: 6.6716 (6.8652)
+2024-09-09,17:50:47 | INFO | Start epoch 3
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:50:59 | INFO | Train Epoch: 3 [   4096/2572288 (0%)] Data (t): 9.254 Batch (t): 11.257, 363.856/s, 181.928/s/gpu LR: 0.000231 Logit Scale: 15.820 Contrastive_loss: 6.7272 (6.7272) Loss: 6.7272 (6.7272)
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:51:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:52:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:53:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:54:55 | INFO | Train Epoch: 3 [ 413696/2572288 (16%)] Data (t): 0.347 Batch (t): 2.363, 1721.70/s, 860.850/s/gpu LR: 0.000202 Logit Scale: 15.988 Contrastive_loss: 6.6811 (6.7042) Loss: 6.6811 (6.7042)
+2024-09-09,17:55:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:55:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:23 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:33 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:56:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:57:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:58:53 | INFO | Train Epoch: 3 [ 823296/2572288 (32%)] Data (t): 0.374 Batch (t): 2.378, 1731.53/s, 865.767/s/gpu LR: 0.000173 Logit Scale: 16.166 Contrastive_loss: 6.6978 (6.7020) Loss: 6.6978 (6.7020)
+2024-09-09,17:58:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,17:59:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:20 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:00:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:01:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:07 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:02:51 | INFO | Train Epoch: 3 [1232896/2572288 (48%)] Data (t): 0.377 Batch (t): 2.382, 1720.12/s, 860.062/s/gpu LR: 0.000145 Logit Scale: 16.314 Contrastive_loss: 6.7061 (6.7031) Loss: 6.7061 (6.7031)
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:03:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:24 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:04:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:02 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:05:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:06:49 | INFO | Train Epoch: 3 [1642496/2572288 (64%)] Data (t): 0.374 Batch (t): 2.379, 1729.10/s, 864.550/s/gpu LR: 0.000119 Logit Scale: 16.440 Contrastive_loss: 6.5106 (6.6646) Loss: 6.5106 (6.6646)
+2024-09-09,18:07:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:07:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:26 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:08:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:09:09 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:09:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:09:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:09:46 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:41 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:47 | INFO | Train Epoch: 3 [2052096/2572288 (80%)] Data (t): 0.377 Batch (t): 2.382, 1715.22/s, 857.611/s/gpu LR: 0.000095 Logit Scale: 16.531 Contrastive_loss: 6.4518 (6.6291) Loss: 6.4518 (6.6291)
+2024-09-09,18:10:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:10:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:11:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:11:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:11:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:11:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:11:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:11:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:12:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:45 | INFO | Train Epoch: 3 [2461696/2572288 (96%)] Data (t): 0.375 Batch (t): 2.380, 1728.09/s, 864.045/s/gpu LR: 0.000072 Logit Scale: 16.627 Contrastive_loss: 6.5475 (6.6174) Loss: 6.5475 (6.6174)
+2024-09-09,18:14:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:14:59 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:49 | INFO | Train Epoch: 3 [2572288/2572288 (100%)] Data (t): 0.372 Batch (t): 2.376, 1734.44/s, 867.218/s/gpu LR: 0.000067 Logit Scale: 16.652 Contrastive_loss: 6.2974 (6.5774) Loss: 6.2974 (6.5774)
+2024-09-09,18:15:51 | INFO | Start epoch 4
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:15:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:02 | INFO | Train Epoch: 4 [   4096/2572288 (0%)] Data (t): 9.203 Batch (t): 11.207, 365.482/s, 182.741/s/gpu LR: 0.000067 Logit Scale: 16.652 Contrastive_loss: 6.4165 (6.4165) Loss: 6.4165 (6.4165)
+2024-09-09,18:16:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:16:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:17:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:18:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:18:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:18:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:18:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:18:29 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:10 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:19:59 | INFO | Train Epoch: 4 [ 413696/2572288 (16%)] Data (t): 0.349 Batch (t): 2.371, 1725.36/s, 862.681/s/gpu LR: 0.000048 Logit Scale: 16.717 Contrastive_loss: 6.3834 (6.3999) Loss: 6.3834 (6.3999)
+2024-09-09,18:20:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:20:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:20:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:20:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:20:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:20:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:20:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:21:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:19 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:51 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:22:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:35 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:23:58 | INFO | Train Epoch: 4 [ 823296/2572288 (32%)] Data (t): 0.382 Batch (t): 2.387, 1716.01/s, 858.007/s/gpu LR: 0.000032 Logit Scale: 16.761 Contrastive_loss: 6.3468 (6.3822) Loss: 6.3468 (6.3822)
+2024-09-09,18:24:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:45 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:24:48 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:25:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:25:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:25:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:25:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:01 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:04 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:36 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:38 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:26:39 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:13 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:16 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:52 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:27:56 | INFO | Train Epoch: 4 [1232896/2572288 (48%)] Data (t): 0.373 Batch (t): 2.377, 1716.68/s, 858.340/s/gpu LR: 0.000019 Logit Scale: 16.794 Contrastive_loss: 6.2868 (6.3583) Loss: 6.2868 (6.3583)
+2024-09-09,18:28:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:28:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:28:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:28:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:28:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:28:30 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:06 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:29:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:30:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:30:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:30:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:30:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:30:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:30:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:31:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:31:00 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:31:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:31:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:31:43 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:31:53 | INFO | Train Epoch: 4 [1642496/2572288 (64%)] Data (t): 0.374 Batch (t): 2.378, 1726.23/s, 863.117/s/gpu LR: 0.000009 Logit Scale: 16.811 Contrastive_loss: 6.3906 (6.3648) Loss: 6.3906 (6.3648)
+2024-09-09,18:32:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:21 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:54 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:32:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:17 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:32 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:37 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:33:55 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:15 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:50 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:34:53 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:11 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:31 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:35:52 | INFO | Train Epoch: 4 [2052096/2572288 (80%)] Data (t): 0.378 Batch (t): 2.382, 1710.97/s, 855.486/s/gpu LR: 0.000003 Logit Scale: 16.818 Contrastive_loss: 6.4415 (6.3776) Loss: 6.4415 (6.3776)
+2024-09-09,18:36:03 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:27 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:40 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:44 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:36:47 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:05 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:18 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:22 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:25 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:42 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:37:56 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:38:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:38:34 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:38:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:38:58 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:12 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:49 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:39:50 | INFO | Train Epoch: 4 [2461696/2572288 (96%)] Data (t): 0.378 Batch (t): 2.383, 1717.70/s, 858.848/s/gpu LR: 0.000000 Logit Scale: 16.819 Contrastive_loss: 6.3287 (6.3706) Loss: 6.3287 (6.3706)
+2024-09-09,18:39:57 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:40:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:40:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:40:14 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:40:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:40:28 | WARNING | Handling webdataset error (FileNotFoundError(2, 'No such file or directory', '')). Ignoring.
+2024-09-09,18:40:54 | INFO | Train Epoch: 4 [2572288/2572288 (100%)] Data (t): 0.375 Batch (t): 2.381, 1722.11/s, 861.056/s/gpu LR: 0.000000 Logit Scale: 16.819 Contrastive_loss: 6.3303 (6.3656) Loss: 6.3303 (6.3656)