Add configs

prismer/configs/caption.yaml

@@ -0,0 +1,48 @@
+coco:
+  dataset: 'coco'
+  data_path: '/workspace_dataset/dataset_vqa'
+  label_path: '/workspace_dataset/dataset_experts'
+  experts: ['depth', 'normal', 'seg_coco', 'edge', 'obj_detection', 'ocr_detection']  # 'none' for PrismerZ
+  image_resolution: 480
+  prismer_model: 'prismer_base'  # 'prismer-large' for Prismer(Z)-Large
+  freeze: 'freeze_vision'
+
+  batch_size_train: 4  # for 8 * 8 nodes [effective batch-size: 256]
+  batch_size_test: 8
+  init_lr: 5e-5
+  weight_decay: 0.05
+  min_lr: 0
+  max_epoch: 3
+
+  prefix: 'A picture of'  # use prefix for fine-tuning or no pre-fix '' for zero-shot experiments
+
+nocaps:
+  dataset: 'nocaps'
+  data_path: '/workspace_dataset/dataset_vqa'
+  label_path: '/workspace_dataset/dataset_experts'
+  experts: ['depth', 'normal', 'seg_coco', 'edge', 'obj_detection', 'ocr_detection']  # 'none' for PrismerZ
+
+  image_resolution: 480
+  prismer_model: 'prismer_base'  # 'prismer-large' for Prismer(Z)-Large
+  freeze: 'freeze_vision'
+
+  batch_size_train: 4  # for 8 * 8 nodes [effective batch-size: 256]
+  batch_size_test: 8
+  init_lr: 5e-5
+  weight_decay: 0.05
+  min_lr: 0
+  max_epoch: 3
+
+  prefix: 'A picture of' # use prefix for fine-tuning or no pre-fix '' for zero-shot experiments
+
+demo:
+  dataset: 'demo'
+  data_path: 'helpers'
+  label_path: 'helpers/labels'
+  experts: ['depth', 'normal', 'seg_coco', 'edge', 'obj_detection', 'ocr_detection']  # 'none' for PrismerZ
+
+  image_resolution: 480
+  prismer_model: 'prismer_base'  # 'prismer-large' for Prismer(Z)-Large
+  freeze: 'freeze_vision'
+
+  prefix: 'A picture of'

prismer/configs/classification.yaml

@@ -0,0 +1,21 @@
+data_path: '/workspace_dataset/dataset_zero'
+label_path: '/workspace_dataset/dataset_experts'
+
+experts: ['depth', 'normal', 'seg_coco', 'edge', 'obj_detection', 'ocr_detection']   # 'none' for PrismerZ
+
+freeze: 'freeze_vision'
+dataset: 'imagenet'
+shots: 1
+
+image_resolution: 384
+prismer_model: 'prismer_base'  # 'prismer-large' for Prismer(Z)-Large
+
+batch_size_train: 2  # for 4 * 8 nodes [effective batch-size: 64]
+batch_size_test: 8
+init_lr: 5e-5
+weight_decay: 0.05
+min_lr: 0
+max_epoch: 20
+
+k_test: 32
+prefix: 'A photo of a'

prismer/configs/experts.yaml

@@ -0,0 +1,2 @@
+data_path: 'helpers'
+save_path: 'helpers/labels'

prismer/configs/pretrain.yaml

@@ -0,0 +1,21 @@
+datasets: ['cc12m', 'cc3m_sgu', 'coco', 'vg']
+
+cc12m_data_path: '/workspace_dataset/cc12m'
+cc3m_data_path: '/home/datasets/cc3m'
+coco_data_path: '/workspace_dataset/dataset_vqa'
+vg_data_path: '/home/datasets/vqa'
+label_path: '/workspace_dataset/dataset_experts'
+
+experts: ['depth', 'normal', 'seg_coco', 'edge', 'obj_detection', 'ocr_detection']    # 'none' for PrismerZ
+
+image_resolution: 224
+prismer_model: 'prismer_base'  # 'prismer-large' for Prismer(Z)-Large
+freeze: 'freeze_lang_vision'
+batch_size_train: 32  #   for 4 * 8 nodes [effective batch-size: 1024]
+
+max_epoch: 20
+weight_decay: 0.05
+init_lr: 3e-4  # 1e-4 for prismer_large
+min_lr: 1e-6
+warmup_lr: 1e-6
+warmup_steps: 2000

prismer/configs/prismer.json

@@ -0,0 +1,74 @@
+{
+  "prismer_base": {
+     "roberta_model": {
+        "attention_probs_dropout_prob": 0.1,
+        "bos_token_id": 0,
+        "eos_token_id": 2,
+        "hidden_act": "gelu",
+        "hidden_dropout_prob": 0.1,
+        "hidden_size": 768,
+        "vision_hidden_size": 768,
+        "initializer_range": 0.02,
+        "intermediate_size": 3072,
+        "layer_norm_eps": 1e-05,
+        "max_position_embeddings": 514,
+        "model_name": "roberta-base",
+        "num_attention_heads": 12,
+        "num_hidden_layers": 12,
+        "pad_token_id": 1,
+        "type_vocab_size": 1,
+        "vocab_size": 50265,
+        "num_decoder_layers": 4,
+        "is_decoder": true
+   },
+    "vit_model": "ViT-B/16"
+  },
+  "prismer_large": {
+     "roberta_model": {
+        "attention_probs_dropout_prob": 0.1,
+        "bos_token_id": 0,
+        "eos_token_id": 2,
+        "hidden_act": "gelu",
+        "hidden_dropout_prob": 0.1,
+        "hidden_size": 1024,
+        "vision_hidden_size": 1024,
+        "initializer_range": 0.02,
+        "intermediate_size": 4096,
+        "layer_norm_eps": 1e-05,
+        "max_position_embeddings": 514,
+        "model_name": "roberta-large",
+        "num_attention_heads": 16,
+        "num_hidden_layers": 24,
+        "pad_token_id": 1,
+        "type_vocab_size": 1,
+        "vocab_size": 50265,
+        "num_decoder_layers": 4,
+        "is_decoder": true
+    },
+    "vit_model": "ViT-L/14@336px"
+  },
+  "prismer_huge": {
+    "roberta_model": {
+        "attention_probs_dropout_prob": 0.1,
+        "bos_token_id": 0,
+        "eos_token_id": 2,
+        "hidden_act": "gelu",
+        "hidden_dropout_prob": 0.1,
+        "hidden_size": 1024,
+        "vision_hidden_size": 1280,
+        "initializer_range": 0.02,
+        "intermediate_size": 4096,
+        "layer_norm_eps": 1e-05,
+        "max_position_embeddings": 514,
+        "model_name": "roberta-large",
+        "num_attention_heads": 16,
+        "num_hidden_layers": 24,
+        "pad_token_id": 1,
+        "type_vocab_size": 1,
+        "vocab_size": 50265,
+        "num_decoder_layers": 4,
+        "is_decoder": true
+    },
+    "vit_model": "ViT-H/14"
+  }
+}

prismer/configs/roberta.json

@@ -0,0 +1,42 @@
+{
+  "roberta-base": {
+    "attention_probs_dropout_prob": 0.1,
+    "bos_token_id": 0,
+    "eos_token_id": 2,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 768,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "layer_norm_eps": 1e-05,
+    "max_position_embeddings": 514,
+    "model_type": "roberta",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 12,
+    "pad_token_id": 1,
+    "type_vocab_size": 1,
+    "vocab_size": 50265,
+    "num_decoder_layers": 4,
+    "is_decoder": true
+  },
+  "roberta-large": {
+    "attention_probs_dropout_prob": 0.1,
+    "bos_token_id": 0,
+    "eos_token_id": 2,
+    "hidden_act": "gelu",
+    "hidden_dropout_prob": 0.1,
+    "hidden_size": 1024,
+    "initializer_range": 0.02,
+    "intermediate_size": 4096,
+    "layer_norm_eps": 1e-05,
+    "max_position_embeddings": 514,
+    "model_type": "roberta",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 24,
+    "pad_token_id": 1,
+    "type_vocab_size": 1,
+    "vocab_size": 50265,
+    "num_decoder_layers": 4,
+    "is_decoder": true
+  }
+}

prismer/configs/vqa.yaml

@@ -0,0 +1,18 @@
+datasets: ['vqav2', 'vg']
+data_path: '/workspace_dataset/dataset_vqa'
+label_path: '/workspace_dataset/dataset_experts'
+experts: ['depth', 'normal', 'seg_coco', 'edge', 'obj_detection', 'ocr_detection']     # 'none' for PrismerZ
+
+image_resolution: 480
+prismer_model: 'prismer_base'  # 'prismer-large' for Prismer(Z)-Large
+freeze: 'freeze_vision'
+
+batch_size_train: 8   #   for 8 * 8 nodes [effective batch-size: 512]
+batch_size_test: 32
+init_lr: 5e-5
+weight_decay: 0.05
+min_lr: 0
+max_epoch: 10
+
+k_test: 16
+inference: 'rank'