Add files using upload-large-folder tool

adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-500M-Base-22750",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d63a77b22bcc13e29da69bec5e1c59a242c86973f51c0d40c72555b2a5c5e0
+size 69532280

added_tokens.json

@@ -0,0 +1,42 @@
+{
+  "<end_of_utterance>": 49191,
+  "<fake_token_around_image>": 49189,
+  "<global-img>": 49152,
+  "<image>": 49190,
+  "<row_1_col_1>": 49153,
+  "<row_1_col_2>": 49154,
+  "<row_1_col_3>": 49155,
+  "<row_1_col_4>": 49156,
+  "<row_1_col_5>": 49157,
+  "<row_1_col_6>": 49158,
+  "<row_2_col_1>": 49159,
+  "<row_2_col_2>": 49160,
+  "<row_2_col_3>": 49161,
+  "<row_2_col_4>": 49162,
+  "<row_2_col_5>": 49163,
+  "<row_2_col_6>": 49164,
+  "<row_3_col_1>": 49165,
+  "<row_3_col_2>": 49166,
+  "<row_3_col_3>": 49167,
+  "<row_3_col_4>": 49168,
+  "<row_3_col_5>": 49169,
+  "<row_3_col_6>": 49170,
+  "<row_4_col_1>": 49171,
+  "<row_4_col_2>": 49172,
+  "<row_4_col_3>": 49173,
+  "<row_4_col_4>": 49174,
+  "<row_4_col_5>": 49175,
+  "<row_4_col_6>": 49176,
+  "<row_5_col_1>": 49177,
+  "<row_5_col_2>": 49178,
+  "<row_5_col_3>": 49179,
+  "<row_5_col_4>": 49180,
+  "<row_5_col_5>": 49181,
+  "<row_5_col_6>": 49182,
+  "<row_6_col_1>": 49183,
+  "<row_6_col_2>": 49184,
+  "<row_6_col_3>": 49185,
+  "<row_6_col_4>": 49186,
+  "<row_6_col_5>": 49187,
+  "<row_6_col_6>": 49188
+}

chat_template.json

@@ -0,0 +1,3 @@
+{
+  "chat_template": "<|im_start|>{% for message in messages %}{{message['role'].capitalize()}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<|endoftext|>\n{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}"
+}

checkpoint-11082/README.md

@@ -0,0 +1,202 @@
+---
+base_model: ./models/ColSmolVLM-500M-Base-22750
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.14.0

checkpoint-11082/adapter_config.json

@@ -0,0 +1,29 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-500M-Base-22750",
+  "bias": "none",
+  "eva_config": null,
+  "exclude_modules": null,
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_bias": false,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-11082/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3d63a77b22bcc13e29da69bec5e1c59a242c86973f51c0d40c72555b2a5c5e0
+size 69532280

checkpoint-11082/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c1ea642a2fa9ac57a3f0d136c59cba31643c60bc65e30c7ef51c72380ba08a69
+size 139313234

checkpoint-11082/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08787b4c84ebe89b7e59f5736f7ca49e58e23ff7d9d4ba8bb3f329511b062150
+size 14244

checkpoint-11082/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6492f98570ecb136d3a3c030cd8d38f5d102dfae6ace1f3bfc893278748c61b
+size 1064

checkpoint-11082/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9544ee67dd5650ee41dced03f8b10c5fec082b4e2db4164006d12d43e1c192c6
+size 5304

config.json

@@ -0,0 +1,265 @@
+{
+  "_name_or_path": "models/SmolVLM-500M-Base-22750/",
+  "architectures": [
+    "ColIdefics3"
+  ],
+  "image_token_id": 49190,
+  "model_type": "idefics3",
+  "scale_factor": 4,
+  "text_config": {
+    "_attn_implementation_autoset": false,
+    "_flash_attn_2_enabled": true,
+    "_name_or_path": "/fsx/m4/experiments/local_experiment_dir/tr_343_vsmollm2_05b/opt_step-22750/unwrapped_model",
+    "add_cross_attention": false,
+    "architectures": [
+      "VLlama3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 1,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 2,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 960,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 2560,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "is_llama_config": true,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 8192,
+    "min_length": 0,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "neftune_noise_alpha": 0.0,
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 15,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 32,
+    "num_key_value_heads": 5,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": 2,
+    "perceiver_config": {
+      "_attn_implementation_autoset": false,
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "silu",
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "vllama3",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_key_value_heads": 1,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "pruned_heads": {},
+      "qk_layer_norms_perceiver": false,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "resampler_depth": 6,
+      "resampler_head_dim": 96,
+      "resampler_n_heads": 16,
+      "resampler_n_latents": 64,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "transformers_version": "4.46.0",
+      "typical_p": 1.0,
+      "use_bfloat16": false
+    },
+    "pixel_shuffle_factor": 4,
+    "prefix": null,
+    "pretraining_tp": 1,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qk_layer_norms": false,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-05,
+    "rope_interleaved": false,
+    "rope_scaling": null,
+    "rope_theta": 100000,
+    "sep_token_id": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers.js_config": {
+      "kv_cache_dtype": {
+        "fp16": "float16",
+        "q4f16": "float16"
+      }
+    },
+    "typical_p": 1.0,
+    "use_bfloat16": false,
+    "use_cache": true,
+    "use_resampler": false,
+    "vocab_size": 49280
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vision_config": {
+    "_attn_implementation_autoset": false,
+    "_name_or_path": "",
+    "add_cross_attention": false,
+    "architectures": null,
+    "attention_dropout": 0.0,
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 768,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 512,
+    "initializer_range": 0.02,
+    "intermediate_size": 3072,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_image_size": {
+      "longest_edge": 512
+    },
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "idefics3",
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_hidden_layers": 12,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 16,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "size": {
+      "longest_edge": 2048
+    },
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": null,
+    "torchscript": false,
+    "typical_p": 1.0,
+    "use_bfloat16": false
+  },
+  "vocab_size": 49280
+}

dump/tmp_adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "./models/ColSmolVLM-500M-Base-22750",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

dump/tmp_adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5408581fc45f84241560c7f2835bc0178d5ff4985af0d5e75f606bda632720c
+size 34798496

git_hash.txt

@@ -0,0 +1 @@
+8b4f75e476bddc6c34a02722761bd3ada6ac0d3d

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:015e1791f643baae6f2d74a8a082f0f3b24ee740041993d36be002ba1de9d4c0
+size 920654272

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "do_convert_rgb": true,
+  "do_image_splitting": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Idefics3ImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_image_size": {
+    "longest_edge": 512
+  },
+  "processor_class": "ColIdefics3Processor",
+  "resample": 1,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 2048
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_len": 64,
+  "processor_class": "Idefics3Processor"
+}

results.json

@@ -0,0 +1 @@
+{"./data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.648, "ndcg_at_3": 0.71821, "ndcg_at_5": 0.73147, "ndcg_at_10": 0.7444, "ndcg_at_20": 0.75657, "ndcg_at_50": 0.76882, "ndcg_at_100": 0.77437, "map_at_1": 0.648, "map_at_3": 0.701, "map_at_5": 0.7084, "map_at_10": 0.71373, "map_at_20": 0.71708, "map_at_50": 0.71904, "map_at_100": 0.71954, "recall_at_1": 0.648, "recall_at_3": 0.768, "recall_at_5": 0.8, "recall_at_10": 0.84, "recall_at_20": 0.888, "recall_at_50": 0.95, "recall_at_100": 0.984, "precision_at_1": 0.648, "precision_at_3": 0.256, "precision_at_5": 0.16, "precision_at_10": 0.084, "precision_at_20": 0.0444, "precision_at_50": 0.019, "precision_at_100": 0.00984, "mrr_at_1": 0.652, "mrr_at_3": 0.703, "mrr_at_5": 0.7096, "mrr_at_10": 0.7163452380952381, "mrr_at_20": 0.7195925928990171, "mrr_at_50": 0.7213793558075785, "mrr_at_100": 0.7218459925368063, "naucs_at_1_max": 0.5320415548033148, "naucs_at_1_std": -0.09459100164714764, "naucs_at_1_diff1": 0.9007496465766573, "naucs_at_3_max": 0.5235472456346243, "naucs_at_3_std": -0.09979906824566964, "naucs_at_3_diff1": 0.8154895266545747, "naucs_at_5_max": 0.5494205794205791, "naucs_at_5_std": -0.07948051948052083, "naucs_at_5_diff1": 0.7935714285714291, "naucs_at_10_max": 0.6103399607265577, "naucs_at_10_std": -0.012493863524791362, "naucs_at_10_diff1": 0.7950233186057926, "naucs_at_20_max": 0.6541277732683969, "naucs_at_20_std": 0.12420522186147001, "naucs_at_20_diff1": 0.8013984713203455, "naucs_at_50_max": 0.7840896358543438, "naucs_at_50_std": 0.21335200746965483, "naucs_at_50_diff1": 0.8345471521942116, "naucs_at_100_max": 0.8932072829131755, "naucs_at_100_std": 0.2987861811391213, "naucs_at_100_diff1": 0.9183006535947711}, "./data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.46785, "ndcg_at_3": 0.53315, "ndcg_at_5": 0.56073, "ndcg_at_10": 0.58063, "ndcg_at_20": 0.59491, "ndcg_at_50": 0.60895, "ndcg_at_100": 0.61612, "map_at_1": 0.46785, "map_at_3": 0.51737, "map_at_5": 0.53278, "map_at_10": 0.5413, "map_at_20": 0.54536, "map_at_50": 0.5476, "map_at_100": 0.54822, "recall_at_1": 0.46785, "recall_at_3": 0.57871, "recall_at_5": 0.64523, "recall_at_10": 0.7051, "recall_at_20": 0.76053, "recall_at_50": 0.83149, "recall_at_100": 0.87583, "precision_at_1": 0.46785, "precision_at_3": 0.1929, "precision_at_5": 0.12905, "precision_at_10": 0.07051, "precision_at_20": 0.03803, "precision_at_50": 0.01663, "precision_at_100": 0.00876, "mrr_at_1": 0.4722838137472284, "mrr_at_3": 0.5203252032520325, "mrr_at_5": 0.5344050258684405, "mrr_at_10": 0.54401418364833, "mrr_at_20": 0.547432357060943, "mrr_at_50": 0.5496278114926427, "mrr_at_100": 0.5502293224133955, "naucs_at_1_max": 0.22191473813157103, "naucs_at_1_std": 0.43792922230870507, "naucs_at_1_diff1": 0.846614423933555, "naucs_at_3_max": 0.14455270946312818, "naucs_at_3_std": 0.5599978573011983, "naucs_at_3_diff1": 0.7812484213095876, "naucs_at_5_max": 0.07904307020932648, "naucs_at_5_std": 0.5873613578608561, "naucs_at_5_diff1": 0.7507308300016783, "naucs_at_10_max": -0.00569065025671971, "naucs_at_10_std": 0.6086341964529604, "naucs_at_10_diff1": 0.7216978702993005, "naucs_at_20_max": -0.08386122115622482, "naucs_at_20_std": 0.6774737565173128, "naucs_at_20_diff1": 0.6877353492008539, "naucs_at_50_max": -0.22730282183008066, "naucs_at_50_std": 0.7387768986158063, "naucs_at_50_diff1": 0.6755005771418886, "naucs_at_100_max": -0.3500275419596926, "naucs_at_100_std": 0.8080829651530356, "naucs_at_100_diff1": 0.6506258023643423}, "./data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.93286, "ndcg_at_5": 0.94103, "ndcg_at_10": 0.94419, "ndcg_at_20": 0.94419, "ndcg_at_50": 0.94609, "ndcg_at_100": 0.94609, "map_at_1": 0.89, "map_at_3": 0.92333, "map_at_5": 0.92783, "map_at_10": 0.92908, "map_at_20": 0.92908, "map_at_50": 0.92935, "map_at_100": 0.92935, "recall_at_1": 0.89, "recall_at_3": 0.96, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.32, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.9333333333333332, "mrr_at_5": 0.9378333333333333, "mrr_at_10": 0.9395, "mrr_at_20": 0.9395, "mrr_at_50": 0.9397777777777777, "mrr_at_100": 0.9397777777777777, "naucs_at_1_max": 0.36255476333648357, "naucs_at_1_std": -0.16244308908169428, "naucs_at_1_diff1": 0.9488875526157551, "naucs_at_3_max": 0.6091269841269833, "naucs_at_3_std": 0.12313258636788006, "naucs_at_3_diff1": 0.9305555555555564, "naucs_at_5_max": 0.7957516339869297, "naucs_at_5_std": 0.09337068160597826, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.8692810457516413, "naucs_at_10_std": -0.1713352007469681, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.8692810457516413, "naucs_at_20_std": -0.1713352007469681, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.62333, "ndcg_at_3": 0.72622, "ndcg_at_5": 0.75278, "ndcg_at_10": 0.774, "ndcg_at_20": 0.78172, "ndcg_at_50": 0.78651, "ndcg_at_100": 0.78807, "map_at_1": 0.62333, "map_at_3": 0.70089, "map_at_5": 0.71565, "map_at_10": 0.72464, "map_at_20": 0.72678, "map_at_50": 0.72758, "map_at_100": 0.72772, "recall_at_1": 0.62333, "recall_at_3": 0.79951, "recall_at_5": 0.86391, "recall_at_10": 0.92831, "recall_at_20": 0.95869, "recall_at_50": 0.98238, "recall_at_100": 0.9921, "precision_at_1": 0.62333, "precision_at_3": 0.2665, "precision_at_5": 0.17278, "precision_at_10": 0.09283, "precision_at_20": 0.04793, "precision_at_50": 0.01965, "precision_at_100": 0.00992, "mrr_at_1": 0.6245443499392467, "mrr_at_3": 0.7006885378695827, "mrr_at_5": 0.7164540299716484, "mrr_at_10": 0.7250101255569056, "mrr_at_20": 0.7271291712945228, "mrr_at_50": 0.7279352643901574, "mrr_at_100": 0.728074338884466, "naucs_at_1_max": 0.18350079884403742, "naucs_at_1_std": -0.24504541716152195, "naucs_at_1_diff1": 0.788310462608764, "naucs_at_3_max": 0.2174478161548601, "naucs_at_3_std": -0.20875995619294158, "naucs_at_3_diff1": 0.6747002810297338, "naucs_at_5_max": 0.20232749356597957, "naucs_at_5_std": -0.2066338463888926, "naucs_at_5_diff1": 0.6660532813247335, "naucs_at_10_max": 0.3515538559874108, "naucs_at_10_std": 0.028956552272515996, "naucs_at_10_diff1": 0.6143796841665605, "naucs_at_20_max": 0.4075526493649992, "naucs_at_20_std": 0.30084166201853924, "naucs_at_20_diff1": 0.569699607771098, "naucs_at_50_max": 0.5357294959420121, "naucs_at_50_std": 0.5086348340492322, "naucs_at_50_diff1": 0.5257337269588929, "naucs_at_100_max": 0.5784455619019262, "naucs_at_100_std": 0.5499580299143445, "naucs_at_100_diff1": 0.6538225911609866}, "./data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.76518, "ndcg_at_3": 0.82176, "ndcg_at_5": 0.83004, "ndcg_at_10": 0.84013, "ndcg_at_20": 0.84547, "ndcg_at_50": 0.85151, "ndcg_at_100": 0.85344, "map_at_1": 0.76518, "map_at_3": 0.80837, "map_at_5": 0.81292, "map_at_10": 0.81725, "map_at_20": 0.81883, "map_at_50": 0.81982, "map_at_100": 0.81997, "recall_at_1": 0.76518, "recall_at_3": 0.86032, "recall_at_5": 0.88057, "recall_at_10": 0.91093, "recall_at_20": 0.93117, "recall_at_50": 0.96154, "recall_at_100": 0.97368, "precision_at_1": 0.76518, "precision_at_3": 0.28677, "precision_at_5": 0.17611, "precision_at_10": 0.09109, "precision_at_20": 0.04656, "precision_at_50": 0.01923, "precision_at_100": 0.00974, "mrr_at_1": 0.7692307692307693, "mrr_at_3": 0.8120782726045884, "mrr_at_5": 0.8161268556005399, "mrr_at_10": 0.8204557869031553, "mrr_at_20": 0.8221355547683245, "mrr_at_50": 0.822944584158538, "mrr_at_100": 0.8231490232346909, "naucs_at_1_max": 0.5149096264576689, "naucs_at_1_std": 0.028876109416755995, "naucs_at_1_diff1": 0.8423502810401294, "naucs_at_3_max": 0.5357633382771757, "naucs_at_3_std": 0.006279786216531792, "naucs_at_3_diff1": 0.7627339364022564, "naucs_at_5_max": 0.5194080180551114, "naucs_at_5_std": 0.016244825987351323, "naucs_at_5_diff1": 0.7459578410036671, "naucs_at_10_max": 0.5866947683147792, "naucs_at_10_std": 0.24003115026037278, "naucs_at_10_diff1": 0.7051949282237744, "naucs_at_20_max": 0.5682575132928513, "naucs_at_20_std": 0.2770052352671666, "naucs_at_20_diff1": 0.6887081088160373, "naucs_at_50_max": 0.7594010289684164, "naucs_at_50_std": 0.7008338816869777, "naucs_at_50_diff1": 0.658391093237802, "naucs_at_100_max": 0.8815298509718311, "naucs_at_100_std": 0.8291345987814791, "naucs_at_100_diff1": 0.5748643414505342}, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.9, "ndcg_at_3": 0.96178, "ndcg_at_5": 0.96178, "ndcg_at_10": 0.96178, "ndcg_at_20": 0.96178, "ndcg_at_50": 0.96178, "ndcg_at_100": 0.96178, "map_at_1": 0.9, "map_at_3": 0.94833, "map_at_5": 0.94833, "map_at_10": 0.94833, "map_at_20": 0.94833, "map_at_50": 0.94833, "map_at_100": 0.94833, "recall_at_1": 0.9, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.9483333333333333, "mrr_at_5": 0.9483333333333333, "mrr_at_10": 0.9483333333333333, "mrr_at_20": 0.9483333333333333, "mrr_at_50": 0.9483333333333333, "mrr_at_100": 0.9483333333333333, "naucs_at_1_max": 0.38183940242763664, "naucs_at_1_std": -0.3154995331465938, "naucs_at_1_diff1": 0.9607843137254906, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.67143, "ndcg_at_3": 0.72355, "ndcg_at_5": 0.73831, "ndcg_at_10": 0.76065, "ndcg_at_20": 0.77133, "ndcg_at_50": 0.78726, "ndcg_at_100": 0.78967, "map_at_1": 0.67143, "map_at_3": 0.71071, "map_at_5": 0.71893, "map_at_10": 0.72838, "map_at_20": 0.73124, "map_at_50": 0.73393, "map_at_100": 0.73417, "recall_at_1": 0.67143, "recall_at_3": 0.76071, "recall_at_5": 0.79643, "recall_at_10": 0.86429, "recall_at_20": 0.90714, "recall_at_50": 0.98571, "recall_at_100": 1.0, "precision_at_1": 0.67143, "precision_at_3": 0.25357, "precision_at_5": 0.15929, "precision_at_10": 0.08643, "precision_at_20": 0.04536, "precision_at_50": 0.01971, "precision_at_100": 0.01, "mrr_at_1": 0.6714285714285714, "mrr_at_3": 0.7107142857142857, "mrr_at_5": 0.71875, "mrr_at_10": 0.7274503968253968, "mrr_at_20": 0.7311492984354826, "mrr_at_50": 0.7336928024479792, "mrr_at_100": 0.7339324686957099, "naucs_at_1_max": 0.2918131988067234, "naucs_at_1_std": 0.10674613761490172, "naucs_at_1_diff1": 0.7514915961291264, "naucs_at_3_max": 0.2682433845224536, "naucs_at_3_std": 0.08520936427913174, "naucs_at_3_diff1": 0.6522779546035362, "naucs_at_5_max": 0.23648809157215928, "naucs_at_5_std": 0.11365575977265355, "naucs_at_5_diff1": 0.6022576302800486, "naucs_at_10_max": 0.1419792355845087, "naucs_at_10_std": 0.04070181688635578, "naucs_at_10_diff1": 0.6177159142958931, "naucs_at_20_max": 0.15348703583997816, "naucs_at_20_std": -0.004776269482152739, "naucs_at_20_diff1": 0.5820045967104782, "naucs_at_50_max": 0.8558590102707799, "naucs_at_50_std": 0.6790382819794502, "naucs_at_50_diff1": 0.8231792717086844, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.87, "ndcg_at_3": 0.94047, "ndcg_at_5": 0.94478, "ndcg_at_10": 0.94478, "ndcg_at_20": 0.94478, "ndcg_at_50": 0.94478, "ndcg_at_100": 0.94478, "map_at_1": 0.87, "map_at_3": 0.92333, "map_at_5": 0.92583, "map_at_10": 0.92583, "map_at_20": 0.92583, "map_at_50": 0.92583, "map_at_100": 0.92583, "recall_at_1": 0.87, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.87, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.88, "mrr_at_3": 0.93, "mrr_at_5": 0.9325, "mrr_at_10": 0.9325, "mrr_at_20": 0.9325, "mrr_at_50": 0.9325, "mrr_at_100": 0.9325, "naucs_at_1_max": 0.16200103680663608, "naucs_at_1_std": -0.2467229504554533, "naucs_at_1_diff1": 0.8407761238243359, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 0.35807656395889226, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.54, "ndcg_at_3": 0.64309, "ndcg_at_5": 0.69258, "ndcg_at_10": 0.71262, "ndcg_at_20": 0.72955, "ndcg_at_50": 0.73167, "ndcg_at_100": 0.73337, "map_at_1": 0.54, "map_at_3": 0.61667, "map_at_5": 0.64417, "map_at_10": 0.65282, "map_at_20": 0.65707, "map_at_50": 0.65747, "map_at_100": 0.65764, "recall_at_1": 0.54, "recall_at_3": 0.72, "recall_at_5": 0.84, "recall_at_10": 0.9, "recall_at_20": 0.97, "recall_at_50": 0.98, "recall_at_100": 0.99, "precision_at_1": 0.54, "precision_at_3": 0.24, "precision_at_5": 0.168, "precision_at_10": 0.09, "precision_at_20": 0.0485, "precision_at_50": 0.0196, "precision_at_100": 0.0099, "mrr_at_1": 0.54, "mrr_at_3": 0.6233333333333334, "mrr_at_5": 0.6463333333333334, "mrr_at_10": 0.655984126984127, "mrr_at_20": 0.659218253968254, "mrr_at_50": 0.66014898989899, "mrr_at_100": 0.6603275613275613, "naucs_at_1_max": 0.06050123642021599, "naucs_at_1_std": -0.24067127893084392, "naucs_at_1_diff1": 0.586299352615932, "naucs_at_3_max": 0.2029532967032969, "naucs_at_3_std": -0.11450702075702028, "naucs_at_3_diff1": 0.5169413919413928, "naucs_at_5_max": 0.22760186548846306, "naucs_at_5_std": -0.12404884634265974, "naucs_at_5_diff1": 0.44949680903289124, "naucs_at_10_max": 0.06274509803921678, "naucs_at_10_std": -0.3894957983193242, "naucs_at_10_diff1": 0.5401960784313737, "naucs_at_20_max": -0.30376595082477437, "naucs_at_20_std": -0.4344849050731363, "naucs_at_20_diff1": 0.5154061624649885, "naucs_at_50_max": -0.9556489262371552, "naucs_at_50_std": -1.1517273576097031, "naucs_at_50_diff1": 0.3384687208216665, "naucs_at_100_max": -0.17133520074697067, "naucs_at_100_std": -0.5634920634920583, "naucs_at_100_diff1": 0.554154995331464}, "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.95, "ndcg_at_3": 0.98024, "ndcg_at_5": 0.98024, "ndcg_at_10": 0.98024, "ndcg_at_20": 0.98024, "ndcg_at_50": 0.98024, "ndcg_at_100": 0.98024, "map_at_1": 0.95, "map_at_3": 0.97333, "map_at_5": 0.97333, "map_at_10": 0.97333, "map_at_20": 0.97333, "map_at_50": 0.97333, "map_at_100": 0.97333, "recall_at_1": 0.95, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_50": 0.9783333333333333, "mrr_at_100": 0.9783333333333333, "naucs_at_1_max": 0.3213818860877688, "naucs_at_1_std": -0.07721755368813897, "naucs_at_1_diff1": 0.9738562091503253, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}}

special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,479 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<global-img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<row_1_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<row_1_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49155": {
+      "content": "<row_1_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49156": {
+      "content": "<row_1_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49157": {
+      "content": "<row_1_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49158": {
+      "content": "<row_1_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49159": {
+      "content": "<row_2_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49160": {
+      "content": "<row_2_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49161": {
+      "content": "<row_2_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49162": {
+      "content": "<row_2_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49163": {
+      "content": "<row_2_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49164": {
+      "content": "<row_2_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49165": {
+      "content": "<row_3_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49166": {
+      "content": "<row_3_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49167": {
+      "content": "<row_3_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49168": {
+      "content": "<row_3_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49169": {
+      "content": "<row_3_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49170": {
+      "content": "<row_3_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49171": {
+      "content": "<row_4_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49172": {
+      "content": "<row_4_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49173": {
+      "content": "<row_4_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49174": {
+      "content": "<row_4_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49175": {
+      "content": "<row_4_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49176": {
+      "content": "<row_4_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49177": {
+      "content": "<row_5_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49178": {
+      "content": "<row_5_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49179": {
+      "content": "<row_5_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49180": {
+      "content": "<row_5_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49181": {
+      "content": "<row_5_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49182": {
+      "content": "<row_5_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49183": {
+      "content": "<row_6_col_1>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49184": {
+      "content": "<row_6_col_2>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49185": {
+      "content": "<row_6_col_3>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49186": {
+      "content": "<row_6_col_4>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49187": {
+      "content": "<row_6_col_5>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49188": {
+      "content": "<row_6_col_6>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49189": {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49190": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49191": {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<fake_token_around_image>",
+    "<image>",
+    "<end_of_utterance>"
+  ],
+  "bos_token": "<|im_start|>",
+  "chat_template": "{% for message in messages %}{% if loop.first and messages[0]['role'] != 'system' %}{{ '<|im_start|>system\nYou are a helpful AI assistant named SmolLM, trained by Hugging Face<|im_end|>\n' }}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 8192,
+  "pad_token": "<|im_end|>",
+  "processor_class": "ColIdefics3Processor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}

training_config.yml

@@ -0,0 +1,72 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/ColSmolVLM-500M-Base-22750
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColIdefics3Processor
+    pretrained_model_name_or_path:  "./models/ColSmolVLM-500M-Base-22750"
+    # num_image_tokens: 2048
+    # max_length: 50
+
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColIdefics3
+    pretrained_model_name_or_path: "./models/ColSmolVLM-500M-Base-22750"
+    torch_dtype:  !ext torch.bfloat16
+    # use_cache: false
+    attn_implementation: "flash_attention_2"
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  # max_length: 50
+  run_eval: true
+  loss_func:
+    (): colpali_engine.loss.late_interaction_losses.ColbertPairwiseCELoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 3
+    per_device_train_batch_size: 8
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    # gradient_checkpointing: true
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 4
+    eval_strategy: "steps"
+    dataloader_num_workers: 6
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_steps: 100
+    learning_rate: 5e-4
+    save_total_limit: 1
+    resume_from_checkpoint: true
+    # optim: "paged_adamw_8bit"
+    # wandb logging
+    # wandb_project: "colqwen2"
+    # run_name: "colqwen2-ba32-nolora"
+    report_to: "wandb"
+
+
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(model.text_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+