Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: vidore/colpaligemma-3b-pt-448-base
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "vidore/colpaligemma-3b-pt-448-base",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1f2fb23f46e0f545ee309449f02f9a31236a36dc76da24b9f8e6a8dd4fda66f3
+size 78625112

git_hash.txt

@@ -0,0 +1 @@
+7fecd19a97ea79b26776b8e6c70ee060cde20687

preprocessor_config.json

@@ -0,0 +1,25 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SiglipImageProcessor",
+  "image_seq_length": 1024,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "ColPaliProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 448,
+    "width": 448
+  }
+}

results.json

@@ -0,0 +1 @@
+{"./data_dir/eval_vidore/tatdqa_test": {"ndcg_at_1": 0.57108, "ndcg_at_3": 0.67493, "ndcg_at_5": 0.70361, "ndcg_at_10": 0.72693, "ndcg_at_20": 0.73559, "ndcg_at_50": 0.74195, "ndcg_at_100": 0.74529, "map_at_1": 0.57108, "map_at_3": 0.64895, "map_at_5": 0.6648, "map_at_10": 0.67439, "map_at_20": 0.6768, "map_at_50": 0.67786, "map_at_100": 0.67815, "recall_at_1": 0.57108, "recall_at_3": 0.7503, "recall_at_5": 0.82017, "recall_at_10": 0.89247, "recall_at_20": 0.92649, "recall_at_50": 0.95808, "recall_at_100": 0.97874, "precision_at_1": 0.57108, "precision_at_3": 0.2501, "precision_at_5": 0.16403, "precision_at_10": 0.08925, "precision_at_20": 0.04632, "precision_at_50": 0.01916, "precision_at_100": 0.00979, "mrr_at_1": 0.56865127582017, "mrr_at_3": 0.6471243418388012, "mrr_at_5": 0.6632239773187526, "mrr_at_10": 0.6727636984319852, "mrr_at_20": 0.6753542309195802, "mrr_at_50": 0.6765612284057695, "mrr_at_100": 0.6767635994564822, "naucs_at_1_max": 0.19627696906414918, "naucs_at_1_std": 0.25490113312887763, "naucs_at_1_diff1": 0.7311472112111653, "naucs_at_3_max": 0.12505762179060695, "naucs_at_3_std": 0.3250883135485268, "naucs_at_3_diff1": 0.6368932102920861, "naucs_at_5_max": 0.12681693174662004, "naucs_at_5_std": 0.36553579145484827, "naucs_at_5_diff1": 0.5963185435608184, "naucs_at_10_max": 0.2443283094718615, "naucs_at_10_std": 0.5378975291557723, "naucs_at_10_diff1": 0.6056682957261833, "naucs_at_20_max": 0.2941297289999579, "naucs_at_20_std": 0.5490669907768836, "naucs_at_20_diff1": 0.5965035030640417, "naucs_at_50_max": 0.21733999338436122, "naucs_at_50_std": 0.5631005588549655, "naucs_at_50_diff1": 0.6097410948123246, "naucs_at_100_max": 0.21831800388731984, "naucs_at_100_std": 0.710384903173119, "naucs_at_100_diff1": 0.6734222845071824}, "./data_dir/eval_vidore/shiftproject_test": {"ndcg_at_1": 0.63, "ndcg_at_3": 0.74071, "ndcg_at_5": 0.77385, "ndcg_at_10": 0.78346, "ndcg_at_20": 0.78886, "ndcg_at_50": 0.79529, "ndcg_at_100": 0.79852, "map_at_1": 0.63, "map_at_3": 0.71333, "map_at_5": 0.73183, "map_at_10": 0.73575, "map_at_20": 0.73742, "map_at_50": 0.73865, "map_at_100": 0.73894, "recall_at_1": 0.63, "recall_at_3": 0.82, "recall_at_5": 0.9, "recall_at_10": 0.93, "recall_at_20": 0.95, "recall_at_50": 0.98, "recall_at_100": 1.0, "precision_at_1": 0.63, "precision_at_3": 0.27333, "precision_at_5": 0.18, "precision_at_10": 0.093, "precision_at_20": 0.0475, "precision_at_50": 0.0196, "precision_at_100": 0.01, "mrr_at_1": 0.63, "mrr_at_3": 0.7183333333333333, "mrr_at_5": 0.7348333333333333, "mrr_at_10": 0.7408611111111111, "mrr_at_20": 0.741361111111111, "mrr_at_50": 0.7421598822324629, "mrr_at_100": 0.7424503305755428, "naucs_at_1_max": -0.0974004606778548, "naucs_at_1_std": -0.25468904244817375, "naucs_at_1_diff1": 0.6640185524687, "naucs_at_3_max": -0.3256691265557873, "naucs_at_3_std": -0.5354389249917381, "naucs_at_3_diff1": 0.624628263024562, "naucs_at_5_max": -0.22777777777777553, "naucs_at_5_std": -0.5987861811391184, "naucs_at_5_diff1": 0.46909430438842176, "naucs_at_10_max": -0.18354008269974378, "naucs_at_10_std": -0.4598506069094287, "naucs_at_10_diff1": 0.47258903561424775, "naucs_at_20_max": -0.1884220354808507, "naucs_at_20_std": -0.3007469654528384, "naucs_at_20_diff1": 0.4926237161531324, "naucs_at_50_max": 0.5401493930905731, "naucs_at_50_std": -0.22035480859009582, "naucs_at_50_diff1": 0.19140989729225052, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.97, "ndcg_at_5": 0.97431, "ndcg_at_10": 0.9812, "ndcg_at_20": 0.9812, "ndcg_at_50": 0.9812, "ndcg_at_100": 0.9812, "map_at_1": 0.97, "map_at_3": 0.97, "map_at_5": 0.9725, "map_at_10": 0.9756, "map_at_20": 0.9756, "map_at_50": 0.9756, "map_at_100": 0.9756, "recall_at_1": 0.97, "recall_at_3": 0.97, "recall_at_5": 0.98, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.32333, "precision_at_5": 0.196, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.97, "mrr_at_3": 0.9733333333333333, "mrr_at_5": 0.9733333333333333, "mrr_at_10": 0.9764285714285714, "mrr_at_20": 0.9764285714285714, "mrr_at_50": 0.9764285714285714, "mrr_at_100": 0.9764285714285714, "naucs_at_1_max": 0.5659819483348937, "naucs_at_1_std": -0.7791783380018684, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 0.5659819483348902, "naucs_at_3_std": -0.7791783380018689, "naucs_at_3_diff1": 0.9564270152505466, "naucs_at_5_max": 0.9346405228758136, "naucs_at_5_std": -0.5929038281979465, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.91, "ndcg_at_3": 0.95786, "ndcg_at_5": 0.96216, "ndcg_at_10": 0.96216, "ndcg_at_20": 0.96216, "ndcg_at_50": 0.96216, "ndcg_at_100": 0.96216, "map_at_1": 0.91, "map_at_3": 0.94667, "map_at_5": 0.94917, "map_at_10": 0.94917, "map_at_20": 0.94917, "map_at_50": 0.94917, "map_at_100": 0.94917, "recall_at_1": 0.91, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.91, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.91, "mrr_at_3": 0.9466666666666668, "mrr_at_5": 0.9491666666666667, "mrr_at_10": 0.9491666666666667, "mrr_at_20": 0.9491666666666667, "mrr_at_50": 0.9491666666666667, "mrr_at_100": 0.9491666666666667, "naucs_at_1_max": 0.3833903931943138, "naucs_at_1_std": 0.48848428260192905, "naucs_at_1_diff1": 0.8264861500155604, "naucs_at_3_max": 1.0, "naucs_at_3_std": 0.8692810457516356, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/arxivqa_test_subsampled": {"ndcg_at_1": 0.776, "ndcg_at_3": 0.81376, "ndcg_at_5": 0.83037, "ndcg_at_10": 0.84451, "ndcg_at_20": 0.85313, "ndcg_at_50": 0.85831, "ndcg_at_100": 0.85992, "map_at_1": 0.776, "map_at_3": 0.80467, "map_at_5": 0.81397, "map_at_10": 0.81973, "map_at_20": 0.82211, "map_at_50": 0.82294, "map_at_100": 0.82308, "recall_at_1": 0.776, "recall_at_3": 0.84, "recall_at_5": 0.88, "recall_at_10": 0.924, "recall_at_20": 0.958, "recall_at_50": 0.984, "recall_at_100": 0.994, "precision_at_1": 0.776, "precision_at_3": 0.28, "precision_at_5": 0.176, "precision_at_10": 0.0924, "precision_at_20": 0.0479, "precision_at_50": 0.01968, "precision_at_100": 0.00994, "mrr_at_1": 0.774, "mrr_at_3": 0.8046666666666666, "mrr_at_5": 0.8133666666666667, "mrr_at_10": 0.8186833333333333, "mrr_at_20": 0.8212445641733567, "mrr_at_50": 0.8219611912851504, "mrr_at_100": 0.8221248888102938, "naucs_at_1_max": 0.6541204220072772, "naucs_at_1_std": -0.004310581265656427, "naucs_at_1_diff1": 0.9333299811669199, "naucs_at_3_max": 0.6774423171330395, "naucs_at_3_std": 0.02982326951399226, "naucs_at_3_diff1": 0.9019698085419732, "naucs_at_5_max": 0.7342904867960547, "naucs_at_5_std": 0.12409322303531663, "naucs_at_5_diff1": 0.9104756601972646, "naucs_at_10_max": 0.7714383999213704, "naucs_at_10_std": 0.19174160892426748, "naucs_at_10_diff1": 0.9069978868740466, "naucs_at_20_max": 0.7963629896402987, "naucs_at_20_std": 0.374772131074646, "naucs_at_20_diff1": 0.8955137610599756, "naucs_at_50_max": 0.8293067226890712, "naucs_at_50_std": 0.8605275443510737, "naucs_at_50_diff1": 0.9183006535947711, "naucs_at_100_max": 0.6934329287270289, "naucs_at_100_std": 0.8078120136943184, "naucs_at_100_diff1": 0.9128540305010608}, "./data_dir/eval_vidore/docvqa_test_subsampled": {"ndcg_at_1": 0.49446, "ndcg_at_3": 0.56477, "ndcg_at_5": 0.58452, "ndcg_at_10": 0.59669, "ndcg_at_20": 0.60668, "ndcg_at_50": 0.61757, "ndcg_at_100": 0.62554, "map_at_1": 0.49446, "map_at_3": 0.54767, "map_at_5": 0.55843, "map_at_10": 0.56343, "map_at_20": 0.56612, "map_at_50": 0.56802, "map_at_100": 0.56873, "recall_at_1": 0.49446, "recall_at_3": 0.61419, "recall_at_5": 0.66297, "recall_at_10": 0.70067, "recall_at_20": 0.74058, "recall_at_50": 0.79379, "recall_at_100": 0.84257, "precision_at_1": 0.49446, "precision_at_3": 0.20473, "precision_at_5": 0.13259, "precision_at_10": 0.07007, "precision_at_20": 0.03703, "precision_at_50": 0.01588, "precision_at_100": 0.00843, "mrr_at_1": 0.49889135254988914, "mrr_at_3": 0.5447154471544715, "mrr_at_5": 0.5566888396156688, "mrr_at_10": 0.5629526977087953, "mrr_at_20": 0.5652806848414477, "mrr_at_50": 0.5670115584789515, "mrr_at_100": 0.5677538817333595, "naucs_at_1_max": -0.27864580796328964, "naucs_at_1_std": 0.7722092976730863, "naucs_at_1_diff1": 0.8943248317988693, "naucs_at_3_max": -0.44768791859908297, "naucs_at_3_std": 0.8209416026426787, "naucs_at_3_diff1": 0.8513206720284808, "naucs_at_5_max": -0.5206642942420097, "naucs_at_5_std": 0.8590967813938516, "naucs_at_5_diff1": 0.8553385206717447, "naucs_at_10_max": -0.5861667680971102, "naucs_at_10_std": 0.8921235191636205, "naucs_at_10_diff1": 0.8508472594083637, "naucs_at_20_max": -0.6153314648071393, "naucs_at_20_std": 0.8966962245596599, "naucs_at_20_diff1": 0.8514213312670023, "naucs_at_50_max": -0.7316708344718492, "naucs_at_50_std": 0.8827437458106799, "naucs_at_50_diff1": 0.8348654637400126, "naucs_at_100_max": -0.890172253801331, "naucs_at_100_std": 0.8797216559719966, "naucs_at_100_diff1": 0.8182326755467386}, "./data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.93, "ndcg_at_3": 0.96524, "ndcg_at_5": 0.96911, "ndcg_at_10": 0.96911, "ndcg_at_20": 0.96911, "ndcg_at_50": 0.96911, "ndcg_at_100": 0.96911, "map_at_1": 0.93, "map_at_3": 0.95667, "map_at_5": 0.95867, "map_at_10": 0.95867, "map_at_20": 0.95867, "map_at_50": 0.95867, "map_at_100": 0.95867, "recall_at_1": 0.93, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.93, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.9616666666666667, "mrr_at_5": 0.9641666666666667, "mrr_at_10": 0.9641666666666667, "mrr_at_20": 0.9641666666666667, "mrr_at_50": 0.9641666666666667, "mrr_at_100": 0.9641666666666667, "naucs_at_1_max": 0.5512204881952787, "naucs_at_1_std": -0.5522875816993494, "naucs_at_1_diff1": 0.9626517273576113, "naucs_at_3_max": -0.5634920634921204, "naucs_at_3_std": -1.7399626517274398, "naucs_at_3_diff1": 0.8692810457516356, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": NaN, "naucs_at_50_std": NaN, "naucs_at_50_diff1": NaN, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/infovqa_test_subsampled": {"ndcg_at_1": 0.79757, "ndcg_at_3": 0.84618, "ndcg_at_5": 0.85716, "ndcg_at_10": 0.86704, "ndcg_at_20": 0.86964, "ndcg_at_50": 0.87372, "ndcg_at_100": 0.87532, "map_at_1": 0.79757, "map_at_3": 0.83367, "map_at_5": 0.83984, "map_at_10": 0.84396, "map_at_20": 0.84469, "map_at_50": 0.84538, "map_at_100": 0.8455, "recall_at_1": 0.79757, "recall_at_3": 0.88259, "recall_at_5": 0.90891, "recall_at_10": 0.93927, "recall_at_20": 0.94939, "recall_at_50": 0.96964, "recall_at_100": 0.97976, "precision_at_1": 0.79757, "precision_at_3": 0.2942, "precision_at_5": 0.18178, "precision_at_10": 0.09393, "precision_at_20": 0.04747, "precision_at_50": 0.01939, "precision_at_100": 0.0098, "mrr_at_1": 0.7894736842105263, "mrr_at_3": 0.8306342780026991, "mrr_at_5": 0.8363022941970311, "mrr_at_10": 0.8404167469956944, "mrr_at_20": 0.8411396743317877, "mrr_at_50": 0.8418165179225722, "mrr_at_100": 0.8419436154362733, "naucs_at_1_max": 0.5658752085914066, "naucs_at_1_std": 0.08703668327449693, "naucs_at_1_diff1": 0.8969893821656988, "naucs_at_3_max": 0.5035089171640572, "naucs_at_3_std": 0.10634666680591591, "naucs_at_3_diff1": 0.8617291252020982, "naucs_at_5_max": 0.672890311854327, "naucs_at_5_std": 0.2917197093395984, "naucs_at_5_diff1": 0.8715729540210408, "naucs_at_10_max": 0.7402193471144418, "naucs_at_10_std": 0.3670976593963592, "naucs_at_10_diff1": 0.8958712928076012, "naucs_at_20_max": 0.7849669612599903, "naucs_at_20_std": 0.5713453092645688, "naucs_at_20_diff1": 0.8802696095317121, "naucs_at_50_max": 0.7192418710631517, "naucs_at_50_std": 0.7316569995970472, "naucs_at_50_diff1": 0.8624988164249825, "naucs_at_100_max": 0.6626137244104312, "naucs_at_100_std": 0.8415749940072255, "naucs_at_100_diff1": 0.8476419892262101}, "./data_dir/eval_vidore/syntheticDocQA_energy_test": {"ndcg_at_1": 0.94, "ndcg_at_3": 0.94631, "ndcg_at_5": 0.95405, "ndcg_at_10": 0.95405, "ndcg_at_20": 0.95684, "ndcg_at_50": 0.95902, "ndcg_at_100": 0.96074, "map_at_1": 0.94, "map_at_3": 0.945, "map_at_5": 0.949, "map_at_10": 0.949, "map_at_20": 0.94991, "map_at_50": 0.95034, "map_at_100": 0.95053, "recall_at_1": 0.94, "recall_at_3": 0.95, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_50": 0.99, "recall_at_100": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.31667, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_50": 0.0198, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.945, "mrr_at_5": 0.9490000000000001, "mrr_at_10": 0.950111111111111, "mrr_at_20": 0.950111111111111, "mrr_at_50": 0.9505873015873015, "mrr_at_100": 0.9507724867724867, "naucs_at_1_max": 0.5278555866791174, "naucs_at_1_std": -0.5602240896358519, "naucs_at_1_diff1": 0.9782135076252712, "naucs_at_3_max": 0.4595704948646117, "naucs_at_3_std": -0.7438842203548082, "naucs_at_3_diff1": 0.973856209150328, "naucs_at_5_max": 0.4352629940865253, "naucs_at_5_std": -0.8445378151260495, "naucs_at_5_diff1": 0.9564270152505424, "naucs_at_10_max": 0.4352629940865253, "naucs_at_10_std": -0.8445378151260495, "naucs_at_10_diff1": 0.9564270152505424, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": -0.3968253968253954, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 1.0, "naucs_at_50_std": 0.35807656395892007, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN}, "./data_dir/eval_vidore/tabfquad_test_subsampled": {"ndcg_at_1": 0.81429, "ndcg_at_3": 0.86688, "ndcg_at_5": 0.87441, "ndcg_at_10": 0.88583, "ndcg_at_20": 0.89213, "ndcg_at_50": 0.89426, "ndcg_at_100": 0.89548, "map_at_1": 0.81429, "map_at_3": 0.85417, "map_at_5": 0.85845, "map_at_10": 0.8631, "map_at_20": 0.86482, "map_at_50": 0.86516, "map_at_100": 0.86528, "recall_at_1": 0.81429, "recall_at_3": 0.90357, "recall_at_5": 0.92143, "recall_at_10": 0.95714, "recall_at_20": 0.98214, "recall_at_50": 0.99286, "recall_at_100": 1.0, "precision_at_1": 0.81429, "precision_at_3": 0.30119, "precision_at_5": 0.18429, "precision_at_10": 0.09571, "precision_at_20": 0.04911, "precision_at_50": 0.01986, "precision_at_100": 0.01, "mrr_at_1": 0.8142857142857143, "mrr_at_3": 0.8535714285714285, "mrr_at_5": 0.8576785714285714, "mrr_at_10": 0.8620280612244897, "mrr_at_20": 0.8640784760874046, "mrr_at_50": 0.8644115436805921, "mrr_at_100": 0.8645378286859094, "naucs_at_1_max": 0.5099174501261181, "naucs_at_1_std": 0.10524153481617399, "naucs_at_1_diff1": 0.8698884048001222, "naucs_at_3_max": 0.5539647957948607, "naucs_at_3_std": 0.17413632119514313, "naucs_at_3_diff1": 0.8326416986547719, "naucs_at_5_max": 0.478800611153553, "naucs_at_5_std": 0.10332314744079385, "naucs_at_5_diff1": 0.8250572956455312, "naucs_at_10_max": 0.6088935574229674, "naucs_at_10_std": 0.23650015561779797, "naucs_at_10_diff1": 0.8465608465608483, "naucs_at_20_max": 0.8216619981325874, "naucs_at_20_std": 0.6112044817927197, "naucs_at_20_diff1": 0.7984126984127048, "naucs_at_50_max": 1.0, "naucs_at_50_std": 0.9346405228758071, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0}}

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff84f53c290d0348c4e206da6094ef781cf8c0e482fec8b268a996b32257cfd
+size 34600975

training_config.yml

@@ -0,0 +1,62 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/colpali-pt-448-256
+  processor:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColPaliProcessor
+    pretrained_model_name_or_path: "./models/colpaligemma-3b-pt-448-base"
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColPali
+    pretrained_model_name_or_path: "./models/colpaligemma-3b-pt-448-base"
+    torch_dtype:  !ext torch.bfloat16
+    attn_implementation: "flash_attention_2"
+  #    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_train_set
+  eval_dataset_loader: !import ../data/test_data.yaml
+
+  max_length: 50
+  run_eval: true
+  
+  loss_func:
+    (): colpali_engine.loss.late_interaction_losses.ColbertPairwiseCELoss
+  tr_args:
+    (): transformers.training_args.TrainingArguments
+    output_dir: null
+    overwrite_output_dir: true
+    num_train_epochs: 3
+    per_device_train_batch_size: 64
+    gradient_checkpointing: true
+    gradient_checkpointing_kwargs: { "use_reentrant": false }
+    # 6 x 8 gpus = 48 batch size
+    # gradient_accumulation_steps: 4
+    per_device_eval_batch_size: 64
+    eval_strategy: "steps"
+    dataloader_num_workers: 8
+    # bf16: true
+    save_steps: 500
+    logging_steps: 10
+    eval_steps: 100
+    warmup_steps: 100
+    learning_rate: 5e-4
+    save_total_limit: 1
+    resume_from_checkpoint: false
+    report_to: "wandb"
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+