Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,202 @@
+---
+base_model: vidore/colpaligemma-3b-mix-448-base
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "vidore/colpaligemma-3b-mix-448-base",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0acb85e5a2974f7bf545565ff8be191311d76452829b574c56b96b13991d4ac
+size 78625112

checkpoint-587/README.md

@@ -0,0 +1,202 @@
+---
+base_model: vidore/colpaligemma-3b-mix-448-base
+library_name: peft
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

checkpoint-587/adapter_config.json

@@ -0,0 +1,26 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "vidore/colpaligemma-3b-mix-448-base",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": "gaussian",
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 32,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": "(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)",
+  "task_type": "FEATURE_EXTRACTION",
+  "use_dora": false,
+  "use_rslora": false
+}

checkpoint-587/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0acb85e5a2974f7bf545565ff8be191311d76452829b574c56b96b13991d4ac
+size 78625112

checkpoint-587/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81771e4afee0643d18f8bebb85aac331a78f506ebf7936bfa3e051e339e977b6
+size 157385722

checkpoint-587/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4acbeb798b18916fc470fd0b90b29f85ec4ba7e8170d1fd46e14613afbdf3e2b
+size 14244

checkpoint-587/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1568f92601bf6cee1d56c4be80f5028e33227b17fe5eb7a730a7007d77dcf9ee
+size 1064

checkpoint-587/trainer_state.json

@@ -0,0 +1,527 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 50,
+  "global_step": 587,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.017035775127768313,
+      "grad_norm": 1.6640625,
+      "learning_rate": 5e-06,
+      "loss": 0.3494,
+      "step": 10
+    },
+    {
+      "epoch": 0.034071550255536626,
+      "grad_norm": 1.203125,
+      "learning_rate": 1e-05,
+      "loss": 0.3494,
+      "step": 20
+    },
+    {
+      "epoch": 0.05110732538330494,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.5e-05,
+      "loss": 0.3523,
+      "step": 30
+    },
+    {
+      "epoch": 0.06814310051107325,
+      "grad_norm": 1.53125,
+      "learning_rate": 2e-05,
+      "loss": 0.3428,
+      "step": 40
+    },
+    {
+      "epoch": 0.08517887563884156,
+      "grad_norm": 1.3046875,
+      "learning_rate": 2.5e-05,
+      "loss": 0.359,
+      "step": 50
+    },
+    {
+      "epoch": 0.08517887563884156,
+      "eval_loss": 0.3472045063972473,
+      "eval_runtime": 44.9607,
+      "eval_samples_per_second": 3.27,
+      "eval_steps_per_second": 3.27,
+      "step": 50
+    },
+    {
+      "epoch": 0.10221465076660988,
+      "grad_norm": 1.21875,
+      "learning_rate": 3e-05,
+      "loss": 0.3494,
+      "step": 60
+    },
+    {
+      "epoch": 0.11925042589437819,
+      "grad_norm": 1.2734375,
+      "learning_rate": 3.5e-05,
+      "loss": 0.3527,
+      "step": 70
+    },
+    {
+      "epoch": 0.1362862010221465,
+      "grad_norm": 0.64453125,
+      "learning_rate": 4e-05,
+      "loss": 0.349,
+      "step": 80
+    },
+    {
+      "epoch": 0.15332197614991483,
+      "grad_norm": 1.0703125,
+      "learning_rate": 4.5e-05,
+      "loss": 0.3432,
+      "step": 90
+    },
+    {
+      "epoch": 0.17035775127768313,
+      "grad_norm": 1.40625,
+      "learning_rate": 5e-05,
+      "loss": 0.3252,
+      "step": 100
+    },
+    {
+      "epoch": 0.17035775127768313,
+      "eval_loss": 0.35169535875320435,
+      "eval_runtime": 45.0372,
+      "eval_samples_per_second": 3.264,
+      "eval_steps_per_second": 3.264,
+      "step": 100
+    },
+    {
+      "epoch": 0.18739352640545145,
+      "grad_norm": 1.2890625,
+      "learning_rate": 4.897330595482547e-05,
+      "loss": 0.3191,
+      "step": 110
+    },
+    {
+      "epoch": 0.20442930153321975,
+      "grad_norm": 2.28125,
+      "learning_rate": 4.7946611909650925e-05,
+      "loss": 0.289,
+      "step": 120
+    },
+    {
+      "epoch": 0.22146507666098808,
+      "grad_norm": 3.765625,
+      "learning_rate": 4.691991786447639e-05,
+      "loss": 0.2395,
+      "step": 130
+    },
+    {
+      "epoch": 0.23850085178875638,
+      "grad_norm": 9.5625,
+      "learning_rate": 4.5893223819301853e-05,
+      "loss": 0.3263,
+      "step": 140
+    },
+    {
+      "epoch": 0.2555366269165247,
+      "grad_norm": 4.84375,
+      "learning_rate": 4.486652977412731e-05,
+      "loss": 0.1648,
+      "step": 150
+    },
+    {
+      "epoch": 0.2555366269165247,
+      "eval_loss": 0.611720085144043,
+      "eval_runtime": 44.8864,
+      "eval_samples_per_second": 3.275,
+      "eval_steps_per_second": 3.275,
+      "step": 150
+    },
+    {
+      "epoch": 0.272572402044293,
+      "grad_norm": 21.125,
+      "learning_rate": 4.383983572895277e-05,
+      "loss": 0.163,
+      "step": 160
+    },
+    {
+      "epoch": 0.28960817717206133,
+      "grad_norm": 7.25,
+      "learning_rate": 4.281314168377823e-05,
+      "loss": 0.4368,
+      "step": 170
+    },
+    {
+      "epoch": 0.30664395229982966,
+      "grad_norm": 29.625,
+      "learning_rate": 4.17864476386037e-05,
+      "loss": 0.2398,
+      "step": 180
+    },
+    {
+      "epoch": 0.32367972742759793,
+      "grad_norm": 15.3125,
+      "learning_rate": 4.075975359342916e-05,
+      "loss": 0.1639,
+      "step": 190
+    },
+    {
+      "epoch": 0.34071550255536626,
+      "grad_norm": 0.119140625,
+      "learning_rate": 3.973305954825462e-05,
+      "loss": 0.2869,
+      "step": 200
+    },
+    {
+      "epoch": 0.34071550255536626,
+      "eval_loss": 0.6622754335403442,
+      "eval_runtime": 44.5126,
+      "eval_samples_per_second": 3.302,
+      "eval_steps_per_second": 3.302,
+      "step": 200
+    },
+    {
+      "epoch": 0.3577512776831346,
+      "grad_norm": 0.021484375,
+      "learning_rate": 3.8706365503080084e-05,
+      "loss": 0.1419,
+      "step": 210
+    },
+    {
+      "epoch": 0.3747870528109029,
+      "grad_norm": 9.6875,
+      "learning_rate": 3.767967145790555e-05,
+      "loss": 0.1224,
+      "step": 220
+    },
+    {
+      "epoch": 0.39182282793867124,
+      "grad_norm": 0.0693359375,
+      "learning_rate": 3.6652977412731007e-05,
+      "loss": 0.3138,
+      "step": 230
+    },
+    {
+      "epoch": 0.4088586030664395,
+      "grad_norm": 6.8125,
+      "learning_rate": 3.562628336755647e-05,
+      "loss": 0.1766,
+      "step": 240
+    },
+    {
+      "epoch": 0.42589437819420783,
+      "grad_norm": 0.07421875,
+      "learning_rate": 3.459958932238193e-05,
+      "loss": 0.1243,
+      "step": 250
+    },
+    {
+      "epoch": 0.42589437819420783,
+      "eval_loss": 0.6063656210899353,
+      "eval_runtime": 43.927,
+      "eval_samples_per_second": 3.346,
+      "eval_steps_per_second": 3.346,
+      "step": 250
+    },
+    {
+      "epoch": 0.44293015332197616,
+      "grad_norm": 7.59375,
+      "learning_rate": 3.357289527720739e-05,
+      "loss": 0.1272,
+      "step": 260
+    },
+    {
+      "epoch": 0.4599659284497445,
+      "grad_norm": 28.375,
+      "learning_rate": 3.254620123203286e-05,
+      "loss": 0.4251,
+      "step": 270
+    },
+    {
+      "epoch": 0.47700170357751276,
+      "grad_norm": 0.462890625,
+      "learning_rate": 3.1519507186858315e-05,
+      "loss": 0.4472,
+      "step": 280
+    },
+    {
+      "epoch": 0.4940374787052811,
+      "grad_norm": 40.25,
+      "learning_rate": 3.049281314168378e-05,
+      "loss": 0.4209,
+      "step": 290
+    },
+    {
+      "epoch": 0.5110732538330494,
+      "grad_norm": 0.031494140625,
+      "learning_rate": 2.9466119096509244e-05,
+      "loss": 0.1857,
+      "step": 300
+    },
+    {
+      "epoch": 0.5110732538330494,
+      "eval_loss": 0.7174944281578064,
+      "eval_runtime": 43.835,
+      "eval_samples_per_second": 3.353,
+      "eval_steps_per_second": 3.353,
+      "step": 300
+    },
+    {
+      "epoch": 0.5281090289608177,
+      "grad_norm": 3.015625,
+      "learning_rate": 2.8439425051334705e-05,
+      "loss": 0.2324,
+      "step": 310
+    },
+    {
+      "epoch": 0.545144804088586,
+      "grad_norm": 0.0184326171875,
+      "learning_rate": 2.7412731006160163e-05,
+      "loss": 0.1289,
+      "step": 320
+    },
+    {
+      "epoch": 0.5621805792163543,
+      "grad_norm": 0.1201171875,
+      "learning_rate": 2.6386036960985628e-05,
+      "loss": 0.1295,
+      "step": 330
+    },
+    {
+      "epoch": 0.5792163543441227,
+      "grad_norm": 0.005859375,
+      "learning_rate": 2.5359342915811092e-05,
+      "loss": 0.393,
+      "step": 340
+    },
+    {
+      "epoch": 0.596252129471891,
+      "grad_norm": 23.375,
+      "learning_rate": 2.433264887063655e-05,
+      "loss": 0.3171,
+      "step": 350
+    },
+    {
+      "epoch": 0.596252129471891,
+      "eval_loss": 0.7911351919174194,
+      "eval_runtime": 43.7143,
+      "eval_samples_per_second": 3.363,
+      "eval_steps_per_second": 3.363,
+      "step": 350
+    },
+    {
+      "epoch": 0.6132879045996593,
+      "grad_norm": 0.0242919921875,
+      "learning_rate": 2.3305954825462014e-05,
+      "loss": 0.535,
+      "step": 360
+    },
+    {
+      "epoch": 0.6303236797274276,
+      "grad_norm": 0.50390625,
+      "learning_rate": 2.2279260780287475e-05,
+      "loss": 0.2197,
+      "step": 370
+    },
+    {
+      "epoch": 0.6473594548551959,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.125256673511294e-05,
+      "loss": 0.3187,
+      "step": 380
+    },
+    {
+      "epoch": 0.6643952299829642,
+      "grad_norm": 7.8125,
+      "learning_rate": 2.02258726899384e-05,
+      "loss": 0.1459,
+      "step": 390
+    },
+    {
+      "epoch": 0.6814310051107325,
+      "grad_norm": 39.25,
+      "learning_rate": 1.919917864476386e-05,
+      "loss": 0.5212,
+      "step": 400
+    },
+    {
+      "epoch": 0.6814310051107325,
+      "eval_loss": 0.7614322304725647,
+      "eval_runtime": 43.7037,
+      "eval_samples_per_second": 3.364,
+      "eval_steps_per_second": 3.364,
+      "step": 400
+    },
+    {
+      "epoch": 0.6984667802385008,
+      "grad_norm": 3.640625,
+      "learning_rate": 1.8172484599589323e-05,
+      "loss": 0.1773,
+      "step": 410
+    },
+    {
+      "epoch": 0.7155025553662692,
+      "grad_norm": 0.005645751953125,
+      "learning_rate": 1.7145790554414784e-05,
+      "loss": 0.2599,
+      "step": 420
+    },
+    {
+      "epoch": 0.7325383304940375,
+      "grad_norm": 30.5,
+      "learning_rate": 1.611909650924025e-05,
+      "loss": 0.3468,
+      "step": 430
+    },
+    {
+      "epoch": 0.7495741056218058,
+      "grad_norm": 5.5,
+      "learning_rate": 1.5092402464065708e-05,
+      "loss": 0.2754,
+      "step": 440
+    },
+    {
+      "epoch": 0.7666098807495741,
+      "grad_norm": 32.0,
+      "learning_rate": 1.406570841889117e-05,
+      "loss": 0.3287,
+      "step": 450
+    },
+    {
+      "epoch": 0.7666098807495741,
+      "eval_loss": 0.8178677558898926,
+      "eval_runtime": 43.7442,
+      "eval_samples_per_second": 3.36,
+      "eval_steps_per_second": 3.36,
+      "step": 450
+    },
+    {
+      "epoch": 0.7836456558773425,
+      "grad_norm": 3.0,
+      "learning_rate": 1.3039014373716632e-05,
+      "loss": 0.1921,
+      "step": 460
+    },
+    {
+      "epoch": 0.8006814310051107,
+      "grad_norm": 28.125,
+      "learning_rate": 1.2012320328542096e-05,
+      "loss": 0.4395,
+      "step": 470
+    },
+    {
+      "epoch": 0.817717206132879,
+      "grad_norm": 4.25,
+      "learning_rate": 1.0985626283367557e-05,
+      "loss": 0.2133,
+      "step": 480
+    },
+    {
+      "epoch": 0.8347529812606473,
+      "grad_norm": 7.25,
+      "learning_rate": 9.95893223819302e-06,
+      "loss": 0.4362,
+      "step": 490
+    },
+    {
+      "epoch": 0.8517887563884157,
+      "grad_norm": 5.78125,
+      "learning_rate": 8.932238193018481e-06,
+      "loss": 0.418,
+      "step": 500
+    },
+    {
+      "epoch": 0.8517887563884157,
+      "eval_loss": 0.8019587993621826,
+      "eval_runtime": 43.8445,
+      "eval_samples_per_second": 3.353,
+      "eval_steps_per_second": 3.353,
+      "step": 500
+    },
+    {
+      "epoch": 0.868824531516184,
+      "grad_norm": 5.84375,
+      "learning_rate": 7.905544147843944e-06,
+      "loss": 0.236,
+      "step": 510
+    },
+    {
+      "epoch": 0.8858603066439523,
+      "grad_norm": 9.0625,
+      "learning_rate": 6.878850102669406e-06,
+      "loss": 0.5388,
+      "step": 520
+    },
+    {
+      "epoch": 0.9028960817717206,
+      "grad_norm": 0.00543212890625,
+      "learning_rate": 5.852156057494867e-06,
+      "loss": 0.0928,
+      "step": 530
+    },
+    {
+      "epoch": 0.919931856899489,
+      "grad_norm": 10.9375,
+      "learning_rate": 4.825462012320329e-06,
+      "loss": 0.1343,
+      "step": 540
+    },
+    {
+      "epoch": 0.9369676320272572,
+      "grad_norm": 6.59375,
+      "learning_rate": 3.7987679671457908e-06,
+      "loss": 0.099,
+      "step": 550
+    },
+    {
+      "epoch": 0.9369676320272572,
+      "eval_loss": 0.8314433097839355,
+      "eval_runtime": 43.8265,
+      "eval_samples_per_second": 3.354,
+      "eval_steps_per_second": 3.354,
+      "step": 550
+    },
+    {
+      "epoch": 0.9540034071550255,
+      "grad_norm": 32.75,
+      "learning_rate": 2.7720739219712527e-06,
+      "loss": 0.1544,
+      "step": 560
+    },
+    {
+      "epoch": 0.9710391822827938,
+      "grad_norm": 22.75,
+      "learning_rate": 1.7453798767967144e-06,
+      "loss": 0.2434,
+      "step": 570
+    },
+    {
+      "epoch": 0.9880749574105622,
+      "grad_norm": 0.33203125,
+      "learning_rate": 7.186858316221766e-07,
+      "loss": 0.1046,
+      "step": 580
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 587,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 8836792572744000.0,
+  "train_batch_size": 1,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-587/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68e821eaad6d4dd2b6f1b80fc29291f65db04129307960603f655433b95a3b28
+size 5176

git_hash.txt

@@ -0,0 +1 @@
+347ab05e5a44584027e84e8fbb8b98fe92cbf103

preprocessor_config.json

@@ -0,0 +1,25 @@
+{
+  "do_convert_rgb": null,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "SiglipImageProcessor",
+  "image_seq_length": 1024,
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "processor_class": "ColPaliProcessor",
+  "resample": 3,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "height": 448,
+    "width": 448
+  }
+}

special_tokens_map.json

@@ -0,0 +1,39 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<bos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<eos>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff84f53c290d0348c4e206da6094ef781cf8c0e482fec8b268a996b32257cfd
+size 34600975

training_config.yml

@@ -0,0 +1,40 @@
+config:
+  (): colpali_engine.trainer.colmodel_training.ColModelTrainingConfig
+  output_dir: !path ../../../models/train_colpali_docmatix_hardneg_ib_3b-mix-448
+  processor:
+    () : colpali_engine.utils.transformers_wrappers.AutoProcessorWrapper
+    pretrained_model_name_or_path:  "vidore/colpaligemma-3b-mix-448-base" # "./models/paligemma-3b-mix-448"
+    max_length: 50
+  model:
+    (): colpali_engine.utils.transformers_wrappers.AllPurposeWrapper
+    class_to_instanciate: !ext colpali_engine.models.ColPali
+    pretrained_model_name_or_path: "vidore/colpaligemma-3b-mix-448-base"
+    torch_dtype:  !ext torch.bfloat16
+#    device_map: "auto"
+#    quantization_config:
+#      (): transformers.BitsAndBytesConfig
+#      load_in_4bit: true
+#      bnb_4bit_quant_type: "nf4"
+#      bnb_4bit_compute_dtype:  "bfloat16"
+#      bnb_4bit_use_double_quant: true
+
+  dataset_loading_func: !ext colpali_engine.utils.dataset_transformation.load_tai_hard_negs # load_tai_hard_negs
+  eval_dataset_loader: !import ../data/test_data_tai.yaml
+
+  max_length: 50
+  run_eval: true
+  add_suffix: true
+  loss_func:
+    (): colpali_engine.loss.late_interaction_losses.ColbertPairwiseNegativeCELoss
+    in_batch_term: true
+  tr_args: !import ../tr_args/default_neg_tr_args.yaml
+  peft_config:
+    (): peft.LoraConfig
+    r: 32
+    lora_alpha: 32
+    lora_dropout: 0.1
+    init_lora_weights: "gaussian"
+    bias: "none"
+    task_type: "FEATURE_EXTRACTION"
+    target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'
+    # target_modules: '(.*(language_model).*(down_proj|gate_proj|up_proj|k_proj|q_proj|v_proj|o_proj).*$|.*(custom_text_proj).*$)'