Repository: mlflow/mlflow

Issue:
[BUG] INVALID_PARAMETER_VALUE for parameter 'params'
### Willingness to contribute

Yes. I would be willing to contribute a fix for this bug with guidance from the MLflow community.

### MLflow version

1.27.0

### System information

- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Linux Debian 10
- **Python version**: 3.9.13
- **yarn version, if running the dev UI**: N/A


### Describe the problem

I receive a `RestException: INVAILD_PARAMETER_VALUE` error when using the mlflow tracking server with an sqlite backend, but not when using the default file-storage tracker.

### Tracking information

MLflow version: 1.27.0
Tracking URI: http://localhost:5000
Artifact URI: ./mlruns/1/db6e636a9fb644cc9102a5621d2a7425/artifacts

### Code to reproduce issue

Initiate mlflow tracking sever from command line with:
```
mlflow server \
    --backend-store-uri sqlite:///mlflow.db \
    --default-artifact-root ./mlruns \
    --host 0.0.0.0
```
Run the transformers trainer with:
```python
import os
from transformers import (
    AutoTokenizer,
    AutoModelForTokenClassification,
    TrainingArguments,
    Trainer,
    DataCollatorForTokenClassification
)
os.environ['MLFLOW_TRACKING_URI'] = 'http://localhost:5000'
os.environ['MLFLOW_EXPERIMENT_NAME'] = 'NER'

model_name = "distilbert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForTokenClassification.from_pretrained(model_name, num_labels=9)
# save the model label config so it plays nicely with downstream tasks like pipeline()

data_collator = DataCollatorForTokenClassification(tokenizer, padding=True)

dataset = [{'text': 'A dog and a cat'} for i in range(100)]

training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)

trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    eval_dataset=dataset,
    tokenizer=tokenizer,
    data_collator=data_collator
)

trainer.train()
```

### Other info / logs

```python-traceback
Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForTokenClassification: ['vocab_layer_norm.bias', 'vocab_transform.bias', 'vocab_transform.weight', 'vocab_layer_norm.weight', 'vocab_projector.bias', 'vocab_projector.weight']
- This IS expected if you are initializing DistilBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing DistilBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
Some weights of DistilBertForTokenClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
/venv/lib/python3.9/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning
  warnings.warn(
***** Running training *****
  Num examples = 100
  Num Epochs = 3
  Instantaneous batch size per device = 16
  Total train batch size (w. parallel, distributed & accumulation) = 16
  Gradient Accumulation steps = 1
  Total optimization steps = 21
---------------------------------------------------------------------------
RestException                             Traceback (most recent call last)
File /workspace/model/bug_report.py:40, in <module>
     21 training_args = TrainingArguments(
     22     output_dir="./results",
     23     evaluation_strategy="epoch",
   (...)
     28     weight_decay=0.01,
     29 )
     31 trainer = Trainer(
     32     model=model,
     33     args=training_args,
   (...)
     37     data_collator=data_collator
     38 )
---> 40 trainer.train()

File /venv/lib/python3.9/site-packages/transformers/trainer.py:1409, in Trainer.train(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)
   1404     self.model_wrapped = self.model
   1406 inner_training_loop = find_executable_batch_size(
   1407     self._inner_training_loop, self._train_batch_size, args.auto_find_batch_size
   1408 )
-> 1409 return inner_training_loop(
   1410     args=args,
   1411     resume_from_checkpoint=resume_from_checkpoint,
   1412     trial=trial,
   1413     ignore_keys_for_eval=ignore_keys_for_eval,
   1414 )

File /venv/lib/python3.9/site-packages/transformers/trainer.py:1580, in Trainer._inner_training_loop(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)
   1577 self._globalstep_last_logged = self.state.global_step
   1578 model.zero_grad()
-> 1580 self.control = self.callback_handler.on_train_begin(args, self.state, self.control)
   1582 # Skip the first epochs_trained epochs to get the random state of the dataloader at the right point.
   1583 if not args.ignore_data_skip:

File /venv/lib/python3.9/site-packages/transformers/trainer_callback.py:347, in CallbackHandler.on_train_begin(self, args, state, control)
    345 def on_train_begin(self, args: TrainingArguments, state: TrainerState, control: TrainerControl):
    346     control.should_training_stop = False
--> 347     return self.call_event("on_train_begin", args, state, control)

File /venv/lib/python3.9/site-packages/transformers/trainer_callback.py:388, in CallbackHandler.call_event(self, event, args, state, control, **kwargs)
    386 def call_event(self, event, args, state, control, **kwargs):
    387     for callback in self.callbacks:
--> 388         result = getattr(callback, event)(
    389             args,
    390             state,
    391             control,
    392             model=self.model,
    393             tokenizer=self.tokenizer,
    394             optimizer=self.optimizer,
    395             lr_scheduler=self.lr_scheduler,
    396             train_dataloader=self.train_dataloader,
    397             eval_dataloader=self.eval_dataloader,
    398             **kwargs,
    399         )
    400         # A Callback can skip the return of `control` if it doesn't change it.
    401         if result is not None:

File /venv/lib/python3.9/site-packages/transformers/integrations.py:856, in MLflowCallback.on_train_begin(self, args, state, control, model, **kwargs)
    854 def on_train_begin(self, args, state, control, model=None, **kwargs):
    855     if not self._initialized:
--> 856         self.setup(args, state, model)

File /venv/lib/python3.9/site-packages/transformers/integrations.py:847, in MLflowCallback.setup(self, args, state, model)
    845 combined_dict_items = list(combined_dict.items())
    846 for i in range(0, len(combined_dict_items), self._MAX_PARAMS_TAGS_PER_BATCH):
--> 847     self._ml_flow.log_params(dict(combined_dict_items[i : i + self._MAX_PARAMS_TAGS_PER_BATCH]))
    848 mlflow_tags = os.getenv("MLFLOW_TAGS", None)
    849 if mlflow_tags:

File /venv/lib/python3.9/site-packages/mlflow/tracking/fluent.py:675, in log_params(params)
    673 run_id = _get_or_start_run().info.run_id
    674 params_arr = [Param(key, str(value)) for key, value in params.items()]
--> 675 MlflowClient().log_batch(run_id=run_id, metrics=[], params=params_arr, tags=[])

File /venv/lib/python3.9/site-packages/mlflow/tracking/client.py:918, in MlflowClient.log_batch(self, run_id, metrics, params, tags)
    861 def log_batch(
    862     self,
    863     run_id: str,
   (...)
    866     tags: Sequence[RunTag] = (),
    867 ) -> None:
    868     """
    869     Log multiple metrics, params, and/or tags.
    870 
   (...)
    916         status: FINISHED
    917     """
--> 918     self._tracking_client.log_batch(run_id, metrics, params, tags)

File /venv/lib/python3.9/site-packages/mlflow/tracking/_tracking_service/client.py:315, in TrackingServiceClient.log_batch(self, run_id, metrics, params, tags)
    312     metrics_batch = metrics[:metrics_batch_size]
    313     metrics = metrics[metrics_batch_size:]
--> 315     self.store.log_batch(
    316         run_id=run_id, metrics=metrics_batch, params=params_batch, tags=tags_batch
    317     )
    319 for metrics_batch in chunk_list(metrics, chunk_size=MAX_METRICS_PER_BATCH):
    320     self.store.log_batch(run_id=run_id, metrics=metrics_batch, params=[], tags=[])

File /venv/lib/python3.9/site-packages/mlflow/store/tracking/rest_store.py:309, in RestStore.log_batch(self, run_id, metrics, params, tags)
    305 tag_protos = [tag.to_proto() for tag in tags]
    306 req_body = message_to_json(
    307     LogBatch(metrics=metric_protos, params=param_protos, tags=tag_protos, run_id=run_id)
    308 )
--> 309 self._call_endpoint(LogBatch, req_body)

File /venv/lib/python3.9/site-packages/mlflow/store/tracking/rest_store.py:56, in RestStore._call_endpoint(self, api, json_body)
     54 endpoint, method = _METHOD_TO_INFO[api]
     55 response_proto = api.Response()
---> 56 return call_endpoint(self.get_host_creds(), endpoint, method, json_body, response_proto)

File /venv/lib/python3.9/site-packages/mlflow/utils/rest_utils.py:256, in call_endpoint(host_creds, endpoint, method, json_body, response_proto)
    252 else:
    253     response = http_request(
    254         host_creds=host_creds, endpoint=endpoint, method=method, json=json_body
    255     )
--> 256 response = verify_rest_response(response, endpoint)
    257 js_dict = json.loads(response.text)
    258 parse_dict(js_dict=js_dict, message=response_proto)

File /venv/lib/python3.9/site-packages/mlflow/utils/rest_utils.py:185, in verify_rest_response(response, endpoint)
    183 if response.status_code != 200:
    184     if _can_parse_as_json_object(response.text):
--> 185         raise RestException(json.loads(response.text))
    186     else:
    187         base_msg = "API request to endpoint %s failed with error code %s != 200" % (
    188             endpoint,
    189             response.status_code,
    190         )

RestException: INVALID_PARAMETER_VALUE: Invalid value [{'key': 'save_strategy', 'value': 'steps'}, {'key': 'save_steps', 'value': '500'}, {'key': 'save_total_limit', 'value': 'None'}, {'key': 'save_on_each_node', 'value': 'False'}, {'key': 'no_cuda', 'value': 'False'}, {'key': 'seed', 'value': '42'}, {'key': 'data_seed', 'value': 'None'}, {'key': 'jit_mode_eval', 'value': 'False'}, {'key': 'use_ipex', 'value': 'False'}, {'key': 'bf16', 'value': 'False'}, {'key': 'fp16', 'value': 'False'}, {'key': 'fp16_opt_level', 'value': 'O1'}, {'key': 'half_precision_backend', 'value': 'auto'}, {'key': 'bf16_full_eval', 'value': 'False'}, {'key': 'fp16_full_eval', 'value': 'False'}, {'key': 'tf32', 'value': 'None'}, {'key': 'local_rank', 'value': '-1'}, {'key': 'xpu_backend', 'value': 'None'}, {'key': 'tpu_num_cores', 'value': 'None'}, {'key': 'tpu_metrics_debug', 'value': 'False'}, {'key': 'debug', 'value': '[]'}, {'key': 'dataloader_drop_last', 'value': 'False'}, {'key': 'eval_steps', 'value': 'None'}, {'key': 'dataloader_num_workers', 'value': '0'}, {'key': 'past_index', 'value': '-1'}, {'key': 'run_name', 'value': './results'}, {'key': 'disable_tqdm', 'value': 'False'}, {'key': 'remove_unused_columns', 'value': 'True'}, {'key': 'label_names', 'value': 'None'}, {'key': 'load_best_model_at_end', 'value': 'False'}, {'key': 'metric_for_best_model', 'value': 'None'}, {'key': 'greater_is_better', 'value': 'None'}, {'key': 'ignore_data_skip', 'value': 'False'}, {'key': 'sharded_ddp', 'value': '[]'}, {'key': 'fsdp', 'value': '[]'}, {'key': 'fsdp_min_num_params', 'value': '0'}, {'key': 'deepspeed', 'value': 'None'}, {'key': 'label_smoothing_factor', 'value': '0.0'}, {'key': 'optim', 'value': 'adamw_hf'}, {'key': 'adafactor', 'value': 'False'}, {'key': 'group_by_length', 'value': 'False'}, {'key': 'length_column_name', 'value': 'length'}, {'key': 'report_to', 'value': "['mlflow']"}, {'key': 'ddp_find_unused_parameters', 'value': 'None'}, {'key': 'ddp_bucket_cap_mb', 'value': 'None'}, {'key': 'dataloader_pin_memory', 'value': 'True'}, {'key': 'skip_memory_metrics', 'value': 'True'}, {'key': 'use_legacy_prediction_loop', 'value': 'False'}, {'key': 'push_to_hub', 'value': 'False'}, {'key': 'resume_from_checkpoint', 'value': 'None'}, {'key': 'hub_model_id', 'value': 'None'}, {'key': 'hub_strategy', 'value': 'every_save'}, {'key': 'hub_token', 'value': '<HUB_TOKEN>'}, {'key': 'hub_private_repo', 'value': 'False'}, {'key': 'gradient_checkpointing', 'value': 'False'}, {'key': 'include_inputs_for_metrics', 'value': 'False'}, {'key': 'fp16_backend', 'value': 'auto'}, {'key': 'push_to_hub_model_id', 'value': 'None'}, {'key': 'push_to_hub_organization', 'value': 'None'}, {'key': 'push_to_hub_token', 'value': '<PUSH_TO_HUB_TOKEN>'}, {'key': '_n_gpu', 'value': '0'}, {'key': 'mp_parameters', 'value': ''}, {'key': 'auto_find_batch_size', 'value': 'False'}, {'key': 'full_determinism', 'value': 'False'}, {'key': 'torchdynamo', 'value': 'None'}, {'key': 'ray_scope', 'value': 'last'}] for parameter 'params' supplied. Hint: Value was of type 'list'. See the API docs for more information about request parameters.
```

### What component(s) does this bug affect?

- [ ] `area/artifacts`: Artifact stores and artifact logging
- [ ] `area/build`: Build and test infrastructure for MLflow
- [ ] `area/docs`: MLflow documentation pages
- [ ] `area/examples`: Example code
- [ ] `area/model-registry`: Model Registry service, APIs, and the fluent client calls for Model Registry
- [ ] `area/models`: MLmodel format, model serialization/deserialization, flavors
- [ ] `area/pipelines`: Pipelines, Pipeline APIs, Pipeline configs, Pipeline Templates
- [ ] `area/projects`: MLproject format, project running backends
- [ ] `area/scoring`: MLflow Model server, model deployment tools, Spark UDFs
- [X] `area/server-infra`: MLflow Tracking server backend
- [X] `area/tracking`: Tracking Service, tracking client APIs, autologging

### What interface(s) does this bug affect?

- [ ] `area/uiux`: Front-end, user experience, plotting, JavaScript, JavaScript dev server
- [ ] `area/docker`: Docker use across MLflow's components, such as MLflow Projects and MLflow Models
- [X] `area/sqlalchemy`: Use of SQLAlchemy in the Tracking Service or Model Registry
- [ ] `area/windows`: Windows support

### What language(s) does this bug affect?

- [ ] `language/r`: R APIs and clients
- [ ] `language/java`: Java APIs and clients
- [ ] `language/new`: Proposals for new client languages

### What integration(s) does this bug affect?

- [ ] `integrations/azure`: Azure and Azure ML integrations
- [ ] `integrations/sagemaker`: SageMaker integrations
- [ ] `integrations/databricks`: Databricks integrations


Discussion:

Resolving patch:
diff --git a/mlflow/server/handlers.py b/mlflow/server/handlers.py
--- a/mlflow/server/handlers.py
+++ b/mlflow/server/handlers.py
@@ -763,7 +763,7 @@ def _set_experiment_tag():
         schema={
             "experiment_id": [_assert_required, _assert_string],
             "key": [_assert_required, _assert_string],
-            "value": [_assert_required, _assert_string],
+            "value": [_assert_string],
         },
     )
     tag = ExperimentTag(request_message.key, request_message.value)
@@ -782,7 +782,7 @@ def _set_tag():
         schema={
             "run_id": [_assert_required, _assert_string],
             "key": [_assert_required, _assert_string],
-            "value": [_assert_required, _assert_string],
+            "value": [_assert_string],
         },
     )
     tag = RunTag(request_message.key, request_message.value)
@@ -981,10 +981,9 @@ def _assert_metrics_fields_present(metrics):
             _assert_required(m["timestamp"])
             _assert_required(m["step"])
 
-    def _assert_params_tags_fields_present(params):
-        for p in params:
-            _assert_required(p["key"])
-            _assert_required(p["value"])
+    def _assert_params_tags_fields_present(params_or_tags):
+        for param_or_tag in params_or_tags:
+            _assert_required(param_or_tag["key"])
 
     _validate_batch_log_api_req(_get_request_json())
     request_message = _get_request_message(
@@ -1204,7 +1203,7 @@ def _set_registered_model_tag():
         schema={
             "name": [_assert_string, _assert_required],
             "key": [_assert_string, _assert_required],
-            "value": [_assert_string, _assert_required],
+            "value": [_assert_string],
         },
     )
     tag = RegisteredModelTag(key=request_message.key, value=request_message.value)
@@ -1374,7 +1373,7 @@ def _set_model_version_tag():
             "name": [_assert_string, _assert_required],
             "version": [_assert_string, _assert_required],
             "key": [_assert_string, _assert_required],
-            "value": [_assert_string, _assert_required],
+            "value": [_assert_string],
         },
     )
     tag = ModelVersionTag(key=request_message.key, value=request_message.value)


Test patch:
diff --git a/tests/tracking/test_model_registry.py b/tests/tracking/test_model_registry.py
--- a/tests/tracking/test_model_registry.py
+++ b/tests/tracking/test_model_registry.py
@@ -344,6 +344,13 @@ def test_set_delete_registered_model_tag_flow(mlflow_client, backend_store_uri):
     assert registered_model_detailed.tags == {"numeric value": "12345"}
 
 
+def test_set_registered_model_tag_with_empty_string_as_value(mlflow_client):
+    name = "SetRMTagEmptyValueTest"
+    mlflow_client.create_registered_model(name)
+    mlflow_client.set_registered_model_tag(name, "tag_key", "")
+    assert {"tag_key": ""}.items() <= mlflow_client.get_registered_model(name).tags.items()
+
+
 def test_create_and_query_model_version_flow(mlflow_client, backend_store_uri):
     name = "CreateMVTest"
     tags = {"key": "value", "another key": "some other value", "numeric value": 12345}
@@ -583,3 +590,11 @@ def test_set_delete_model_version_tag_flow(mlflow_client, backend_store_uri):
     mlflow_client.delete_model_version_tag(name, "1", "key")
     model_version_detailed = mlflow_client.get_model_version(name, "1")
     assert model_version_detailed.tags == {"numeric value": "12345"}
+
+
+def test_set_model_version_tag_with_empty_string_as_value(mlflow_client):
+    name = "SetMVTagEmptyValueTest"
+    mlflow_client.create_registered_model(name)
+    mlflow_client.create_model_version(name, "path/to/model", "run_id_1")
+    mlflow_client.set_model_version_tag(name, "1", "tag_key", "")
+    assert {"tag_key": ""}.items() <= mlflow_client.get_model_version(name, "1").tags.items()
diff --git a/tests/tracking/test_rest_tracking.py b/tests/tracking/test_rest_tracking.py
--- a/tests/tracking/test_rest_tracking.py
+++ b/tests/tracking/test_rest_tracking.py
@@ -461,16 +461,37 @@ def test_log_param_with_empty_string_as_value(mlflow_client, tracking_server_uri
     created_run = mlflow_client.create_run(experiment_id)
     run_id = created_run.info.run_id
 
-    response = _send_rest_tracking_post_request(
-        tracking_server_uri,
-        "/api/2.0/mlflow/runs/log-parameter",
-        {
-            "run_id": run_id,
-            "key": "param",
-            "value": "",
-        },
+    mlflow_client.log_param(run_id, "param_key", "")
+    assert {"param_key": ""}.items() <= mlflow_client.get_run(run_id).data.params.items()
+
+
+def test_set_tag_with_empty_string_as_value(mlflow_client, tracking_server_uri):
+    experiment_id = mlflow_client.create_experiment(
+        test_set_tag_with_empty_string_as_value.__name__
     )
-    assert response.status_code == 200
+    created_run = mlflow_client.create_run(experiment_id)
+    run_id = created_run.info.run_id
+
+    mlflow_client.set_tag(run_id, "tag_key", "")
+    assert {"tag_key": ""}.items() <= mlflow_client.get_run(run_id).data.tags.items()
+
+
+def test_log_batch_containing_params_and_tags_with_empty_string_values(
+    mlflow_client, tracking_server_uri
+):
+    experiment_id = mlflow_client.create_experiment(
+        test_log_batch_containing_params_and_tags_with_empty_string_values.__name__
+    )
+    created_run = mlflow_client.create_run(experiment_id)
+    run_id = created_run.info.run_id
+
+    mlflow_client.log_batch(
+        run_id=run_id,
+        params=[Param("param_key", "")],
+        tags=[RunTag("tag_key", "")],
+    )
+    assert {"param_key": ""}.items() <= mlflow_client.get_run(run_id).data.params.items()
+    assert {"tag_key": ""}.items() <= mlflow_client.get_run(run_id).data.tags.items()
 
 
 def test_set_tag_validation(mlflow_client, tracking_server_uri):
@@ -552,6 +573,14 @@ def test_set_experiment_tag(mlflow_client, backend_store_uri):
     )
 
 
+def test_set_experiment_tag_with_empty_string_as_value(mlflow_client, tracking_server_uri):
+    experiment_id = mlflow_client.create_experiment(
+        test_set_experiment_tag_with_empty_string_as_value.__name__
+    )
+    mlflow_client.set_experiment_tag(experiment_id, "tag_key", "")
+    assert {"tag_key": ""}.items() <= mlflow_client.get_experiment(experiment_id).tags.items()
+
+
 def test_delete_tag(mlflow_client, backend_store_uri):
     experiment_id = mlflow_client.create_experiment("DeleteTagExperiment")
     created_run = mlflow_client.create_run(experiment_id)