Repository: apache/airflow

Issue:
DatabricksRunNowOperator missing jar_params as a kwarg
<!--

Welcome to Apache Airflow!  For a smooth issue process, try to answer the following questions.
Don't worry if they're not all applicable; just try to include what you can :-)

If you need to include code snippets or logs, please put them in fenced code
blocks.  If they're super-long, please use the details tag like
<details><summary>super-long log</summary> lots of stuff </details>

Please delete these comment blocks before submitting the issue.

-->

**Description**

DatabricksRunNowOperator is missing the option to take in key word arguement _jar_params_ , it already can take the other ones notebook_params,python_params,spark_submit_params. https://docs.databricks.com/dev-tools/api/latest/jobs.html#run-now


**Use case / motivation**

Provide parity with the other options



Discussion:
Thanks for opening your first issue here! Be sure to follow the issue template!

@neil90 Sounds. good to me. Should I assign you to this ticket? 
Yes I have made the PR https://github.com/apache/airflow/pull/10787
I assigned you to this ticket.
 
> If you create pull-request, you don't have to create an issue first, but if you want, you can do it. Creating an issue will allow you to collect feedback or share plans with other people.

https://github.com/apache/airflow/blob/master/CONTRIBUTING.rst#contribution-workflow
@neil90 Would you like to finish the PR you started? I can help with review.
I can pickup that issue - who can assign it to me?
@alexott assigned 

Resolving patch:
diff --git a/airflow/providers/databricks/operators/databricks.py b/airflow/providers/databricks/operators/databricks.py
--- a/airflow/providers/databricks/operators/databricks.py
+++ b/airflow/providers/databricks/operators/databricks.py
@@ -142,7 +142,9 @@ class DatabricksSubmitRunOperator(BaseOperator):
         - ``spark_jar_task``
         - ``notebook_task``
         - ``spark_python_task``
+        - ``spark_jar_task``
         - ``spark_submit_task``
+        - ``pipeline_task``
         - ``new_cluster``
         - ``existing_cluster_id``
         - ``libraries``
@@ -167,7 +169,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     :param spark_jar_task: The main class and parameters for the JAR task. Note that
         the actual JAR is specified in the ``libraries``.
         *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` *OR* ``spark_python_task``
-        *OR* ``spark_submit_task`` should be specified.
+        *OR* ``spark_submit_task`` *OR* ``pipeline_task`` should be specified.
         This field will be templated.
 
         .. seealso::
@@ -175,7 +177,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     :type spark_jar_task: dict
     :param notebook_task: The notebook path and parameters for the notebook task.
         *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` *OR* ``spark_python_task``
-        *OR* ``spark_submit_task`` should be specified.
+        *OR* ``spark_submit_task`` *OR* ``pipeline_task`` should be specified.
         This field will be templated.
 
         .. seealso::
@@ -183,7 +185,7 @@ class DatabricksSubmitRunOperator(BaseOperator):
     :type notebook_task: dict
     :param spark_python_task: The python file path and parameters to run the python file with.
         *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` *OR* ``spark_python_task``
-        *OR* ``spark_submit_task`` should be specified.
+        *OR* ``spark_submit_task`` *OR* ``pipeline_task`` should be specified.
         This field will be templated.
 
         .. seealso::
@@ -191,21 +193,32 @@ class DatabricksSubmitRunOperator(BaseOperator):
     :type spark_python_task: dict
     :param spark_submit_task: Parameters needed to run a spark-submit command.
         *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` *OR* ``spark_python_task``
-        *OR* ``spark_submit_task`` should be specified.
+        *OR* ``spark_submit_task`` *OR* ``pipeline_task`` should be specified.
         This field will be templated.
 
         .. seealso::
             https://docs.databricks.com/api/latest/jobs.html#jobssparksubmittask
     :type spark_submit_task: dict
+    :param pipeline_task: Parameters needed to execute a Delta Live Tables pipeline task.
+        The provided dictionary must contain at least ``pipeline_id`` field!
+        *EITHER* ``spark_jar_task`` *OR* ``notebook_task`` *OR* ``spark_python_task``
+        *OR* ``spark_submit_task`` *OR* ``pipeline_task`` should be specified.
+        This field will be templated.
+
+        .. seealso::
+            https://docs.databricks.com/dev-tools/api/2.0/jobs.html#jobspipelinetask
+    :type pipeline_task: dict
     :param new_cluster: Specs for a new cluster on which this task will be run.
-        *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.
+        *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified
+        (except when ``pipeline_task`` is used).
         This field will be templated.
 
         .. seealso::
             https://docs.databricks.com/api/latest/jobs.html#jobsclusterspecnewcluster
     :type new_cluster: dict
     :param existing_cluster_id: ID for existing cluster on which to run this task.
-        *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified.
+        *EITHER* ``new_cluster`` *OR* ``existing_cluster_id`` should be specified
+        (except when ``pipeline_task`` is used).
         This field will be templated.
     :type existing_cluster_id: str
     :param libraries: Libraries which this run will use.
@@ -255,6 +268,7 @@ def __init__(
         notebook_task: Optional[Dict[str, str]] = None,
         spark_python_task: Optional[Dict[str, Union[str, List[str]]]] = None,
         spark_submit_task: Optional[Dict[str, List[str]]] = None,
+        pipeline_task: Optional[Dict[str, str]] = None,
         new_cluster: Optional[Dict[str, object]] = None,
         existing_cluster_id: Optional[str] = None,
         libraries: Optional[List[Dict[str, str]]] = None,
@@ -282,6 +296,8 @@ def __init__(
             self.json['spark_python_task'] = spark_python_task
         if spark_submit_task is not None:
             self.json['spark_submit_task'] = spark_submit_task
+        if pipeline_task is not None:
+            self.json['pipeline_task'] = pipeline_task
         if new_cluster is not None:
             self.json['new_cluster'] = new_cluster
         if existing_cluster_id is not None:
@@ -356,12 +372,15 @@ class DatabricksRunNowOperator(BaseOperator):
 
         python_params = ["douglas adams", "42"]
 
+        jar_params = ["douglas adams", "42"]
+
         spark_submit_params = ["--class", "org.apache.spark.examples.SparkPi"]
 
         notebook_run = DatabricksRunNowOperator(
             job_id=job_id,
             notebook_params=notebook_params,
             python_params=python_params,
+            jar_params=jar_params,
             spark_submit_params=spark_submit_params
         )
 
@@ -374,6 +393,7 @@ class DatabricksRunNowOperator(BaseOperator):
         - ``json``
         - ``notebook_params``
         - ``python_params``
+        - ``jar_params``
         - ``spark_submit_params``
 
 
@@ -420,6 +440,18 @@ class DatabricksRunNowOperator(BaseOperator):
         .. seealso::
             https://docs.databricks.com/api/latest/jobs.html#run-now
     :type python_params: list[str]
+    :param jar_params: A list of parameters for jobs with JAR tasks,
+        e.g. "jar_params": ["john doe", "35"].
+        The parameters will be passed to JAR file as command line parameters.
+        If specified upon run-now, it would overwrite the parameters specified in
+        job setting.
+        The json representation of this field (i.e. {"jar_params":["john doe","35"]})
+        cannot exceed 10,000 bytes.
+        This field will be templated.
+
+        .. seealso::
+            https://docs.databricks.com/api/latest/jobs.html#run-now
+    :type jar_params: list[str]
     :param spark_submit_params: A list of parameters for jobs with spark submit task,
         e.g. "spark_submit_params": ["--class", "org.apache.spark.examples.SparkPi"].
         The parameters will be passed to spark-submit script as command line parameters.
@@ -463,6 +495,7 @@ def __init__(
         json: Optional[Any] = None,
         notebook_params: Optional[Dict[str, str]] = None,
         python_params: Optional[List[str]] = None,
+        jar_params: Optional[List[str]] = None,
         spark_submit_params: Optional[List[str]] = None,
         databricks_conn_id: str = 'databricks_default',
         polling_period_seconds: int = 30,
@@ -485,6 +518,8 @@ def __init__(
             self.json['notebook_params'] = notebook_params
         if python_params is not None:
             self.json['python_params'] = python_params
+        if jar_params is not None:
+            self.json['jar_params'] = jar_params
         if spark_submit_params is not None:
             self.json['spark_submit_params'] = spark_submit_params
 


Test patch:
diff --git a/tests/providers/databricks/operators/test_databricks.py b/tests/providers/databricks/operators/test_databricks.py
--- a/tests/providers/databricks/operators/test_databricks.py
+++ b/tests/providers/databricks/operators/test_databricks.py
@@ -46,7 +46,7 @@
 EXISTING_CLUSTER_ID = 'existing-cluster-id'
 RUN_NAME = 'run-name'
 RUN_ID = 1
-JOB_ID = 42
+JOB_ID = "42"
 NOTEBOOK_PARAMS = {"dry-run": "true", "oldest-time-to-consider": "1457570074236"}
 JAR_PARAMS = ["param1", "param2"]
 RENDERED_TEMPLATED_JAR_PARAMS = [f'/test-{DATE}']
@@ -137,6 +137,18 @@ def test_init_with_specified_run_name(self):
         )
         assert expected == op.json
 
+    def test_pipeline_task(self):
+        """
+        Test the initializer with a specified run_name.
+        """
+        pipeline_task = {"pipeline_id": "test-dlt"}
+        json = {'new_cluster': NEW_CLUSTER, 'run_name': RUN_NAME, "pipeline_task": pipeline_task}
+        op = DatabricksSubmitRunOperator(task_id=TASK_ID, json=json)
+        expected = databricks_operator._deep_string_coerce(
+            {'new_cluster': NEW_CLUSTER, "pipeline_task": pipeline_task, 'run_name': RUN_NAME}
+        )
+        assert expected == op.json
+
     def test_init_with_merging(self):
         """
         Test the initializer when json and other named parameters are both
@@ -301,7 +313,8 @@ def test_init_with_merging(self):
         provided. The named parameters should override top level keys in the
         json dict.
         """
-        override_notebook_params = {'workers': 999}
+        override_notebook_params = {'workers': "999"}
+        override_jar_params = ['workers', "998"]
         json = {'notebook_params': NOTEBOOK_PARAMS, 'jar_params': JAR_PARAMS}
 
         op = DatabricksRunNowOperator(
@@ -310,13 +323,14 @@ def test_init_with_merging(self):
             job_id=JOB_ID,
             notebook_params=override_notebook_params,
             python_params=PYTHON_PARAMS,
+            jar_params=override_jar_params,
             spark_submit_params=SPARK_SUBMIT_PARAMS,
         )
 
         expected = databricks_operator._deep_string_coerce(
             {
                 'notebook_params': override_notebook_params,
-                'jar_params': JAR_PARAMS,
+                'jar_params': override_jar_params,
                 'python_params': PYTHON_PARAMS,
                 'spark_submit_params': SPARK_SUBMIT_PARAMS,
                 'job_id': JOB_ID,