Repository: ASFHyP3/hyp3-sdk

Issue:
use fewer requests when submitting multiple jobs
When submitting large jobs, multiple api request are created, would be nice for a way to aggrigate jobs into one request

Discussion:

Patch:
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 2ef243f..620eb3f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) 
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [unreleased]
+
+### Added
+- Methods to prepare jobs for submission to HyP3
+  - `HyP3.prepare_autorift_job`
+  - `HyP3.prepare_rtc_job`
+  - `HyP3.prepare_insar_job`
+    
+### Changed
+- HyP3 submit methods will always return a `Batch` containing the submitted job(s)
+- `HyP3.submit_job_dict` has been renamed to `HyP3.submit_prepared_jobs` and can
+  submit one or more prepared job dictionaries.
+
 ## [0.4.0](https://github.com/ASFHyP3/hyp3-sdk/compare/v0.3.3...v0.4.0)
 
 ### Added
diff --git a/conda-env.yml b/conda-env.yml
index 19264fd..60e7b10 100644
--- a/conda-env.yml
+++ b/conda-env.yml
@@ -5,6 +5,10 @@ channels:
 dependencies:
   - pip
   # For packaging, and testing
+  - flake8
+  - flake8-import-order
+  - flake8-blind-except
+  - flake8-builtins
   - setuptools
   - setuptools_scm
   - wheel
@@ -18,4 +22,3 @@ dependencies:
   - pip:
     # For packaging and testing
     - s3pypi
-    - safety
diff --git a/hyp3_sdk/exceptions.py b/hyp3_sdk/exceptions.py
index eeb5f3f..3c1fac5 100644
--- a/hyp3_sdk/exceptions.py
+++ b/hyp3_sdk/exceptions.py
@@ -5,9 +5,5 @@ class HyP3Error(Exception):
     """Base Exception for Hyp3_sdk"""
 
 
-class ValidationError(HyP3Error):
-    """Raise when jobs do not pass validation"""
-
-
 class AuthenticationError(HyP3Error):
     """Raise when authentication does not succeed"""
diff --git a/hyp3_sdk/hyp3.py b/hyp3_sdk/hyp3.py
index 8f85510..3acd82e 100644
--- a/hyp3_sdk/hyp3.py
+++ b/hyp3_sdk/hyp3.py
@@ -2,13 +2,13 @@ import time
 import warnings
 from datetime import datetime, timedelta
 from functools import singledispatchmethod
-from typing import Optional, Union
+from typing import List, Optional, Union
 from urllib.parse import urljoin
 
 from requests.exceptions import HTTPError, RequestException
 
 import hyp3_sdk
-from hyp3_sdk.exceptions import HyP3Error, ValidationError
+from hyp3_sdk.exceptions import HyP3Error
 from hyp3_sdk.jobs import Batch, Job
 from hyp3_sdk.util import get_authenticated_session
 
@@ -105,7 +105,7 @@ class HyP3:
             job_or_batch: A Batch of Job object to refresh
 
         Returns:
-            obj: A Batch or Job object with refreshed information
+            A Batch or Job object with refreshed information
         """
         raise NotImplementedError(f'Cannot refresh {type(job_or_batch)} type object')
 
@@ -120,71 +120,134 @@ class HyP3:
     def _refresh_job(self, job: Job):
         return self._get_job_by_id(job.job_id)
 
-    def submit_job_dict(self, job_dict: dict, name: Optional[str] = None, validate_only: bool = False) -> Job:
-        if name is not None:
-            if len(name) > 20:
-                raise ValidationError('Job name too long; must be less than 20 characters')
-            job_dict['name'] = name
+    def submit_prepared_jobs(self, prepared_jobs: Union[dict, List[dict]]) -> Batch:
+        """Submit a prepared job dictionary, or list of prepared job dictionaries
+
+        Args:
+            prepared_jobs: A prepared job dictionary, or list of prepared job dictionaries
+
+        Returns:
+            A Batch object containing the submitted job(s)
+        """
+        if isinstance(prepared_jobs, dict):
+            payload = {'jobs': [prepared_jobs]}
+        else:
+            payload = {'jobs': prepared_jobs}
 
-        payload = {'jobs': [job_dict], 'validate_only': validate_only}
         response = self.session.post(urljoin(self.url, '/jobs'), json=payload)
         try:
             response.raise_for_status()
         except HTTPError:
             raise HyP3Error('Error while submitting job to HyP3')
-        return Job.from_dict(response.json()['jobs'][0])
 
-    def submit_autorift_job(self, granule1: str, granule2: str, name: Optional[str] = None) -> Job:
+        batch = Batch()
+        for job in response.json()['jobs']:
+            batch += Job.from_dict(job)
+        return batch
+
+    def submit_autorift_job(self, granule1: str, granule2: str, name: Optional[str] = None) -> Batch:
         """Submit an autoRIFT job
 
         Args:
             granule1: The first granule (scene) to use
             granule2: The second granule (scene) to use
-            name: A name for the job (must be <= 20 characters)
+            name: A name for the job
 
         Returns:
             A Batch object containing the autoRIFT job
         """
+        job_dict = self.prepare_autorift_job(granule1, granule2, name=name)
+        return self.submit_prepared_jobs(prepared_jobs=job_dict)
+
+    @classmethod
+    def prepare_autorift_job(cls, granule1: str, granule2: str, name: Optional[str] = None) -> dict:
+        """Submit an autoRIFT job
+
+        Args:
+            granule1: The first granule (scene) to use
+            granule2: The second granule (scene) to use
+            name: A name for the job
+
+        Returns:
+            A dictionary containing the prepared autoRIFT job
+        """
         job_dict = {
             'job_parameters': {'granules': [granule1, granule2]},
             'job_type': 'AUTORIFT',
         }
-        return self.submit_job_dict(job_dict=job_dict, name=name)
+        if name is not None:
+            job_dict['name'] = name
+        return job_dict
 
-    def submit_rtc_job(self, granule: str, name: Optional[str] = None, **kwargs) -> Job:
+    def submit_rtc_job(self, granule: str, name: Optional[str] = None, **kwargs) -> Batch:
         """Submit an RTC job
 
         Args:
             granule: The granule (scene) to use
-            name: A name for the job (must be <= 20 characters)
+            name: A name for the job
             **kwargs: Extra job parameters specifying custom processing options
 
         Returns:
             A Batch object containing the RTC job
         """
+        job_dict = self.prepare_rtc_job(granule, name=name, **kwargs)
+        return self.submit_prepared_jobs(prepared_jobs=job_dict)
+
+    @classmethod
+    def prepare_rtc_job(cls, granule: str, name: Optional[str] = None, **kwargs) -> dict:
+        """Submit an RTC job
+
+        Args:
+            granule: The granule (scene) to use
+            name: A name for the job
+            **kwargs: Extra job parameters specifying custom processing options
+
+        Returns:
+            A dictionary containing the prepared RTC job
+        """
         job_dict = {
             'job_parameters': {'granules': [granule], **kwargs},
             'job_type': 'RTC_GAMMA',
         }
-        return self.submit_job_dict(job_dict=job_dict, name=name)
+        if name is not None:
+            job_dict['name'] = name
+        return job_dict
 
-    def submit_insar_job(self, granule1: str, granule2: str, name: Optional[str] = None, **kwargs) -> Job:
+    def submit_insar_job(self, granule1: str, granule2: str, name: Optional[str] = None, **kwargs) -> Batch:
         """Submit an InSAR job
 
         Args:
             granule1: The first granule (scene) to use
             granule2: The second granule (scene) to use
-            name: A name for the job (must be <= 20 characters)
+            name: A name for the job
             **kwargs: Extra job parameters specifying custom processing options
 
         Returns:
             A Batch object containing the InSAR job
         """
+        job_dict = self.prepare_insar_job(granule1, granule2, name=name, **kwargs)
+        return self.submit_prepared_jobs(prepared_jobs=job_dict)
+
+    @classmethod
+    def prepare_insar_job(cls, granule1: str, granule2: str, name: Optional[str] = None, **kwargs) -> dict:
+        """Submit an InSAR job
+
+        Args:
+            granule1: The first granule (scene) to use
+            granule2: The second granule (scene) to use
+            name: A name for the job
+            **kwargs: Extra job parameters specifying custom processing options
+
+        Returns:
+            A dictionary containing the prepared InSAR job
+        """
         job_dict = {
             'job_parameters': {'granules': [granule1, granule2], **kwargs},
             'job_type': 'INSAR_GAMMA',
         }
-        return self.submit_job_dict(job_dict=job_dict, name=name)
+        if name is not None:
+            job_dict['name'] = name
+        return job_dict
 
     def my_info(self) -> dict:
         """


Test patch:
diff --git a/tests/test_hyp3.py b/tests/test_hyp3.py
index 383ac87..743fbac 100644
--- a/tests/test_hyp3.py
+++ b/tests/test_hyp3.py
@@ -85,17 +85,88 @@ def test_refresh(get_mock_job):
 
 
 @responses.activate
-def test_submit_job_dict(get_mock_job):
-    job = get_mock_job()
+def test_submit_prepared_jobs(get_mock_job):
+    rtc_job = get_mock_job('RTC_GAMMA', job_parameters={'granules': ['g1']})
+    insar_job = get_mock_job('INSAR_GAMMA', job_parameters={'granules': ['g1', 'g2']})
     api_response = {
         'jobs': [
-            job.to_dict()
+            rtc_job.to_dict(),
+            insar_job.to_dict(),
         ]
     }
+
     api = HyP3()
     responses.add(responses.POST, urljoin(api.url, '/jobs'), json=api_response)
-    response = api.submit_job_dict(job.to_dict(for_resubmit=True))
-    assert response == job
+
+    batch = api.submit_prepared_jobs(
+        [rtc_job.to_dict(for_resubmit=True), insar_job.to_dict(for_resubmit=True)])
+    assert batch.jobs == [rtc_job, insar_job]
+
+
+def test_prepare_autorift_job():
+    assert HyP3.prepare_autorift_job(granule1='my_granule1', granule2='my_granule2') == {
+        'job_type': 'AUTORIFT',
+        'job_parameters': {
+            'granules': ['my_granule1', 'my_granule2'],
+        }
+    }
+    assert HyP3.prepare_autorift_job(granule1='my_granule1',  granule2='my_granule2', name='my_name') == {
+        'job_type': 'AUTORIFT',
+        'name': 'my_name',
+        'job_parameters': {
+            'granules': ['my_granule1', 'my_granule2'],
+        },
+    }
+
+
+def test_prepare_rtc_job():
+    assert HyP3.prepare_rtc_job(granule='my_granule') == {
+        'job_type': 'RTC_GAMMA',
+        'job_parameters': {
+            'granules': ['my_granule'],
+        }
+    }
+    assert HyP3.prepare_rtc_job(granule='my_granule', name='my_name') == {
+        'job_type': 'RTC_GAMMA',
+        'name': 'my_name',
+        'job_parameters': {
+            'granules': ['my_granule'],
+        },
+    }
+    assert HyP3.prepare_rtc_job(granule='my_granule', x='foo', y=1.0, z=True) == {
+        'job_type': 'RTC_GAMMA',
+        'job_parameters': {
+            'granules': ['my_granule'],
+            'x': 'foo',
+            'y': 1.0,
+            'z': True,
+        }
+    }
+
+
+def test_prepare_insar_job():
+    assert HyP3.prepare_insar_job(granule1='my_granule1', granule2='my_granule2') == {
+        'job_type': 'INSAR_GAMMA',
+        'job_parameters': {
+            'granules': ['my_granule1', 'my_granule2'],
+        }
+    }
+    assert HyP3.prepare_insar_job(granule1='my_granule1',  granule2='my_granule2', name='my_name') == {
+        'job_type': 'INSAR_GAMMA',
+        'name': 'my_name',
+        'job_parameters': {
+            'granules': ['my_granule1', 'my_granule2'],
+        },
+    }
+    assert HyP3.prepare_insar_job(granule1='my_granule1',  granule2='my_granule2', x='foo', y=1.0, z=True) == {
+        'job_type': 'INSAR_GAMMA',
+        'job_parameters': {
+            'granules': ['my_granule1', 'my_granule2'],
+            'x': 'foo',
+            'y': 1.0,
+            'z': True,
+        }
+    }
 
 
 @responses.activate
@@ -108,8 +179,8 @@ def test_submit_autorift_job(get_mock_job):
     }
     api = HyP3()
     responses.add(responses.POST, urljoin(api.url, '/jobs'), json=api_response)
-    response = api.submit_autorift_job('g1', 'g2')
-    assert response == job
+    batch = api.submit_autorift_job('g1', 'g2')
+    assert batch.jobs[0] == job
 
 
 @responses.activate
@@ -122,8 +193,8 @@ def test_submit_rtc_job(get_mock_job):
     }
     api = HyP3()
     responses.add(responses.POST, urljoin(api.url, '/jobs'), json=api_response)
-    response = api.submit_rtc_job('g1')
-    assert response == job
+    batch = api.submit_rtc_job('g1')
+    assert batch.jobs[0] == job
 
 
 @responses.activate
@@ -136,8 +207,22 @@ def test_submit_insar_job(get_mock_job):
     }
     api = HyP3()
     responses.add(responses.POST, urljoin(api.url, '/jobs'), json=api_response)
-    response = api.submit_insar_job('g1', 'g2')
-    assert response == job
+    batch = api.submit_insar_job('g1', 'g2')
+    assert batch.jobs[0] == job
+
+
+@responses.activate
+def test_resubmit_previous_job(get_mock_job):
+    job = get_mock_job()
+    api_response = {
+        'jobs': [
+            job.to_dict()
+        ]
+    }
+    api = HyP3()
+    responses.add(responses.POST, urljoin(api.url, '/jobs'), json=api_response)
+    batch = api.submit_prepared_jobs(job.to_dict(for_resubmit=True))
+    assert batch.jobs[0] == job
 
 
 @responses.activate