Repository: getmoto/moto
Pull request #6045 (2023-03-09 13:13:47)

[Glue] Add support to more fields in Glue job
Add support to more fields in Glue job
- `CodeGenConfigurationNodes`
- `ExecutionClass`
- `SourceControlDetails`

codecov[bot]: # [Codecov](https://codecov.io/gh/getmoto/moto/pull/6045?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) Report
> Merging [#6045](https://codecov.io/gh/getmoto/moto/pull/6045?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (7829167) into [master](https://codecov.io/gh/getmoto/moto/commit/84174e129d5a0182282a5f8719426d16cddb019b?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (84174e1) will **increase** coverage by `0.00%`.
> The diff coverage is `100.00%`.

```diff
@@           Coverage Diff           @@
##           master    #6045   +/-   ##
=======================================
  Coverage   96.23%   96.23%           
=======================================
  Files         765      765           
  Lines       75179    75185    +6     
=======================================
+ Hits        72347    72353    +6     
  Misses       2832     2832           
```

| Flag | Coverage Δ | |
|---|---|---|
| servertests | `37.03% <0.00%> (-0.01%)` | :arrow_down: |
| unittests | `96.17% <100.00%> (+<0.01%)` | :arrow_up: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Impacted Files](https://codecov.io/gh/getmoto/moto/pull/6045?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) | Coverage Δ | |
|---|---|---|
| [moto/glue/models.py](https://codecov.io/gh/getmoto/moto/pull/6045?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-bW90by9nbHVlL21vZGVscy5weQ==) | `97.31% <100.00%> (+0.01%)` | :arrow_up: |
| [moto/glue/responses.py](https://codecov.io/gh/getmoto/moto/pull/6045?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-bW90by9nbHVlL3Jlc3BvbnNlcy5weQ==) | `99.39% <100.00%> (+<0.01%)` | :arrow_up: |

Help us with your feedback. Take ten seconds to tell us [how you rate us](https://about.codecov.io/nps?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None). Have a feature suggestion? [Share it here.](https://app.codecov.io/gh/feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None)

szemek: @bblommers if you find time, could you take a look? Thanks in advance!

Patch:
diff --git a/moto/glue/models.py b/moto/glue/models.py
index a0b3566aa..6abbf4aa9 100644
--- a/moto/glue/models.py
+++ b/moto/glue/models.py
@@ -324,6 +324,9 @@ class GlueBackend(BaseBackend):
         glue_version,
         number_of_workers,
         worker_type,
+        code_gen_configuration_nodes,
+        execution_class,
+        source_control_details,
     ):
         self.jobs[name] = FakeJob(
             name,
@@ -345,6 +348,9 @@ class GlueBackend(BaseBackend):
             glue_version,
             number_of_workers,
             worker_type,
+            code_gen_configuration_nodes,
+            execution_class,
+            source_control_details,
             backend=self,
         )
         return name
@@ -1070,6 +1076,9 @@ class FakeJob:
         glue_version=None,
         number_of_workers=None,
         worker_type=None,
+        code_gen_configuration_nodes=None,
+        execution_class=None,
+        source_control_details=None,
         backend=None,
     ):
         self.name = name
@@ -1090,6 +1099,9 @@ class FakeJob:
         self.glue_version = glue_version
         self.number_of_workers = number_of_workers
         self.worker_type = worker_type
+        self.code_gen_configuration_nodes = code_gen_configuration_nodes
+        self.execution_class = execution_class or "STANDARD"
+        self.source_control_details = source_control_details
         self.created_on = datetime.utcnow()
         self.last_modified_on = datetime.utcnow()
         self.arn = (
@@ -1125,6 +1137,9 @@ class FakeJob:
             "SecurityConfiguration": self.security_configuration,
             "NotificationProperty": self.notification_property,
             "GlueVersion": self.glue_version,
+            "CodeGenConfigurationNodes": self.code_gen_configuration_nodes,
+            "ExecutionClass": self.execution_class,
+            "SourceControlDetails": self.source_control_details,
         }
 
     def start_job_run(self):
diff --git a/moto/glue/responses.py b/moto/glue/responses.py
index 7439dbbd7..934b1ed76 100644
--- a/moto/glue/responses.py
+++ b/moto/glue/responses.py
@@ -309,6 +309,9 @@ class GlueResponse(BaseResponse):
         glue_version = self._get_param("GlueVersion")
         number_of_workers = self._get_int_param("NumberOfWorkers")
         worker_type = self._get_param("WorkerType")
+        code_gen_configuration_nodes = self._get_param("CodeGenConfigurationNodes")
+        execution_class = self._get_param("ExecutionClass")
+        source_control_details = self._get_param("SourceControlDetails")
         name = self.glue_backend.create_job(
             name=name,
             description=description,
@@ -329,6 +332,9 @@ class GlueResponse(BaseResponse):
             glue_version=glue_version,
             number_of_workers=number_of_workers,
             worker_type=worker_type,
+            code_gen_configuration_nodes=code_gen_configuration_nodes,
+            execution_class=execution_class,
+            source_control_details=source_control_details,
         )
         return json.dumps(dict(Name=name))
 
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
index 5285cd975..f10bd6500 100644
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -1319,7 +1319,10 @@ class S3Response(BaseResponse):
             response_headers["x-amz-version-id"] = key.version_id
 
         if key.storage_class in ARCHIVE_STORAGE_CLASSES:
-            raise InvalidObjectState(storage_class=key.storage_class)
+            if 'ongoing-request="false"' not in key.response_dict.get(
+                "x-amz-restore", ""
+            ):
+                raise InvalidObjectState(storage_class=key.storage_class)
         if if_unmodified_since:
             if_unmodified_since = str_to_rfc_1123_datetime(if_unmodified_since)
             if key.last_modified.replace(microsecond=0) > if_unmodified_since:


Test patch:
diff --git a/tests/test_glue/test_glue.py b/tests/test_glue/test_glue.py
index 820bbc14e..03d7a54e8 100644
--- a/tests/test_glue/test_glue.py
+++ b/tests/test_glue/test_glue.py
@@ -83,30 +83,36 @@ def test_get_job_exists():
         "SecurityConfiguration": "test_config",
         "NotificationProperty": {"NotifyDelayAfter": 123},
         "GlueVersion": "string",
+        "CodeGenConfigurationNodes": {},
+        "ExecutionClass": "string",
+        "SourceControlDetails": {},
     }
     job_name = create_test_job_w_all_attributes(client, **job_attributes)
     job = client.get_job(JobName=job_name)["Job"]
-    job.should.have.key("Name").equals(job_name)
-    job.should.have.key("Description")
-    job.should.have.key("LogUri")
-    job.should.have.key("Role")
-    job.should.have.key("ExecutionProperty").equals({"MaxConcurrentRuns": 123})
-    job.should.have.key("CreatedOn")
-    job.should.have.key("LastModifiedOn")
-    job.should.have.key("ExecutionProperty")
-    job.should.have.key("Command")
-    job.should.have.key("DefaultArguments")
-    job.should.have.key("NonOverridableArguments")
-    job.should.have.key("Connections")
-    job.should.have.key("MaxRetries")
-    job.should.have.key("AllocatedCapacity")
-    job.should.have.key("Timeout")
-    job.should.have.key("MaxCapacity")
-    job.should.have.key("WorkerType")
-    job.should.have.key("NumberOfWorkers")
-    job.should.have.key("SecurityConfiguration")
-    job.should.have.key("NotificationProperty")
-    job.should.have.key("GlueVersion")
+    assert job["Name"] == job_name
+    assert "Description" in job
+    assert "LogUri" in job
+    assert "Role" in job
+    assert job["ExecutionProperty"] == {"MaxConcurrentRuns": 123}
+    assert "CreatedOn" in job
+    assert "LastModifiedOn" in job
+    assert "ExecutionProperty" in job
+    assert "Command" in job
+    assert "DefaultArguments" in job
+    assert "NonOverridableArguments" in job
+    assert "Connections" in job
+    assert "MaxRetries" in job
+    assert "AllocatedCapacity" in job
+    assert "Timeout" in job
+    assert "MaxCapacity" in job
+    assert "WorkerType" in job
+    assert "NumberOfWorkers" in job
+    assert "SecurityConfiguration" in job
+    assert "NotificationProperty" in job
+    assert "GlueVersion" in job
+    assert "CodeGenConfigurationNodes" in job
+    assert "ExecutionClass" in job
+    assert "SourceControlDetails" in job
 
 
 @mock_glue
diff --git a/tests/test_s3/test_s3_storageclass.py b/tests/test_s3/test_s3_storageclass.py
index 4c71a6b33..c0fa60cd6 100644
--- a/tests/test_s3/test_s3_storageclass.py
+++ b/tests/test_s3/test_s3_storageclass.py
@@ -231,8 +231,18 @@ def test_s3_copy_object_for_deep_archive_storage_class_restored():
         Bucket="Bucket", Key="First_Object", Body="Body", StorageClass="DEEP_ARCHIVE"
     )
 
+    with pytest.raises(ClientError) as exc:
+        s3.get_object(Bucket="Bucket", Key="First_Object")
+    err = exc.value.response["Error"]
+    err["Code"].should.equal("InvalidObjectState")
+    err["Message"].should.equal(
+        "The operation is not valid for the object's storage class"
+    )
+    err["StorageClass"].should.equal("DEEP_ARCHIVE")
+
     s3.create_bucket(Bucket="Bucket2")
     s3.restore_object(Bucket="Bucket", Key="First_Object", RestoreRequest={"Days": 123})
+    s3.get_object(Bucket="Bucket", Key="First_Object")
 
     s3.copy_object(
         CopySource={"Bucket": "Bucket", "Key": "First_Object"},