Repository: apache/airflow

Issue:
Glue Hook not using aws_conn_id for logs access resulting in Error
### Apache Airflow Provider(s)

amazon

### Versions of Apache Airflow Providers

8.16.0

### Apache Airflow version

2.8.1

### Operating System

Apache Airflow Official Docker Image

### Deployment

Official Apache Airflow Helm Chart

### Deployment details

Deployed on AWS EKS

### What happened

When using the Glue Operator and enabling the verbose parameter, we get the following error.

It's seems that the the GlueHook is not using a LogHook, but rather a newly instancied boto3 client here :
https://github.com/apache/airflow/blob/19ebcac2395ef9a6b6ded3a2faa29dc960c1e635/airflow/providers/amazon/aws/hooks/glue.py#L228

**So this new instance of boto3 client is using the pod ServiceAccount IAM credentials and not the provided Hook (via aws_conn_id) which are differents.**

So when we are using the Operator with the verbose parameter set to True, we get this stack trace: 

```
  File "/home/airflow/.local/lib/python3.10/site-packages/airflow/providers/amazon/aws/operators/glue.py", line 207, in execute
    glue_job_run = self.glue_job_hook.job_completion(self.job_name, self._job_run_id, self.verbose)
  File "/home/airflow/.local/lib/python3.10/site-packages/airflow/providers/amazon/aws/hooks/glue.py", line 288, in job_completion
    ret = self._handle_state(job_run_state, job_name, run_id, verbose, next_log_tokens)
  File "/home/airflow/.local/lib/python3.10/site-packages/airflow/providers/amazon/aws/hooks/glue.py", line 325, in _handle_state
    self.print_job_logs(
  File "/home/airflow/.local/lib/python3.10/site-packages/airflow/providers/amazon/aws/hooks/glue.py", line 269, in print_job_logs
    continuation_tokens.output_stream_continuation = display_logs_from(
  File "/home/airflow/.local/lib/python3.10/site-packages/airflow/providers/amazon/aws/hooks/glue.py", line 236, in display_logs_from
    for response in paginator.paginate(
  File "/home/airflow/.local/lib/python3.10/site-packages/botocore/paginate.py", line 269, in __iter__
    response = self._make_request(current_kwargs)
  File "/home/airflow/.local/lib/python3.10/site-packages/botocore/paginate.py", line 357, in _make_request
    return self._method(**current_kwargs)
  File "/home/airflow/.local/lib/python3.10/site-packages/botocore/client.py", line 553, in _api_call
    return self._make_api_call(operation_name, kwargs)
  File "/home/airflow/.local/lib/python3.10/site-packages/botocore/client.py", line 1009, in _make_api_call
    raise error_class(parsed_response, operation_name)
botocore.errorfactory.AccessDeniedException: An error occurred (AccessDeniedException) when calling the FilterLogEvents operation: User: arn:aws:sts::***redacted*** is not authorized to perform: logs:FilterLogEvents on resource: arn:aws:logs:***redacted***: because no identity-based policy allows the logs:FilterLogEvents action
```

### What you think should happen instead

The GlueHook should create a LogHook like in the SageMaker hook 
https://github.com/apache/airflow/blob/19ebcac2395ef9a6b6ded3a2faa29dc960c1e635/airflow/providers/amazon/aws/hooks/sagemaker.py#L156 that will use the Hook provided when instanciating the Operator via the aws_conn_id

### How to reproduce

1. Create an Airflow AWS Connection with a role ARN different than the one used by the Workers.
2. Execute a GlueJobOperator task with verbose parameter set to True 
```
    GlueJobOperator(
            aws_conn_id=connection_id,
            task_id=task_id,
            job_name=job_name,
            verbose=True,
            *args,
            **kwargs,
        )
```

### Anything else

_No response_

### Are you willing to submit PR?

- [x] Yes I am willing to submit a PR!

### Code of Conduct

- [X] I agree to follow this project's [Code of Conduct](https://github.com/apache/airflow/blob/main/CODE_OF_CONDUCT.md)



Discussion:
Thanks for opening your first issue here! Be sure to follow the issue template! If you are willing to raise PR to address this issue please do so, no need to wait for approval.


Resolving patch:
diff --git a/airflow/providers/amazon/aws/hooks/glue.py b/airflow/providers/amazon/aws/hooks/glue.py
--- a/airflow/providers/amazon/aws/hooks/glue.py
+++ b/airflow/providers/amazon/aws/hooks/glue.py
@@ -19,12 +19,13 @@
 
 import asyncio
 import time
+from functools import cached_property
 
-import boto3
 from botocore.exceptions import ClientError
 
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
+from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook
 
 DEFAULT_LOG_SUFFIX = "output"
 ERROR_LOG_SUFFIX = "error"
@@ -213,6 +214,13 @@ async def async_get_job_state(self, job_name: str, run_id: str) -> str:
             job_run = await client.get_job_run(JobName=job_name, RunId=run_id)
         return job_run["JobRun"]["JobRunState"]
 
+    @cached_property
+    def logs_hook(self):
+        """Returns an AwsLogsHook instantiated with the parameters of the GlueJobHook."""
+        return AwsLogsHook(
+            aws_conn_id=self.aws_conn_id, region_name=self.region_name, verify=self.verify, config=self.config
+        )
+
     def print_job_logs(
         self,
         job_name: str,
@@ -225,7 +233,7 @@ def print_job_logs(
         :param continuation_tokens: the tokens where to resume from when reading logs.
             The object gets updated with the new tokens by this method.
         """
-        log_client = boto3.client("logs")
+        log_client = self.logs_hook.get_conn()
         paginator = log_client.get_paginator("filter_log_events")
 
         def display_logs_from(log_group: str, continuation_token: str | None) -> str | None:
@@ -264,7 +272,6 @@ def display_logs_from(log_group: str, continuation_token: str | None) -> str | N
         log_group_prefix = self.conn.get_job_run(JobName=job_name, RunId=run_id)["JobRun"]["LogGroupName"]
         log_group_default = f"{log_group_prefix}/{DEFAULT_LOG_SUFFIX}"
         log_group_error = f"{log_group_prefix}/{ERROR_LOG_SUFFIX}"
-
         # one would think that the error log group would contain only errors, but it actually contains
         # a lot of interesting logs too, so it's valuable to have both
         continuation_tokens.output_stream_continuation = display_logs_from(


Test patch:
diff --git a/tests/providers/amazon/aws/hooks/test_glue.py b/tests/providers/amazon/aws/hooks/test_glue.py
--- a/tests/providers/amazon/aws/hooks/test_glue.py
+++ b/tests/providers/amazon/aws/hooks/test_glue.py
@@ -29,6 +29,7 @@
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.hooks.base_aws import AwsBaseHook
 from airflow.providers.amazon.aws.hooks.glue import GlueJobHook
+from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook
 
 if TYPE_CHECKING:
     from unittest.mock import MagicMock
@@ -376,12 +377,12 @@ def test_initialize_job(self, mock_conn, mock_get_job_state):
         glue_job_run_state = glue_job_hook.get_job_state(glue_job_run["JobName"], glue_job_run["JobRunId"])
         assert glue_job_run_state == mock_job_run_state, "Mocks but be equal"
 
-    @mock.patch("airflow.providers.amazon.aws.hooks.glue.boto3.client")
+    @mock.patch.object(AwsLogsHook, "get_conn")
     @mock.patch.object(GlueJobHook, "conn")
-    def test_print_job_logs_returns_token(self, conn_mock: MagicMock, client_mock: MagicMock, caplog):
-        hook = GlueJobHook()
+    def test_print_job_logs_returns_token(self, conn_mock: MagicMock, log_client_mock: MagicMock, caplog):
+        hook = GlueJobHook(job_name="test")
         conn_mock().get_job_run.return_value = {"JobRun": {"LogGroupName": "my_log_group"}}
-        client_mock().get_paginator().paginate.return_value = [
+        log_client_mock().get_paginator().paginate.return_value = [
             # first response : 2 log lines
             {
                 "events": [
@@ -399,12 +400,11 @@ def test_print_job_logs_returns_token(self, conn_mock: MagicMock, client_mock: M
         tokens = GlueJobHook.LogContinuationTokens()
         with caplog.at_level("INFO"):
             hook.print_job_logs("name", "run", tokens)
-
         assert "\thello\n\tworld\n" in caplog.text
         assert tokens.output_stream_continuation == "my_continuation_token"
         assert tokens.error_stream_continuation == "my_continuation_token"
 
-    @mock.patch("airflow.providers.amazon.aws.hooks.glue.boto3.client")
+    @mock.patch.object(AwsLogsHook, "get_conn")
     @mock.patch.object(GlueJobHook, "conn")
     def test_print_job_logs_no_stream_yet(self, conn_mock: MagicMock, client_mock: MagicMock):
         hook = GlueJobHook()