Repository: getmoto/moto

Issue:
Failing to simulate authentication with `mock_iam` and `mock_s3` due to generated keys not validated successfully
I'm writing unit tests for a function that should authenticate to AWS S3 and then perform some operations on S3. I have a bunch of functions that do various things (like downloading/uploading files, checking for existence, etc).

Since in production these will need authentication, I want to simulate it in the tests, so that I can check that this part of the code is ok too.

For mocking the AWS environment I'm using `moto`. As of now I have the following code and traceback - all below. A quick intro to it: `my_head_bucket` is an example function that needs to have unit tests. `create_user_with_access_key_and_policy` should mock the IAM user and policy needed for authenticated S3 access. It is almost the same as in the examples in the documentation.

Then there are two tests. The first should pass without errors (having the correct authentication). The second should fail with `ClientError`, because `"invalid"` keys are being passed.

For some reason I am not able to pass through the creation of the mock user and policy, getting `botocore.exceptions.ClientError: An error occurred (InvalidClientTokenId) when calling the AttachUserPolicy operation: The security token included in the request is invalid.` It seems that no authentication should be needed for `create_access_key`, so what am I doing wrong?
```
import os
import unittest
import boto3
import botocore
from moto import mock_s3, mock_iam
from botocore.client import ClientError
from moto.core import set_initial_no_auth_action_count

import json

def my_head_bucket(bucket, aws_access_key_id, aws_secret_access_key):
    """
    This is a sample function. In the real case, this function will do more than
    just heading a bucket (like uploading/downloading files or other actions).
    
    It would be imported from another module and should not have any decorators (like moto's)
    """
    s3_client = boto3.client("s3", aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
    s3_client.head_bucket(Bucket=bucket)

@mock_iam
def create_user_with_access_key_and_policy(user_name="test-user"):
    """
    Should create a user with attached policy allowing read/write operations on S3.
    """
    policy_document = {
        "Version": "2012-10-17",
        "Statement": [
            {"Effect": "Allow", "Action": "s3:*", "Resource": "*"}
        ],
    }
    
    # Create client and user
    client = boto3.client("iam", region_name="us-east-1")
    client.create_user(UserName=user_name)

    # Create and attach the policy
    policy_arn = client.create_policy(
        PolicyName="policy1", PolicyDocument=json.dumps(policy_document)
    )["Policy"]["Arn"]
    client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn)
    
    # Return the access keys
    return client.create_access_key(UserName=user_name)["AccessKey"]

class TestMyTest(unittest.TestCase):
    @set_initial_no_auth_action_count(0)
    @mock_s3
    def test_correct_credentials(self):
        """
        Sets the environment (creates user with keys and policy, creates the bucket), then calls
        the function-to-be-tested and expects it to run without exceptions.
        """
        ### Arrange
        iam_keys = create_user_with_access_key_and_policy()
        print(iam_keys)

        s3 = boto3.client('s3', aws_access_key_id=iam_keys["AccessKeyId"],aws_secret_access_key=iam_keys["SecretAccessKey"])
        s3.create_bucket(Bucket='mock_bucket')
        
        my_head_bucket('mock_bucket', aws_access_key_id=iam_keys["AccessKeyId"],aws_secret_access_key=iam_keys["SecretAccessKey"])
        
    @set_initial_no_auth_action_count(0)
    @mock_s3
    def test_incorrect_credentials(self):
        """
        Sets the environment (creates user with keys and policy, creates the bucket), then calls
        the function-to-be-tested and expects it to run without exceptions.
        """
        ### Arrange
        iam_keys = create_user_with_access_key_and_policy()
        print(iam_keys)

        s3 = boto3.client('s3', aws_access_key_id=iam_keys["AccessKeyId"],aws_secret_access_key=iam_keys["SecretAccessKey"])
        s3.create_bucket(Bucket='mock_bucket')
        
        with self.assertRaises(ClientError):
            my_head_bucket('mock_bucket', aws_access_key_id=iam_keys["AccessKeyId"],aws_secret_access_key="invalid")
```

----------------

```
==================================================================================================== test session starts =====================================================================================================
platform linux -- Python 3.6.8, pytest-6.2.5, py-1.10.0, pluggy-0.13.1
rootdir: /home/trusso.storage.maintenance
collected 2 items                                                                                                                                                                                                            

some_test.py FF                                                                                                                                                                                                        [100%]

========================================================================================================== FAILURES ==========================================================================================================
____________________________________________________________________________________________ TestMyTest.test_correct_credentials _____________________________________________________________________________________________

self = <some_test.TestMyTest testMethod=test_correct_credentials>

    @set_initial_no_auth_action_count(0)
    @mock_s3
    def test_correct_credentials(self):
        """
        Sets the environment (creates user with keys and policy, creates the bucket), then calls
        the function-to-be-tested and expects it to run without exceptions.
        """
        ### Arrange
>       iam_keys = create_user_with_access_key_and_policy()

some_test.py:55: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../venvs/storage_env/lib/python3.6/site-packages/moto/core/models.py:115: in wrapper
    result = func(*args, **kwargs)
some_test.py:35: in create_user_with_access_key_and_policy
    client.create_user(UserName=user_name)
../venvs/storage_env/lib/python3.6/site-packages/botocore/client.py:386: in _api_call
    return self._make_api_call(operation_name, kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <botocore.client.IAM object at 0x7f628c16ab00>, operation_name = 'CreateUser', api_params = {'UserName': 'test-user'}

    def _make_api_call(self, operation_name, api_params):
        operation_model = self._service_model.operation_model(operation_name)
        service_name = self._service_model.service_name
        history_recorder.record('API_CALL', {
            'service': service_name,
            'operation': operation_name,
            'params': api_params,
        })
        if operation_model.deprecated:
            logger.debug('Warning: %s.%s() is deprecated',
                         service_name, operation_name)
        request_context = {
            'client_region': self.meta.region_name,
            'client_config': self.meta.config,
            'has_streaming_input': operation_model.has_streaming_input,
            'auth_type': operation_model.auth_type,
        }
        request_dict = self._convert_to_request_dict(
            api_params, operation_model, context=request_context)
    
        service_id = self._service_model.service_id.hyphenize()
        handler, event_response = self.meta.events.emit_until_response(
            'before-call.{service_id}.{operation_name}'.format(
                service_id=service_id,
                operation_name=operation_name),
            model=operation_model, params=request_dict,
            request_signer=self._request_signer, context=request_context)
    
        if event_response is not None:
            http, parsed_response = event_response
        else:
            http, parsed_response = self._make_request(
                operation_model, request_dict, request_context)
    
        self.meta.events.emit(
            'after-call.{service_id}.{operation_name}'.format(
                service_id=service_id,
                operation_name=operation_name),
            http_response=http, parsed=parsed_response,
            model=operation_model, context=request_context
        )
    
        if http.status_code >= 300:
            error_code = parsed_response.get("Error", {}).get("Code")
            error_class = self.exceptions.from_code(error_code)
>           raise error_class(parsed_response, operation_name)
E           botocore.exceptions.ClientError: An error occurred (InvalidClientTokenId) when calling the CreateUser operation: The security token included in the request is invalid.

../venvs/storage_env/lib/python3.6/site-packages/botocore/client.py:705: ClientError
___________________________________________________________________________________________ TestMyTest.test_incorrect_credentials ____________________________________________________________________________________________

self = <some_test.TestMyTest testMethod=test_incorrect_credentials>

    @set_initial_no_auth_action_count(0)
    @mock_s3
    def test_incorrect_credentials(self):
        """
        Sets the environment (creates user with keys and policy, creates the bucket), then calls
        the function-to-be-tested and expects it to run without exceptions.
        """
        ### Arrange
>       iam_keys = create_user_with_access_key_and_policy()

some_test.py:71: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
../venvs/storage_env/lib/python3.6/site-packages/moto/core/models.py:115: in wrapper
    result = func(*args, **kwargs)
some_test.py:35: in create_user_with_access_key_and_policy
    client.create_user(UserName=user_name)
../venvs/storage_env/lib/python3.6/site-packages/botocore/client.py:386: in _api_call
    return self._make_api_call(operation_name, kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <botocore.client.IAM object at 0x7f628b870550>, operation_name = 'CreateUser', api_params = {'UserName': 'test-user'}

    def _make_api_call(self, operation_name, api_params):
        operation_model = self._service_model.operation_model(operation_name)
        service_name = self._service_model.service_name
        history_recorder.record('API_CALL', {
            'service': service_name,
            'operation': operation_name,
            'params': api_params,
        })
        if operation_model.deprecated:
            logger.debug('Warning: %s.%s() is deprecated',
                         service_name, operation_name)
        request_context = {
            'client_region': self.meta.region_name,
            'client_config': self.meta.config,
            'has_streaming_input': operation_model.has_streaming_input,
            'auth_type': operation_model.auth_type,
        }
        request_dict = self._convert_to_request_dict(
            api_params, operation_model, context=request_context)
    
        service_id = self._service_model.service_id.hyphenize()
        handler, event_response = self.meta.events.emit_until_response(
            'before-call.{service_id}.{operation_name}'.format(
                service_id=service_id,
                operation_name=operation_name),
            model=operation_model, params=request_dict,
            request_signer=self._request_signer, context=request_context)
    
        if event_response is not None:
            http, parsed_response = event_response
        else:
            http, parsed_response = self._make_request(
                operation_model, request_dict, request_context)
    
        self.meta.events.emit(
            'after-call.{service_id}.{operation_name}'.format(
                service_id=service_id,
                operation_name=operation_name),
            http_response=http, parsed=parsed_response,
            model=operation_model, context=request_context
        )
    
        if http.status_code >= 300:
            error_code = parsed_response.get("Error", {}).get("Code")
            error_class = self.exceptions.from_code(error_code)
>           raise error_class(parsed_response, operation_name)
E           botocore.exceptions.ClientError: An error occurred (InvalidClientTokenId) when calling the CreateUser operation: The security token included in the request is invalid.

../venvs/storage_env/lib/python3.6/site-packages/botocore/client.py:705: ClientError
```

-------------

Versions:
`moto`: 2.2.7
`boto3`: 1.18.46
`botocore`: 1.21.46


Discussion:
Hi @ivanov-slk!

The @set_initial_no_auth_action_count(0) can be read as:
`@authenticate_all(except_first_0_requests)`

When starting the server, there are no users/roles at all, so almost all requests will fail. To get around this, all requests that deal with creating users/roles should be exempt from authentication.
In your case, there are 4: create_user, create_policy, attach_user_policy and create_access_key, so the decorator-parameter should be set to 4:
`@set_initial_no_auth_action_count(4)`
The initial 4 IAM requests will go through without any problem like this, but the 5th request to S3 will be authenticated as expected.

Changing this will pass the first test.

The second test will still fail though, with this change, and I'm not sure why. Will mark it as a bug until we can figure out what's going wrong there.
Potentially related: #4240 
Thank you for the clarification about `set_initial_no_auth_action_count` !

Confirmed that the first test works and the second still doesn't (which is, passes when it must fail). I'll wait for the resolution.
Hmmm... played around a little bit. Actually, inserting a call to `s3_client.list_objects` call does make the second test pass.

Interestingly enough, AWS's [official documentation](https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html#S3.Client.head_bucket) says that `head_bucket` should return 403 if the caller has no access. Thus, the call to `head_bucket` should raise `ClientError` anyways.

Pasting my code below for full reference and with linting:
```
import json
import os
import unittest

import boto3
import botocore
from botocore.client import ClientError
from moto import mock_iam, mock_s3
from moto.core import set_initial_no_auth_action_count


def my_head_bucket(bucket, aws_access_key_id, aws_secret_access_key):
    """
    This is a sample function. In the real case, this function will do more than
    just heading a bucket (like uploading/downloading files or other actions).
    
    It would be imported from another module and should not have any decorators (like moto's)
    """
    s3_client = boto3.client("s3",
                             aws_access_key_id=aws_access_key_id,
                             aws_secret_access_key=aws_secret_access_key)
    s3_client.head_bucket(Bucket=bucket)
    s3_result = s3_client.list_objects(Bucket=bucket, Prefix='')


@mock_iam
def create_user_with_access_key_and_policy(user_name="test-user"):
    """
    Should create a user with attached policy allowing read/write operations on S3.
    """
    policy_document = {
        "Version": "2012-10-17",
        "Statement": [{
            "Effect": "Allow",
            "Action": "s3:*",
            "Resource": "*"
        }],
    }

    # Create client and user
    client = boto3.client("iam", region_name="us-east-1")
    client.create_user(UserName=user_name)

    # Create and attach the policy
    policy_arn = client.create_policy(PolicyName="policy1",
                                      PolicyDocument=json.dumps(policy_document))["Policy"]["Arn"]
    client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn)

    # Return the access keys
    return client.create_access_key(UserName=user_name)["AccessKey"]


class TestMyTest(unittest.TestCase):
    @set_initial_no_auth_action_count(4)
    @mock_s3
    def test_correct_credentials(self):
        """
        Sets the environment (creates user with keys and policy, creates the bucket), then calls
        the function-to-be-tested and expects it to run without exceptions.
        """
        ### Arrange
        iam_keys = create_user_with_access_key_and_policy()
        print(iam_keys)

        s3 = boto3.client('s3',
                          aws_access_key_id=iam_keys["AccessKeyId"],
                          aws_secret_access_key=iam_keys["SecretAccessKey"])
        s3.create_bucket(Bucket='mock_bucket')

        my_head_bucket('mock_bucket',
                       aws_access_key_id=iam_keys["AccessKeyId"],
                       aws_secret_access_key=iam_keys["SecretAccessKey"])

    @set_initial_no_auth_action_count(4)
    @mock_s3
    def test_incorrect_credentials(self):
        """
        Sets the environment (creates user with keys and policy, creates the bucket), then calls
        the function-to-be-tested and expects it to run without exceptions.
        """
        ### Arrange
        iam_keys = create_user_with_access_key_and_policy()
        print(iam_keys)

        s3 = boto3.client('s3',
                          aws_access_key_id=iam_keys["AccessKeyId"],
                          aws_secret_access_key=iam_keys["SecretAccessKey"])
        s3.create_bucket(Bucket='mock_bucket')

        with self.assertRaises(ClientError):
            my_head_bucket('mock_bucket',
                           aws_access_key_id=iam_keys["AccessKeyId"],
                           aws_secret_access_key="invalid")

```

Resolving patch:
diff --git a/moto/s3/responses.py b/moto/s3/responses.py
--- a/moto/s3/responses.py
+++ b/moto/s3/responses.py
@@ -80,6 +80,7 @@
 
 ACTION_MAP = {
     "BUCKET": {
+        "HEAD": {"DEFAULT": "HeadBucket",},
         "GET": {
             "uploads": "ListBucketMultipartUploads",
             "location": "GetBucketLocation",
@@ -311,7 +312,7 @@ def _bucket_response(self, request, full_url, headers):
         body = "{0}".format(body).encode("utf-8")
 
         if method == "HEAD":
-            return self._bucket_response_head(bucket_name)
+            return self._bucket_response_head(bucket_name, querystring)
         elif method == "GET":
             return self._bucket_response_get(bucket_name, querystring)
         elif method == "PUT":
@@ -335,7 +336,10 @@ def _get_querystring(full_url):
         querystring = parse_qs(parsed_url.query, keep_blank_values=True)
         return querystring
 
-    def _bucket_response_head(self, bucket_name):
+    def _bucket_response_head(self, bucket_name, querystring):
+        self._set_action("BUCKET", "HEAD", querystring)
+        self._authenticate_and_authorize_s3_action()
+
         try:
             self.backend.head_bucket(bucket_name)
         except MissingBucket:


Test patch:
diff --git a/tests/test_s3/test_s3_auth.py b/tests/test_s3/test_s3_auth.py
--- a/tests/test_s3/test_s3_auth.py
+++ b/tests/test_s3/test_s3_auth.py
@@ -1,8 +1,10 @@
 import boto3
+import json
 import pytest
+import sure  # noqa
 
 from botocore.exceptions import ClientError
-from moto import mock_s3, settings
+from moto import mock_iam, mock_s3, settings
 from moto.core import set_initial_no_auth_action_count
 from unittest import SkipTest
 
@@ -24,3 +26,93 @@ def test_load_unexisting_object_without_auth_should_return_403():
     err["Message"].should.equal(
         "The AWS Access Key Id you provided does not exist in our records."
     )
+
+
+@set_initial_no_auth_action_count(4)
+@mock_s3
+def test_head_bucket_with_correct_credentials():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Auth decorator does not work in server mode")
+
+    # These calls are all unauthenticated
+    iam_keys = create_user_with_access_key_and_policy()
+
+    # This S3-client has correct credentials
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=iam_keys["AccessKeyId"],
+        aws_secret_access_key=iam_keys["SecretAccessKey"],
+    )
+    s3.create_bucket(Bucket="mock_bucket")
+
+    # Calling head_bucket with the correct credentials works
+    my_head_bucket(
+        "mock_bucket",
+        aws_access_key_id=iam_keys["AccessKeyId"],
+        aws_secret_access_key=iam_keys["SecretAccessKey"],
+    )
+
+
+@set_initial_no_auth_action_count(4)
+@mock_s3
+def test_head_bucket_with_incorrect_credentials():
+    if settings.TEST_SERVER_MODE:
+        raise SkipTest("Auth decorator does not work in server mode")
+
+    # These calls are all authenticated
+    iam_keys = create_user_with_access_key_and_policy()
+
+    # Create the bucket with correct credentials
+    s3 = boto3.client(
+        "s3",
+        aws_access_key_id=iam_keys["AccessKeyId"],
+        aws_secret_access_key=iam_keys["SecretAccessKey"],
+    )
+    s3.create_bucket(Bucket="mock_bucket")
+
+    # Call head_bucket with incorrect credentials
+    with pytest.raises(ClientError) as ex:
+        my_head_bucket(
+            "mock_bucket",
+            aws_access_key_id=iam_keys["AccessKeyId"],
+            aws_secret_access_key="invalid",
+        )
+    err = ex.value.response["Error"]
+    err["Code"].should.equal("SignatureDoesNotMatch")
+    err["Message"].should.equal(
+        "The request signature we calculated does not match the signature you provided. "
+        "Check your key and signing method."
+    )
+
+
+def my_head_bucket(bucket, aws_access_key_id, aws_secret_access_key):
+    s3_client = boto3.client(
+        "s3",
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+    )
+    s3_client.head_bucket(Bucket=bucket)
+
+
+@mock_iam
+def create_user_with_access_key_and_policy(user_name="test-user"):
+    """
+    Should create a user with attached policy allowing read/write operations on S3.
+    """
+    policy_document = {
+        "Version": "2012-10-17",
+        "Statement": [{"Effect": "Allow", "Action": "s3:*", "Resource": "*"}],
+    }
+
+    # Create client and user
+    client = boto3.client("iam", region_name="us-east-1")
+    client.create_user(UserName=user_name)
+
+    # Create and attach the policy
+    policy_arn = client.create_policy(
+        PolicyName="policy1", PolicyDocument=json.dumps(policy_document)
+    )["Policy"]["Arn"]
+    client.attach_user_policy(UserName=user_name, PolicyArn=policy_arn)
+
+    # Return the access keys
+    return client.create_access_key(UserName=user_name)["AccessKey"]