Repository: localstack/localstack

Issue:
Kinesis to S3 file structure not correct
For Kinesis, it seems to be, that AWS organizes its folders on S3 like: "prefix/2019/02/09/13/kinesis-stream-name-2019-02-09-10-02-32-db1db7a9-c484-40d1-b7ec-7651a55e4677"

The date in the path is UTC-time, the date in the file name is the UTC-time of the first received byte in the Kinesis stream for the specific file. This is the default behavior, so it would be nice to implement it in localstack.


Discussion:

Resolving patch:
diff --git a/localstack/services/firehose/firehose_api.py b/localstack/services/firehose/firehose_api.py
--- a/localstack/services/firehose/firehose_api.py
+++ b/localstack/services/firehose/firehose_api.py
@@ -12,7 +12,7 @@
 from localstack.services import generic_proxy
 from localstack.constants import TEST_AWS_ACCOUNT_ID
 from localstack.utils.aws import aws_responses
-from localstack.utils.common import short_uid, to_str
+from localstack.utils.common import short_uid, to_str, timestamp
 from localstack.utils.aws.aws_stack import (
     get_s3_client, firehose_stream_arn, connect_elasticsearch, extract_region_from_auth_header)
 from localstack.utils.kinesis import kinesis_connector
@@ -94,8 +94,7 @@ def put_records(stream_name, records):
                 elif 'data' in record:
                     data = base64.b64decode(record['data'])
 
-                obj_name = str(uuid.uuid4())
-                obj_path = '%s%s%s' % (prefix, '' if prefix.endswith('/') else '/', obj_name)
+                obj_path = get_s3_object_path(stream_name, prefix)
                 try:
                     s3.Object(bucket, obj_path).put(Body=data)
                 except Exception as e:
@@ -103,6 +102,17 @@ def put_records(stream_name, records):
                     raise e
 
 
+def get_s3_object_path(stream_name, prefix):
+    # See https://aws.amazon.com/kinesis/data-firehose/faqs/#Data_delivery
+    # Path prefix pattern: myApp/YYYY/MM/DD/HH/
+    # Object name pattern: DeliveryStreamName-DeliveryStreamVersion-YYYY-MM-DD-HH-MM-SS-RandomString
+    prefix = '%s%s' % (prefix, '' if prefix.endswith('/') else '/')
+    pattern = '{pre}%Y/%m/%d/%H/{name}-%Y-%m-%d-%H-%M-%S-{rand}'
+    path = pattern.format(pre=prefix, name=stream_name, rand=str(uuid.uuid4()))
+    path = timestamp(format=path)
+    return path
+
+
 def get_destination(stream_name, destination_id):
     stream = get_stream(stream_name)
     destinations = stream['Destinations']
diff --git a/localstack/utils/testutil.py b/localstack/utils/testutil.py
--- a/localstack/utils/testutil.py
+++ b/localstack/utils/testutil.py
@@ -246,7 +246,8 @@ def map_all_s3_objects(to_json=True, buckets=None):
             try:
                 if to_json:
                     value = json.loads(value)
-                result['%s/%s' % (key.bucket_name, key.key)] = value
+                key = '%s%s%s' % (key.bucket_name, '' if key.key.startswith('/') else '/', key.key)
+                result[key] = value
             except Exception:
                 # skip non-JSON or binary objects
                 pass


Test patch:
diff --git a/tests/integration/test_integration.py b/tests/integration/test_integration.py
--- a/tests/integration/test_integration.py
+++ b/tests/integration/test_integration.py
@@ -10,7 +10,6 @@
 from localstack.utils import testutil
 from localstack.utils.common import (
     load_file, short_uid, clone, to_bytes, to_str, run_safe, retry)
-# from localstack.utils.common import profiled
 from localstack.services.awslambda.lambda_api import LAMBDA_RUNTIME_PYTHON27
 from localstack.utils.kinesis import kinesis_connector
 from localstack.utils.aws import aws_stack
@@ -75,6 +74,10 @@ def test_firehose_s3(self):
         # check records in target bucket
         all_objects = testutil.list_all_s3_objects()
         testutil.assert_objects(json.loads(to_str(test_data)), all_objects)
+        # check file layout in target bucket
+        all_objects = testutil.map_all_s3_objects(buckets=[TEST_BUCKET_NAME])
+        for key in all_objects.keys():
+            self.assertRegexpMatches(key, r'.*/\d{4}/\d{2}/\d{2}/\d{2}/.*\-\d{4}\-\d{2}\-\d{2}\-\d{2}.*')
 
     def test_firehose_kinesis_to_s3(self):
         kinesis = aws_stack.connect_to_service('kinesis')