Repository: great-expectations/great_expectations

Issue:
Bug: Creating suites from `.csv.gz` files on S3
**Describe the bug**
When creating a suite on S3 using `.csv.gz` files the `read_options` aren't passed to the `batch_kwargs`.  So the `datasource` doesn't know it should try to decompress the file.  This is using `v2` of the CLI.

**To Reproduce**
Steps to reproduce the behavior:
1. Have a `.csv.gz` on S3
2. Try to scaffold a suite on the CLI

![Screenshot 2021-04-19 at 16 25 24](https://user-images.githubusercontent.com/1042275/115261980-03d73300-a12c-11eb-8aa4-37dd917015f5.png)


**Expected behavior**
Great Expectations to detect the read options needed, as it does in other contexts, and use these

**Environment (please complete the following information):**
 - Operating System: Linux
 - Great Expectations Version: 0.13.17

**Additional context**
I'll submit a PR to fix this



Discussion:

Resolving patch:
diff --git a/great_expectations/cli/v012/datasource.py b/great_expectations/cli/v012/datasource.py
--- a/great_expectations/cli/v012/datasource.py
+++ b/great_expectations/cli/v012/datasource.py
@@ -1163,9 +1163,9 @@ def _get_batch_kwargs_from_generator_or_from_file_path(
 
         reader_method = None
         try:
-            reader_method = datasource.guess_reader_method_from_path(path)[
-                "reader_method"
-            ]
+            reader_kwargs = datasource.guess_reader_method_from_path(path)
+            reader_method = reader_kwargs["reader_method"]
+            reader_options = reader_kwargs.get("reader_options", {})
         except BatchKwargsError:
             pass
 
@@ -1201,6 +1201,12 @@ def _get_batch_kwargs_from_generator_or_from_file_path(
         else:
             try:
                 batch_kwargs["reader_method"] = reader_method
+                reader_options = {
+                    **batch_kwargs.get("reader_options", {}),
+                    **reader_options,
+                }
+                if reader_options:
+                    batch_kwargs["reader_options"] = reader_options
                 if isinstance(datasource, SparkDFDatasource) and reader_method == "csv":
                     header_row = click.confirm(
                         "\nDoes this file contain a header row?", default=True


Test patch:
diff --git a/tests/cli/v012/test_suite.py b/tests/cli/v012/test_suite.py
--- a/tests/cli/v012/test_suite.py
+++ b/tests/cli/v012/test_suite.py
@@ -377,6 +377,41 @@ def test_suite_new_empty_with_no_jupyter(
     )
 
 
+@mock.patch("subprocess.call", return_value=True, side_effect=None)
+@mock.patch("webbrowser.open", return_value=True, side_effect=None)
+def test_suite_new_empty_with_csv_gz(
+    mock_webbroser,
+    mock_subprocess,
+    caplog,
+    data_context_parameterized_expectation_suite,
+    filesystem_csv_2_gz,
+):
+    os.mkdir(
+        os.path.join(
+            data_context_parameterized_expectation_suite.root_directory, "uncommitted"
+        )
+    )
+    root_dir = data_context_parameterized_expectation_suite.root_directory
+    runner = CliRunner(mix_stderr=False)
+    csv = os.path.join(filesystem_csv_2_gz, "f1.csv.gz")
+    result = runner.invoke(
+        cli,
+        ["suite", "new", "-d", root_dir, "--suite", "foo"],
+        input=f"{csv}\n",
+        catch_exceptions=False,
+    )
+    print(result.stdout)
+    stdout = result.stdout
+
+    assert result.exit_code == 0
+    assert "Cannot load file" not in stdout
+    context = DataContext(root_dir)
+    suite = context.get_expectation_suite("foo")
+    citations = suite.get_citations()
+    assert citations[0]["batch_kwargs"]["reader_method"] == "read_csv"
+    assert citations[0]["batch_kwargs"]["reader_options"] == {"compression": "gzip"}
+
+
 @mock.patch("subprocess.call", return_value=True, side_effect=None)
 @mock.patch("webbrowser.open", return_value=True, side_effect=None)
 def test_suite_demo_one_datasource_without_generator_without_suite_name_argument(
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,10 +1,12 @@
 import datetime
+import gzip
 import json
 import locale
 import logging
 import os
 import random
 import shutil
+from pathlib import Path
 from typing import Dict, List
 
 import numpy as np
@@ -3804,6 +3806,20 @@ def filesystem_csv_2(tmp_path):
     return base_dir
 
 
+@pytest.fixture
+def filesystem_csv_2_gz(filesystem_csv_2):
+    _compress_csv_in_dir(filesystem_csv_2)
+    return filesystem_csv_2
+
+
+def _compress_csv_in_dir(dir: os.PathLike):
+    dir = Path(dir)
+    for file in dir.glob("*.csv"):
+        file_gz = file.with_name(file.name + ".gz")
+        file_gz.write_bytes(gzip.compress(file.read_bytes(), compresslevel=1))
+        file.unlink()
+
+
 @pytest.fixture(scope="function")
 def filesystem_csv_3(tmp_path):
     base_dir = tmp_path / "filesystem_csv_3"