Repository: feast-dev/feast

Issue:
Python SDK create_dataset is actually creating dataset in BQ
## Expected Behavior 

"Dataset" that was meant on the sdk is a collection of features selected by the user. While this term is also used in BigQuery, the SDK should *not* create a dataset whenever this function is called, rather just create a view in the feast dataset.

## Current Behavior

It creates a new BigQuery Dataset.

## Steps to reproduce

Use the [quickstart](https://github.com/gojek/feast/blob/master/sdk/python/examples/quickstart/Quickstart.ipynb):

```python
feature_set = FeatureSet(entity="ride", 
  features=["ride.log_trip_duration", 
     "ride.distance_haversine",
     "ride.distance_dummy_manhattan",
     "ride.direction",
     "ride.month",
     "ride.day_of_month",
     "ride.hour",
     "ride.day_of_week",
     "ride.vi_1",
     "ride.vi_2",
     "ride.sf_n",
     "ride.sf_y"])
dataset_info = fs.create_dataset(feature_set, "2016-06-01", "2016-08-01")
dataset = fs.download_dataset_to_df(dataset_info, staging_location=STAGING_LOCATION)

dataset.head()
``` 

### Specifications

- Version: latest
- Platform: as per suggested by [install guide](https://github.com/gojek/feast/blob/master/docs/install.md)
- Subsystem: as per suggested by [install guide](https://github.com/gojek/feast/blob/master/docs/install.md)

## Possible Solution

- Fix `create_dataset` to create view in feast dataset instead
- Write down dataset definition
- Change to `create_view`?


Discussion:

Resolving patch:
diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py
--- a/sdk/python/feast/sdk/client.py
+++ b/sdk/python/feast/sdk/client.py
@@ -266,7 +266,7 @@ def download_dataset(
             str: path to the downloaded file
         """
         return self._table_downloader.download_table_as_file(
-            dataset_info.table_id, dest, staging_location, file_type
+            dataset_info.full_table_id, dest, staging_location, file_type
         )
 
     def download_dataset_to_df(self, dataset_info, staging_location):
@@ -282,7 +282,7 @@ def download_dataset_to_df(self, dataset_info, staging_location):
 
         """
         return self._table_downloader.download_table_as_df(
-            dataset_info.table_id, staging_location
+            dataset_info.full_table_id, staging_location
         )
 
     def close(self):
diff --git a/sdk/python/feast/sdk/resources/feature_set.py b/sdk/python/feast/sdk/resources/feature_set.py
--- a/sdk/python/feast/sdk/resources/feature_set.py
+++ b/sdk/python/feast/sdk/resources/feature_set.py
@@ -66,16 +66,16 @@ class FileType(object):
 
 
 class DatasetInfo:
-    def __init__(self, name, table_id):
+    def __init__(self, name, full_table_id):
         """
             Create instance of DatasetInfo with a BigQuery table as its
             backing store.
         Args:
             name: (str) dataset name
-            table_id: (str) fully qualified table id
+            full_table_id: (str) fully qualified table id
         """
         self._name = name
-        self._table_id = table_id
+        self._full_table_id = full_table_id
 
     @property
     def name(self):
@@ -87,10 +87,10 @@ def name(self):
         return self._name
 
     @property
-    def table_id(self):
+    def full_table_id(self):
         """
 
         Returns: fully qualified table id
 
         """
-        return self._table_id
+        return self._full_table_id
diff --git a/sdk/python/feast/sdk/utils/bq_util.py b/sdk/python/feast/sdk/utils/bq_util.py
--- a/sdk/python/feast/sdk/utils/bq_util.py
+++ b/sdk/python/feast/sdk/utils/bq_util.py
@@ -197,11 +197,11 @@ def bq(self):
             self._bq = BQClient()
         return self._bq
 
-    def download_table_as_file(self, table_id, dest, staging_location, file_type):
+    def download_table_as_file(self, full_table_id, dest, staging_location, file_type):
         """
         Download a bigquery table as file
         Args:
-            table_id (str): fully qualified BigQuery table id
+            full_table_id (str): fully qualified BigQuery table id
             dest (str): destination filename
             staging_location (str): url to staging_location (currently
                 support a folder in GCS)
@@ -218,7 +218,7 @@ def download_table_as_file(self, table_id, dest, staging_location, file_type):
 
         job_config = ExtractJobConfig()
         job_config.destination_format = file_type
-        src_table = Table.from_string(table_id)
+        src_table = Table.from_string(full_table_id)
         job = self.bq.extract_table(src_table, staging_file_path, job_config=job_config)
 
         # await completion
@@ -230,11 +230,11 @@ def download_table_as_file(self, table_id, dest, staging_location, file_type):
         blob.download_to_filename(dest)
         return dest
 
-    def download_table_as_df(self, table_id, staging_location):
+    def download_table_as_df(self, full_table_id, staging_location):
         """
         Download a BigQuery table as Pandas Dataframe
         Args:
-            table_id (src) : fully qualified BigQuery table id
+            full_table_id (src) : fully qualified BigQuery table id
             staging_location: url to staging_location (currently
                 support a folder in GCS)
 
@@ -250,7 +250,7 @@ def download_table_as_df(self, table_id, staging_location):
         job_config = ExtractJobConfig()
         job_config.destination_format = DestinationFormat.CSV
         job = self.bq.extract_table(
-            Table.from_string(table_id), staging_file_path, job_config=job_config
+            Table.from_string(full_table_id), staging_file_path, job_config=job_config
         )
 
         # await completion


Test patch:
diff --git a/core/src/test/java/feast/core/training/BigQueryTraningDatasetCreatorTest.java b/core/src/test/java/feast/core/training/BigQueryTraningDatasetCreatorTest.java
--- a/core/src/test/java/feast/core/training/BigQueryTraningDatasetCreatorTest.java
+++ b/core/src/test/java/feast/core/training/BigQueryTraningDatasetCreatorTest.java
@@ -16,13 +16,6 @@
  */
 package feast.core.training;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.junit.Assert.assertThat;
-import static org.mockito.ArgumentMatchers.any;
-import static org.mockito.ArgumentMatchers.anyLong;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
 import com.google.cloud.bigquery.BigQuery;
 import com.google.protobuf.Timestamp;
 import com.google.protobuf.util.Timestamps;
@@ -30,7 +23,6 @@
 import feast.core.DatasetServiceProto.FeatureSet;
 import feast.core.storage.BigQueryStorageManager;
 import feast.specs.StorageSpecProto.StorageSpec;
-import java.time.Clock;
 import java.time.Instant;
 import java.util.Arrays;
 import org.junit.Before;
@@ -38,6 +30,21 @@
 import org.mockito.Mock;
 import org.mockito.MockitoAnnotations;
 
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.Assert.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+// TODO: Should consider testing with "actual" BigQuery vs mocking it
+//       because the mocked BigQuery client is very basic and may miss important functionalities
+//       such as an actual table / dataset is actually created
+//       In the test method, should probably add a condition so that tests can be skipped if
+//       the user running the tests do not have permission to manage BigQuery (although ideally they should have)
+//       Example of adding the condition whether or not to accept the test result as valid:
+//       https://stackoverflow.com/questions/1689242/conditionally-ignoring-tests-in-junit-4
+
 public class BigQueryTraningDatasetCreatorTest {
 
   public static final String projectId = "the-project";
@@ -48,11 +55,9 @@ public class BigQueryTraningDatasetCreatorTest {
   private BigQueryDatasetTemplater templater;
   @Mock
   private BigQuery bq;
-  @Mock
-  private Clock clock;
 
   @Before
-  public void setUp() throws Exception {
+  public void setUp() {
     MockitoAnnotations.initMocks(this);
     when(templater.getStorageSpec()).thenReturn(StorageSpec.newBuilder()
         .setId("BIGQUERY1")
@@ -60,14 +65,13 @@ public void setUp() throws Exception {
         .putOptions("project", "project")
         .putOptions("dataset", "dataset")
         .build());
-    creator = new BigQueryTraningDatasetCreator(templater, clock, projectId, datasetPrefix, bq);
+    creator = new BigQueryTraningDatasetCreator(templater, projectId, datasetPrefix, bq);
 
     when(templater.createQuery(
         any(FeatureSet.class), any(Timestamp.class), any(Timestamp.class), anyLong()))
         .thenReturn("SELECT * FROM `project.dataset.table`");
   }
 
-
   @Test
   public void shouldCreateCorrectDatasetIfPrefixNotSpecified() {
     String entityName = "myentity";
@@ -85,14 +89,15 @@ public void shouldCreateCorrectDatasetIfPrefixNotSpecified() {
     long limit = 999;
     String namePrefix = "";
 
-    DatasetInfo dsInfo =
-        creator.createDataset(featureSet, startDate, endDate, limit, namePrefix);
-    assertThat(dsInfo.getName(), equalTo("myentity_0_20180101_20190101"));
+    DatasetInfo dsInfo = creator.createDataset(featureSet, startDate, endDate, limit, namePrefix);
+    assertThat(
+        dsInfo.getName(), equalTo("feast_myentity_b0009f0f7df634ddc130571319e0deb9742eb1da"));
     assertThat(
         dsInfo.getTableUrl(),
         equalTo(
             String.format(
-                "%s.%s_%s.%s", projectId, datasetPrefix, entityName, "0_20180101_20190101")));
+                "%s.dataset.%s_%s_%s",
+                projectId, datasetPrefix, entityName, "b0009f0f7df634ddc130571319e0deb9742eb1da")));
   }
 
   @Test
@@ -112,15 +117,20 @@ public void shouldCreateCorrectDatasetIfPrefixIsSpecified() {
     long limit = 999;
     String namePrefix = "mydataset";
 
-    DatasetInfo dsInfo =
-        creator.createDataset(featureSet, startDate, endDate, limit, namePrefix);
+    DatasetInfo dsInfo = creator.createDataset(featureSet, startDate, endDate, limit, namePrefix);
     assertThat(
         dsInfo.getTableUrl(),
         equalTo(
             String.format(
-                "%s.%s_%s.%s", projectId, datasetPrefix, entityName,
-                "mydataset_0_20180101_20190101")));
-    assertThat(dsInfo.getName(), equalTo("mydataset_0_20180101_20190101"));
+                "%s.dataset.%s_%s_%s_%s",
+                projectId,
+                datasetPrefix,
+                entityName,
+                namePrefix,
+                "b0009f0f7df634ddc130571319e0deb9742eb1da")));
+    assertThat(
+        dsInfo.getName(),
+        equalTo("feast_myentity_mydataset_b0009f0f7df634ddc130571319e0deb9742eb1da"));
   }
 
   @Test
diff --git a/sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro b/sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro
Binary files a/sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro and b/sdk/python/tests/data/austin_bikeshare.bikeshare_stations.avro differ
diff --git a/sdk/python/tests/sdk/resources/test_feature_set.py b/sdk/python/tests/sdk/resources/test_feature_set.py
--- a/sdk/python/tests/sdk/resources/test_feature_set.py
+++ b/sdk/python/tests/sdk/resources/test_feature_set.py
@@ -38,7 +38,7 @@ def test_different_entity(self):
 class TestDatasetInfo(object):
     def test_creation(self):
         name = "dataset_name"
-        table_id = "gcp-project.dataset.table_name"
-        dataset = DatasetInfo(name, table_id)
+        full_table_id = "gcp-project.dataset.table_name"
+        dataset = DatasetInfo(name, full_table_id)
         assert dataset.name == name
-        assert dataset.table_id == table_id
+        assert dataset.full_table_id == full_table_id
diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py
--- a/sdk/python/tests/sdk/test_client.py
+++ b/sdk/python/tests/sdk/test_client.py
@@ -198,7 +198,7 @@ def test_create_dataset(self, client, mocker):
         ds = client.create_dataset(fs, start_date, end_date)
 
         assert "dataset_name" == ds.name
-        assert "project.dataset.table" == ds.table_id
+        assert "project.dataset.table" == ds.full_table_id
         mock_trn_stub.CreateDataset.assert_called_once_with(
             DatasetServiceTypes.CreateDatasetRequest(
                 featureSet=fs.proto,
@@ -229,7 +229,7 @@ def test_create_dataset_with_limit(self, client, mocker):
         ds = client.create_dataset(fs, start_date, end_date, limit=limit)
 
         assert "dataset_name" == ds.name
-        assert "project.dataset.table" == ds.table_id
+        assert "project.dataset.table" == ds.full_table_id
         mock_trn_stub.CreateDataset.assert_called_once_with(
             DatasetServiceTypes.CreateDatasetRequest(
                 featureSet=fs.proto,
@@ -263,7 +263,7 @@ def test_create_dataset_with_name_prefix(self, client, mocker):
             fs, start_date, end_date, limit=limit, name_prefix=name_prefix)
 
         assert "dataset_name" == ds.name
-        assert "project.dataset.table" == ds.table_id
+        assert "project.dataset.table" == ds.full_table_id
         mock_dssvc_stub.CreateDataset.assert_called_once_with(
             DatasetServiceTypes.CreateDatasetRequest(
                 featureSet=fs.proto,
@@ -427,9 +427,9 @@ def test_download_dataset_as_file(self, client, mocker):
             table_dlder, "download_table_as_file", return_value=destination)
 
         client._table_downloader = table_dlder
-        table_id = "project.dataset.table"
+        full_table_id = "project.dataset.table"
         staging_location = "gs://gcs_bucket/"
-        dataset = DatasetInfo("mydataset", table_id)
+        dataset = DatasetInfo("mydataset", full_table_id)
 
         result = client.download_dataset(
             dataset,
@@ -439,7 +439,7 @@ def test_download_dataset_as_file(self, client, mocker):
 
         assert result == destination
         table_dlder.download_table_as_file.assert_called_once_with(
-            table_id, destination, staging_location, FileType.CSV)
+            full_table_id, destination, staging_location, FileType.CSV)
 
     def _create_query_features_response(self, entity_name, entities):
         response = QueryFeaturesResponse(entityName=entity_name)
diff --git a/sdk/python/tests/sdk/utils/test_bq_utils.py b/sdk/python/tests/sdk/utils/test_bq_utils.py
--- a/sdk/python/tests/sdk/utils/test_bq_utils.py
+++ b/sdk/python/tests/sdk/utils/test_bq_utils.py
@@ -48,6 +48,23 @@ def test_get_table_name_not_bq():
     with pytest.raises(ValueError, match="storage spec is not BigQuery storage spec"):
         get_table_name(feature_id, storage_spec)
 
+
+@pytest.mark.skipif(
+    os.getenv("SKIP_BIGQUERY_TEST") is not None,
+    reason="SKIP_BIGQUERY_TEST is set in the environment",
+)
+def test_query_to_dataframe():
+    with open(
+        os.path.join(testdata_path, "austin_bikeshare.bikeshare_stations.avro"), "rb"
+    ) as expected_file:
+        avro_reader = fastavro.reader(expected_file)
+        expected = pd.DataFrame.from_records(avro_reader)
+
+    query = "SELECT * FROM `bigquery-public-data.austin_bikeshare.bikeshare_stations`"
+    actual = query_to_dataframe(query)
+    assert expected.equals(actual)
+
+
 @pytest.mark.skipif(
     os.getenv("SKIP_BIGQUERY_TEST") is not None,
     reason="SKIP_BIGQUERY_TEST is set in the environment",
@@ -67,7 +84,7 @@ def test_download_table_as_df(self, mocker):
 
         staging_path = "gs://temp/"
         staging_file_name = "temp_0"
-        table_id = "project_id.dataset_id.table_id"
+        full_table_id = "project_id.dataset_id.table_id"
 
         table_dldr = TableDownloader()
         exp_staging_path = os.path.join(staging_path, staging_file_name)
@@ -75,11 +92,11 @@ def test_download_table_as_df(self, mocker):
         table_dldr._bq = _Mock_BQ_Client()
         mocker.patch.object(table_dldr._bq, "extract_table", return_value=_Job())
 
-        table_dldr.download_table_as_df(table_id, staging_location=staging_path)
+        table_dldr.download_table_as_df(full_table_id, staging_location=staging_path)
 
         assert len(table_dldr._bq.extract_table.call_args_list) == 1
         args, kwargs = table_dldr._bq.extract_table.call_args_list[0]
-        assert args[0].full_table_id == Table.from_string(table_id).full_table_id
+        assert args[0].full_table_id == Table.from_string(full_table_id).full_table_id
         assert args[1] == exp_staging_path
         assert kwargs["job_config"].destination_format == "CSV"
         mocked_gcs_to_df.assert_called_once_with(exp_staging_path)
@@ -97,25 +114,25 @@ def test_download_json(self, mocker):
         self._test_download_file(mocker, FileType.JSON)
 
     def test_download_invalid_staging_url(self):
-        table_id = "project_id.dataset_id.table_id"
+        full_table_id = "project_id.dataset_id.table_id"
         table_dldr = TableDownloader()
         with pytest.raises(
             ValueError, match="staging_uri must be a directory in " "GCS"
         ):
             table_dldr.download_table_as_file(
-                table_id, "/tmp/dst", "/local/directory", FileType.CSV
+                full_table_id, "/tmp/dst", "/local/directory", FileType.CSV
             )
 
         with pytest.raises(
             ValueError, match="staging_uri must be a directory in " "GCS"
         ):
-            table_dldr.download_table_as_df(table_id, "/local/directory")
+            table_dldr.download_table_as_df(full_table_id, "/local/directory")
 
     def _test_download_file(self, mocker, type):
         staging_path = "gs://temp/"
         staging_file_name = "temp_0"
         dst_path = "/tmp/myfile.csv"
-        table_id = "project_id.dataset_id.table_id"
+        full_table_id = "project_id.dataset_id.table_id"
 
         table_dldr = TableDownloader()
         mock_blob = _Blob()
@@ -128,13 +145,13 @@ def _test_download_file(self, mocker, type):
         )
 
         table_dldr.download_table_as_file(
-            table_id, dst_path, staging_location=staging_path, file_type=type
+            full_table_id, dst_path, staging_location=staging_path, file_type=type
         )
 
         exp_staging_path = os.path.join(staging_path, staging_file_name)
         assert len(table_dldr._bq.extract_table.call_args_list) == 1
         args, kwargs = table_dldr._bq.extract_table.call_args_list[0]
-        assert args[0].full_table_id == Table.from_string(table_id).full_table_id
+        assert args[0].full_table_id == Table.from_string(full_table_id).full_table_id
         assert args[1] == exp_staging_path
         assert kwargs["job_config"].destination_format == str(type)