Repository: feast-dev/feast

Issue:
Update Prow for Feast 0.3 to run tests for PRs on GitHub
Adding this issue to track Prow test running for PRs


Discussion:

Resolving patch:
diff --git a/sdk/python/feast/feature_set.py b/sdk/python/feast/feature_set.py
--- a/sdk/python/feast/feature_set.py
+++ b/sdk/python/feast/feature_set.py
@@ -69,7 +69,7 @@ def __init__(
         if entities is not None:
             self.entities = entities
         if source is None:
-            self._source = KafkaSource()
+            self._source = None
         else:
             self._source = source
         self._max_age = max_age
@@ -504,7 +504,7 @@ def to_proto(self) -> FeatureSetSpecProto:
             name=self.name,
             version=self.version,
             max_age=self.max_age,
-            source=self.source.to_proto(),
+            source=self.source.to_proto() if self.source is not None else None,
             features=[
                 field.to_proto()
                 for field in self._fields.values()
diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py
--- a/sdk/python/feast/type_map.py
+++ b/sdk/python/feast/type_map.py
@@ -106,7 +106,7 @@ def dtype_to_value_type(dtype):
 # TODO: to pass test_importer
 def pandas_dtype_to_feast_value_type(dtype: pd.DataFrame.dtypes) -> ValueType:
     type_map = {
-        "float64": ValueType.FLOAT,
+        "float64": ValueType.DOUBLE,
         "float32": ValueType.FLOAT,
         "int64": ValueType.INT64,
         "uint64": ValueType.INT64,
@@ -247,7 +247,7 @@ def pd_value_to_proto_value(feast_value_type, value) -> ProtoValue:
             return ProtoValue(float_val=float(value))
         elif feast_value_type == ValueType.DOUBLE:
             assert type(value) is float
-            return ProtoValue(float_val=value)
+            return ProtoValue(double_val=value)
         elif feast_value_type == ValueType.STRING:
             return ProtoValue(string_val=str(value))
         elif feast_value_type == ValueType.BYTES:
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -36,7 +36,7 @@
     "grpcio==1.*",
     "pandas==0.*",
     "pandavro==1.5.1",
-    "protobuf==3.*",
+    "protobuf==3.10.*",
     "PyYAML==5.1.2",
     "fastavro==0.*",
     "kafka-python==1.4.*",


Test patch:
diff --git a/.prow/scripts/install_feast_and_run_e2e_test.sh b/.prow/scripts/install_feast_and_run_e2e_test.sh
deleted file mode 100755
--- a/.prow/scripts/install_feast_and_run_e2e_test.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-echo "============================================================"
-echo "Installing Feast Release"
-echo "============================================================"
-
-helm install --name ${FEAST_RELEASE_NAME} --wait --timeout 210 ${FEAST_HOME}/charts/feast -f integration-tests/feast-helm-values.yaml
-
-echo "============================================================"
-echo "Testing Batch Import"
-echo "============================================================"
-
-cd ${FEAST_HOME}/integration-tests/testdata
-
-feast apply entity entity_specs/entity_1.yaml
-feast apply feature feature_specs/entity_1*.yaml
-feast jobs run import_specs/batch_from_gcs.yaml --wait
-
-cd $FEAST_HOME/integration-tests
-
-python -m testutils.validate_feature_values \
-  --entity_spec_file=testdata/entity_specs/entity_1.yaml \
-  --feature_spec_files=testdata/feature_specs/entity_1*.yaml \
-  --expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \
-  --expected-serving-values-file=testdata/feature_values/serving_1.csv \
-  --bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \
-  --feast-serving-url=${FEAST_SERVING_URL}
-
-echo "============================================================"
-echo "Testing Streaming Import"
-echo "============================================================"
-
-cd $FEAST_HOME/integration-tests/testdata
-
-feast apply entity entity_specs/entity_2.yaml
-feast apply feature feature_specs/entity_2*.yaml
-feast jobs run import_specs/stream_from_kafka.yaml &
-
-IMPORT_JOB_PID=$!
-sleep 20
-
-cd $FEAST_HOME/integration-tests
-
-python -m testutils.kafka_producer \
-  --bootstrap_servers=$KAFKA_BROKERS \
-  --topic=$KAFKA_TOPICS \
-  --entity_spec_file=testdata/entity_specs/entity_2.yaml \
-  --feature_spec_files=testdata/feature_specs/entity_2*.yaml \
-  --feature_values_file=testdata/feature_values/ingestion_2.csv
-sleep 20
-
-python -m testutils.validate_feature_values \
-  --entity_spec_file=testdata/entity_specs/entity_2.yaml \
-  --feature_spec_files=testdata/feature_specs/entity_2*.yaml \
-  --expected-serving-values-file=testdata/feature_values/serving_2.csv \
-  --feast-serving-url=$FEAST_SERVING_URL
-
-kill -9 ${IMPORT_JOB_PID}
diff --git a/.prow/scripts/install_test_tools.sh b/.prow/scripts/install_test_tools.sh
deleted file mode 100755
--- a/.prow/scripts/install_test_tools.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-# This script installs the following Feast test utilities:
-# ============================================================
-# - gettext package so we can use envsubst command to provide values to helm template file
-# - Python 3.6 because Feast requires Python version 3.6 and above
-# - Golang if we need to build Feast CLI from source
-# - Helm if we want to install Feast release
-
-apt-get -qq update 
-apt-get -y install curl wget gettext &> /dev/null
-
-curl -s https://repo.continuum.io/miniconda/Miniconda3-4.5.12-Linux-x86_64.sh -o /tmp/miniconda.sh
-bash /tmp/miniconda.sh -b -p /miniconda &> /dev/null
-export PATH=/miniconda/bin:$PATH
-
-wget -qO- https://dl.google.com/go/go1.12.5.linux-amd64.tar.gz | tar xzf -
-mv go /usr/local/
-export PATH=/usr/local/go/bin:$PATH
-export GO111MODULE=on
-
-wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz
-mv linux-amd64/helm /usr/local/bin/helm
diff --git a/.prow/scripts/prepare_integration_test.sh b/.prow/scripts/prepare_integration_test.sh
deleted file mode 100755
--- a/.prow/scripts/prepare_integration_test.sh
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-usage()
-{
-    echo "usage: prepare_integration_test.sh [--skip-build true]"
-}
-
-while [ "$1" != "" ]; do
-  case "$1" in
-      --skip-build )       SKIP_BUILD=true;              shift;;
-      * )                  usage; exit 1
-  esac
-  shift
-done
-
-# Authenticate to Google Cloud and GKE
-# ============================================================
-GOOGLE_PROJECT_ID=kf-feast
-KUBE_CLUSTER_NAME=primary-test-cluster
-KUBE_CLUSTER_ZONE=us-central1-a
-KEY_FILE=/etc/service-account/service-account.json
-
-gcloud -q auth activate-service-account --key-file=${KEY_FILE}
-gcloud -q auth configure-docker
-gcloud -q config set project ${GOOGLE_PROJECT_ID}
-gcloud -q container clusters get-credentials ${KUBE_CLUSTER_NAME} --zone ${KUBE_CLUSTER_ZONE} --project ${GOOGLE_PROJECT_ID}
-export GOOGLE_APPLICATION_CREDENTIALS=${KEY_FILE}
-
-# Install Python 3.6, Golang 1.12, Helm and Feast SDK
-# ============================================================
-. .prow/scripts/install_test_tools.sh
-. .prow/scripts/install_feast_sdk.sh
-.prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir ${FEAST_HOME}
-
-# Prepare Feast test data and config
-# ============================================================
-
-bq -q mk --dataset ${FEAST_WAREHOUSE_DATASET}
-gsutil -q cp ${FEAST_HOME}/integration-tests/testdata/feature_values/ingestion_1.csv ${BATCH_IMPORT_DATA_GCS_PATH}
-
-BUILD_ID=${BUILD_ID:0:5}
-envsubst < integration-tests/feast-helm-values.yaml.template > integration-tests/feast-helm-values.yaml
-cd ${FEAST_HOME}/integration-tests/testdata/import_specs
-envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml
-envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml
-
-if [[ ! ${SKIP_BUILD} ]]; then
-
-echo "============================================================"
-echo "Building Feast for Testing"
-echo "============================================================"
-cd ${FEAST_HOME}
-docker build -t us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} -f Dockerfiles/core/Dockerfile . &
-docker build -t us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} -f Dockerfiles/serving/Dockerfile . &
-wait
-docker push us.gcr.io/kf-feast/feast-core:${FEAST_IMAGE_TAG} &
-docker push us.gcr.io/kf-feast/feast-serving:${FEAST_IMAGE_TAG} &
-wait
-
-fi
-
-# Switch back context to original directory
-set +ex
-cd ${FEAST_HOME}
\ No newline at end of file
diff --git a/.prow/scripts/run_unit_test.sh b/.prow/scripts/run_unit_test.sh
deleted file mode 100755
--- a/.prow/scripts/run_unit_test.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-
-# This script will run unit test for a specific Feast component:
-# - core, ingestion, serving or cli
-#
-# This script includes the pre and post test scripts, such as
-# - downloading maven cache repository
-# - saving the test output report so it can be viewed with Spyglass in Prow
-
-# Bucket in GCS used for running unit tests, when the unit tests need an 
-# actual running GCS (e.g. because there is no existing mock implementation of the function to test)
-TEST_BUCKET=feast-templocation-kf-feast
-
-usage()
-{
-    echo "usage: run_unit_test.sh
-    --component {core, ingestion, serving, cli}"
-}
-
-while [ "$1" != "" ]; do
-  case "$1" in
-      --component )       COMPONENT="$2";    shift;;
-      * )                 usage; exit 1
-  esac
-  shift
-done
-
-if [[ ! ${COMPONENT} ]]; then 
-  usage; exit 1; 
-fi
-
-. .prow/scripts/install_google_cloud_sdk.sh
-
-if [[ ${COMPONENT} == "core" ]] || [[ ${COMPONENT} == "ingestion" ]] || [[ ${COMPONENT} == "serving" ]]; then
-
-    .prow/scripts/prepare_maven_cache.sh --archive-uri gs://feast-templocation-kf-feast/.m2.tar --output-dir /root/
-    mvn --projects ${COMPONENT} -Dtestbucket=feast-templocation-kf-feast test
-    TEST_EXIT_CODE=$?
-    cp -r ${COMPONENT}/target/surefire-reports /logs/artifacts/surefire-reports
-
-elif [[ ${COMPONENT} == "cli" ]]; then
-
-    # https://stackoverflow.com/questions/6871859/piping-command-output-to-tee-but-also-save-exit-code-of-command
-    set -o pipefail
-
-    go get -u github.com/jstemmer/go-junit-report
-    go test -v ./cli/feast/... 2>&1 | tee test_output
-    TEST_EXIT_CODE=$?
-    cat test_output | ${GOPATH}/bin/go-junit-report > ${ARTIFACTS}/unittest-cli-report.xml
-
-elif [[ ${COMPONENT} == "python-sdk" ]]; then
-
-    cd sdk/python
-    pip install -r requirements-test.txt
-    pip install .
-    pytest ./tests --junitxml=${ARTIFACTS}/unittest-pythonsdk-report.xml
-    TEST_EXIT_CODE=$?
-
-else
-    usage; exit 1
-fi
-
-exit ${TEST_EXIT_CODE}
diff --git a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
--- a/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
+++ b/core/src/test/java/feast/core/job/direct/DirectRunnerJobManagerTest.java
@@ -73,6 +73,7 @@ public void shouldStartDirectJobAndRegisterPipelineResult() throws IOException {
     expectedPipelineOptions.setAppName("DirectRunnerJobManager");
     expectedPipelineOptions.setRunner(DirectRunner.class);
     expectedPipelineOptions.setBlockOnRun(false);
+    expectedPipelineOptions.setProject("");
     expectedPipelineOptions.setStoreJson(Lists.newArrayList(printer.print(store)));
     expectedPipelineOptions
         .setFeatureSetSpecJson(Lists.newArrayList(printer.print(featureSetSpec)));
diff --git a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
--- a/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
+++ b/ingestion/src/test/java/feast/ingestion/ImportJobTest.java
@@ -116,6 +116,7 @@ public void runPipeline_ShouldWriteToRedisCorrectlyGivenValidSpecAndFeatureRow()
     options.setStoreJson(
         Collections.singletonList(
             JsonFormat.printer().omittingInsignificantWhitespace().print(redis)));
+    options.setProject("");
     options.setBlockOnRun(false);
 
     int inputSize = 4096;
diff --git a/ingestion/src/test/java/feast/test/TestUtil.java b/ingestion/src/test/java/feast/test/TestUtil.java
--- a/ingestion/src/test/java/feast/test/TestUtil.java
+++ b/ingestion/src/test/java/feast/test/TestUtil.java
@@ -92,7 +92,11 @@ public static void start(String kafkaHost, int kafkaPort, short kafkaReplication
 
     public static void stop() {
       if (server != null) {
-        server.shutdown();
+        try {
+          server.shutdown();
+        } catch (Exception e) {
+          e.printStackTrace();
+        }
       }
     }
   }
diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py
--- a/tests/e2e/conftest.py
+++ b/tests/e2e/conftest.py
@@ -1,4 +1,4 @@
 def pytest_addoption(parser):
     parser.addoption("--core_url", action="store", default="localhost:6565")
-    parser.addoption("--serving_url", action="store", default="localhost:6565")
-    parser.addoption("--allow_dirty", action="store", default="false")
+    parser.addoption("--serving_url", action="store", default="localhost:6566")
+    parser.addoption("--allow_dirty", action="store", default="true")
diff --git a/tests/e2e/test_e2e.py b/tests/e2e/test_e2e.py
--- a/tests/e2e/test_e2e.py
+++ b/tests/e2e/test_e2e.py
@@ -71,8 +71,19 @@ def test_basic(client):
         # Register feature set
         client.apply(cust_trans_fs)
 
+        # Feast Core needs some time to fully commit the FeatureSet applied
+        # when there is no existing job yet for the Featureset
+        time.sleep(15)
         cust_trans_fs = client.get_feature_set(name="customer_transactions", version=1)
 
+        if cust_trans_fs is None:
+            raise Exception(
+                "Client cannot retrieve 'customer_transactions' FeatureSet "
+                "after registration. Either Feast Core does not save the "
+                "FeatureSet correctly or the client needs to wait longer for FeatureSet "
+                "to be committed."
+            )
+
     offset = random.randint(1000, 100000)  # ensure a unique key space is used
     customer_data = pd.DataFrame(
         {
@@ -88,6 +99,8 @@ def test_basic(client):
 
     # Poll serving for feature values until the correct values are returned
     while True:
+        time.sleep(1)
+
         response = client.get_online_features(
             entity_rows=[
                 GetOnlineFeaturesRequest.EntityRow(
@@ -103,8 +116,8 @@ def test_basic(client):
                 "customer_transactions:1:total_transactions",
             ],
         )  # type: GetOnlineFeaturesResponse
+
         if response is None:
-            time.sleep(1)
             continue
 
         returned_daily_transactions = float(
@@ -124,7 +137,7 @@ def test_basic(client):
 
 @pytest.mark.timeout(300)
 def test_all_types(client):
-    all_types_fs = client.get_feature_set(name="all_types", version="1")
+    all_types_fs = client.get_feature_set(name="all_types", version=1)
 
     if all_types_fs is None:
         # Register new feature set if it doesnt exist
@@ -152,7 +165,19 @@ def test_all_types(client):
 
         # Register feature set
         client.apply(all_types_fs)
-        all_types_fs = client.get_feature_set(name="all_types", version="1")
+
+        # Feast Core needs some time to fully commit the FeatureSet applied
+        # when there is no existing job yet for the Featureset
+        time.sleep(10)
+        all_types_fs = client.get_feature_set(name="all_types", version=1)
+
+        if all_types_fs is None:
+            raise Exception(
+                "Client cannot retrieve 'all_types_fs' FeatureSet "
+                "after registration. Either Feast Core does not save the "
+                "FeatureSet correctly or the client needs to wait longer for FeatureSet "
+                "to be committed."
+            )
 
     all_types_df = pd.DataFrame(
         {
@@ -205,9 +230,12 @@ def test_all_types(client):
 
     # Ingest user embedding data
     all_types_fs.ingest(dataframe=all_types_df)
+    time.sleep(3)
 
     # Poll serving for feature values until the correct values are returned
     while True:
+        time.sleep(1)
+
         response = client.get_online_features(
             entity_rows=[
                 GetOnlineFeaturesRequest.EntityRow(
@@ -233,7 +261,6 @@ def test_all_types(client):
         )  # type: GetOnlineFeaturesResponse
 
         if response is None:
-            time.sleep(1)
             continue
 
         returned_float_list = (
@@ -268,10 +295,21 @@ def test_large_volume(client):
         # Register feature set
         client.apply(cust_trans_fs)
 
+        # Feast Core needs some time to fully commit the FeatureSet applied
+        # when there is no existing job yet for the Featureset
+        time.sleep(10)
         cust_trans_fs = client.get_feature_set(
             name="customer_transactions_large", version=1
         )
 
+        if cust_trans_fs is None:
+            raise Exception(
+                "Client cannot retrieve 'customer_transactions' FeatureSet "
+                "after registration. Either Feast Core does not save the "
+                "FeatureSet correctly or the client needs to wait longer for FeatureSet "
+                "to be committed."
+            )
+
     offset = random.randint(1000000, 10000000)  # ensure a unique key space
     customer_data = pd.DataFrame(
         {
@@ -289,6 +327,8 @@ def test_large_volume(client):
 
     # Poll serving for feature values until the correct values are returned
     while True:
+        time.sleep(1)
+
         response = client.get_online_features(
             entity_rows=[
                 GetOnlineFeaturesRequest.EntityRow(
@@ -306,7 +346,6 @@ def test_large_volume(client):
         )  # type: GetOnlineFeaturesResponse
 
         if response is None:
-            time.sleep(1)
             continue
 
         returned_daily_transactions = float(