Init iris

huggingface-spaces-iris-monitor/README.md

@@ -0,0 +1,13 @@
+---
+title: Iris Monitoring
+emoji: 💻
+colorFrom: blue
+colorTo: pink
+sdk: gradio
+sdk_version: 3.8.2
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

huggingface-spaces-iris-monitor/app.py

@@ -0,0 +1,31 @@
+import gradio as gr
+from PIL import Image
+import hopsworks
+
+project = hopsworks.login()
+fs = project.get_feature_store()
+
+dataset_api = project.get_dataset_api()
+
+dataset_api.download("Resources/images/latest_iris.png")
+dataset_api.download("Resources/images/actual_iris.png")
+dataset_api.download("Resources/images/df_recent.png")
+dataset_api.download("Resources/images/confusion_matrix.png")
+
+with gr.Blocks() as demo:
+    with gr.Row():
+      with gr.Column():
+          gr.Label("Today's Predicted Image")
+          input_img = gr.Image("latest_iris.png", elem_id="predicted-img")
+      with gr.Column():          
+          gr.Label("Today's Actual Image")
+          input_img = gr.Image("actual_iris.png", elem_id="actual-img")        
+    with gr.Row():
+      with gr.Column():
+          gr.Label("Recent Prediction History")
+          input_img = gr.Image("df_recent.png", elem_id="recent-predictions")
+      with gr.Column():          
+          gr.Label("Confusion Maxtrix with Historical Prediction Performance")
+          input_img = gr.Image("confusion_matrix.png", elem_id="confusion-matrix")        
+
+demo.launch()

huggingface-spaces-iris-monitor/requirements.txt

@@ -0,0 +1 @@
+hopsworks

huggingface-spaces-iris/README.md

@@ -0,0 +1,13 @@
+---
+title: Iris
+emoji: 🐢
+colorFrom: purple
+colorTo: green
+sdk: gradio
+sdk_version: 3.5
+app_file: app.py
+pinned: false
+license: apache-2.0
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

huggingface-spaces-iris/app.py

@@ -0,0 +1,47 @@
+import gradio as gr
+import numpy as np
+from PIL import Image
+import requests
+
+import hopsworks
+import joblib
+
+project = hopsworks.login()
+fs = project.get_feature_store()
+
+
+mr = project.get_model_registry()
+model = mr.get_model("iris_modal", version=1)
+model_dir = model.download()
+model = joblib.load(model_dir + "/iris_model.pkl")
+
+
+def iris(sepal_length, sepal_width, petal_length, petal_width):
+    input_list = []
+    input_list.append(sepal_length)
+    input_list.append(sepal_width)
+    input_list.append(petal_length)
+    input_list.append(petal_width)
+    # 'res' is a list of predictions returned as the label.
+    res = model.predict(np.asarray(input_list).reshape(1, -1)) 
+    # We add '[0]' to the result of the transformed 'res', because 'res' is a list, and we only want 
+    # the first element.
+    flower_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + res[0] + ".png"
+    img = Image.open(requests.get(flower_url, stream=True).raw)            
+    return img
+        
+demo = gr.Interface(
+    fn=iris,
+    title="Iris Flower Predictive Analytics",
+    description="Experiment with sepal/petal lengths/widths to predict which flower it is.",
+    allow_flagging="never",
+    inputs=[
+        gr.inputs.Number(default=1.0, label="sepal length (cm)"),
+        gr.inputs.Number(default=1.0, label="sepal width (cm)"),
+        gr.inputs.Number(default=1.0, label="petal length (cm)"),
+        gr.inputs.Number(default=1.0, label="petal width (cm)"),
+        ],
+    outputs=gr.Image(type="pil"))
+
+demo.launch()
+

huggingface-spaces-iris/requirements.txt

@@ -0,0 +1,3 @@
+hopsworks
+joblib
+scikit-learn

iris-batch-inference-pipeline.py

@@ -0,0 +1,108 @@
+import os
+import modal
+    
+LOCAL=True
+
+if LOCAL == False:
+   stub = modal.Stub()
+   hopsworks_image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"])
+   @stub.function(image=hopsworks_image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("jim-hopsworks-ai"))
+   def f():
+       g()
+
+def g():
+    import pandas as pd
+    import hopsworks
+    import joblib
+    import datetime
+    from PIL import Image
+    from datetime import datetime
+    import dataframe_image as dfi
+    from sklearn.metrics import confusion_matrix
+    from matplotlib import pyplot
+    import seaborn as sns
+    import requests
+
+    project = hopsworks.login()
+    fs = project.get_feature_store()
+    
+    mr = project.get_model_registry()
+    model = mr.get_model("iris_modal", version=1)
+    model_dir = model.download()
+    model = joblib.load(model_dir + "/iris_model.pkl")
+    
+    feature_view = fs.get_feature_view(name="iris_modal", version=1)
+    batch_data = feature_view.get_batch_data()
+    
+    y_pred = model.predict(batch_data)
+    # print(y_pred)
+    flower = y_pred[y_pred.size-1]
+    flower_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + flower + ".png"
+    print("Flower predicted: " + flower)
+    img = Image.open(requests.get(flower_url, stream=True).raw)            
+    img.save("./latest_iris.png")
+    dataset_api = project.get_dataset_api()    
+    dataset_api.upload("./latest_iris.png", "Resources/images", overwrite=True)
+    
+    iris_fg = fs.get_feature_group(name="iris_modal", version=1)
+    df = iris_fg.read()
+    # print(df["variety"])
+    label = df.iloc[-1]["variety"]
+    label_url = "https://raw.githubusercontent.com/featurestoreorg/serverless-ml-course/main/src/01-module/assets/" + label + ".png"
+    print("Flower actual: " + label)
+    img = Image.open(requests.get(label_url, stream=True).raw)            
+    img.save("./actual_iris.png")
+    dataset_api.upload("./actual_iris.png", "Resources/images", overwrite=True)
+    
+    monitor_fg = fs.get_or_create_feature_group(name="iris_predictions",
+                                                version=1,
+                                                primary_key=["datetime"],
+                                                description="Iris flower Prediction/Outcome Monitoring"
+                                                )
+    
+    now = datetime.now().strftime("%m/%d/%Y, %H:%M:%S")
+    data = {
+        'prediction': [flower],
+        'label': [label],
+        'datetime': [now],
+       }
+    monitor_df = pd.DataFrame(data)
+    monitor_fg.insert(monitor_df, write_options={"wait_for_job" : False})
+    
+    history_df = monitor_fg.read()
+    # Add our prediction to the history, as the history_df won't have it - 
+    # the insertion was done asynchronously, so it will take ~1 min to land on App
+    history_df = pd.concat([history_df, monitor_df])
+
+
+    df_recent = history_df.tail(5)
+    dfi.export(df_recent, './df_recent.png', table_conversion = 'matplotlib')
+    dataset_api.upload("./df_recent.png", "Resources/images", overwrite=True)
+    
+    predictions = history_df[['prediction']]
+    labels = history_df[['label']]
+
+    # Only create the confusion matrix when our iris_predictions feature group has examples of all 3 iris flowers
+    print("Number of different flower predictions to date: " + str(predictions.value_counts().count()))
+    if predictions.value_counts().count() == 3:
+        results = confusion_matrix(labels, predictions)
+    
+        df_cm = pd.DataFrame(results, ['True Setosa', 'True Versicolor', 'True Virginica'],
+                             ['Pred Setosa', 'Pred Versicolor', 'Pred Virginica'])
+    
+        cm = sns.heatmap(df_cm, annot=True)
+        fig = cm.get_figure()
+        fig.savefig("./confusion_matrix.png")
+        dataset_api.upload("./confusion_matrix.png", "Resources/images", overwrite=True)
+    else:
+        print("You need 3 different flower predictions to create the confusion matrix.")
+        print("Run the batch inference pipeline more times until you get 3 different iris flower predictions") 
+
+
+if __name__ == "__main__":
+    if LOCAL == True :
+        g()
+    else:
+        with stub.run():
+            f()
+

iris-feature-pipeline-daily.py

@@ -0,0 +1,84 @@
+import os
+import modal
+    
+BACKFILL=False
+LOCAL=False
+
+if LOCAL == False:
+   stub = modal.Stub()
+   image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"]) 
+
+   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("jim-hopsworks-ai"))
+   def f():
+       g()
+
+
+def generate_flower(name, sepal_len_max, sepal_len_min, sepal_width_max, sepal_width_min, 
+                    petal_len_max, petal_len_min, petal_width_max, petal_width_min):
+    """
+    Returns a single iris flower as a single row in a DataFrame
+    """
+    import pandas as pd
+    import random
+
+    df = pd.DataFrame({ "sepal_length": [random.uniform(sepal_len_max, sepal_len_min)],
+                       "sepal_width": [random.uniform(sepal_width_max, sepal_width_min)],
+                       "petal_length": [random.uniform(petal_len_max, petal_len_min)],
+                       "petal_width": [random.uniform(petal_width_max, petal_width_min)]
+                      })
+    df['variety'] = name
+    return df
+
+
+def get_random_iris_flower():
+    """
+    Returns a DataFrame containing one random iris flower
+    """
+    import pandas as pd
+    import random
+
+    virginica_df = generate_flower("Virginica", 8, 5.5, 3.8, 2.2, 7, 4.5, 2.5, 1.4)
+    versicolor_df = generate_flower("Versicolor", 7.5, 4.5, 3.5, 2.1, 3.1, 5.5, 1.8, 1.0)
+    setosa_df =  generate_flower("Setosa", 6, 4.5, 4.5, 2.3, 1.2, 2, 0.7, 0.3)
+
+    # randomly pick one of these 3 and write it to the featurestore
+    pick_random = random.uniform(0,3)
+    if pick_random >= 2:
+        iris_df = virginica_df
+        print("Virginica added")
+    elif pick_random >= 1:
+        iris_df = versicolor_df
+        print("Versicolor added")
+    else:
+        iris_df = setosa_df
+        print("Setosa added")
+
+    return iris_df
+
+
+
+def g():
+    import hopsworks
+    import pandas as pd
+
+    project = hopsworks.login()
+    fs = project.get_feature_store()
+
+    if BACKFILL == True:
+        iris_df = pd.read_csv("https://repo.hops.works/master/hopsworks-tutorials/data/iris.csv")
+    else:
+        iris_df = get_random_iris_flower()
+
+    iris_fg = fs.get_or_create_feature_group(
+        name="iris_modal",
+        version=1,
+        primary_key=["sepal_length","sepal_width","petal_length","petal_width"], 
+        description="Iris flower dataset")
+    iris_fg.insert(iris_df, write_options={"wait_for_job" : False})
+
+if __name__ == "__main__":
+    if LOCAL == True :
+        g()
+    else:
+        with stub.run():
+            f()

iris-feature-pipeline.py

@@ -0,0 +1,33 @@
+import os
+import modal
+    
+LOCAL=False
+
+if LOCAL == False:
+   stub = modal.Stub()
+   image = modal.Image.debian_slim().pip_install(["hopsworks","joblib","seaborn","sklearn","dataframe-image"]) 
+
+   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("jim-hopsworks-ai"))
+   def f():
+       g()
+
+def g():
+    import hopsworks
+    import pandas as pd
+
+    project = hopsworks.login()
+    fs = project.get_feature_store()
+    iris_df = pd.read_csv("https://repo.hops.works/master/hopsworks-tutorials/data/iris.csv")
+    iris_fg = fs.get_or_create_feature_group(
+        name="iris_modal",
+        version=1,
+        primary_key=["sepal_length","sepal_width","petal_length","petal_width"], 
+        description="Iris flower dataset")
+    iris_fg.insert(iris_df, write_options={"wait_for_job" : False})
+
+if __name__ == "__main__":
+    if LOCAL == True :
+        g()
+    else:
+        with stub.run():
+            f()

iris-training-pipeline.py

@@ -0,0 +1,100 @@
+import os
+import modal
+
+LOCAL=True
+
+if LOCAL == False:
+   stub = modal.Stub()
+   image = modal.Image.debian_slim().apt_install(["libgomp1"]).pip_install(["hopsworks", "seaborn", "joblib", "scikit-learn"])
+
+   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("jim-hopsworks-ai"))
+   def f():
+       g()
+
+
+def g():
+    import hopsworks
+    import pandas as pd
+    from sklearn.neighbors import KNeighborsClassifier
+    from sklearn.metrics import accuracy_score
+    from sklearn.metrics import confusion_matrix
+    from sklearn.metrics import classification_report
+    import seaborn as sns
+    from matplotlib import pyplot
+    from hsml.schema import Schema
+    from hsml.model_schema import ModelSchema
+    import joblib
+
+    # You have to set the environment variable 'HOPSWORKS_API_KEY' for login to succeed
+    project = hopsworks.login()
+    # fs is a reference to the Hopsworks Feature Store
+    fs = project.get_feature_store()
+
+    # The feature view is the input set of features for your model. The features can come from different feature groups.    
+    # You can select features from different feature groups and join them together to create a feature view
+    try: 
+        feature_view = fs.get_feature_view(name="iris_modal", version=1)
+    except:
+        iris_fg = fs.get_feature_group(name="iris_modal", version=1)
+        query = iris_fg.select_all()
+        feature_view = fs.create_feature_view(name="iris_modal",
+                                          version=1,
+                                          description="Read from Iris flower dataset",
+                                          labels=["variety"],
+                                          query=query)    
+
+    # You can read training data, randomly split into train/test sets of features (X) and labels (y)        
+    X_train, X_test, y_train, y_test = feature_view.train_test_split(0.2)
+
+    # Train our model with the Scikit-learn K-nearest-neighbors algorithm using our features (X_train) and labels (y_train)
+    model = KNeighborsClassifier(n_neighbors=2)
+    model.fit(X_train, y_train.values.ravel())
+
+    # Evaluate model performance using the features from the test set (X_test)
+    y_pred = model.predict(X_test)
+
+    # Compare predictions (y_pred) with the labels in the test set (y_test)
+    metrics = classification_report(y_test, y_pred, output_dict=True)
+    results = confusion_matrix(y_test, y_pred)
+
+    # Create the confusion matrix as a figure, we will later store it as a PNG image file
+    df_cm = pd.DataFrame(results, ['True Setosa', 'True Versicolor', 'True Virginica'],
+                         ['Pred Setosa', 'Pred Versicolor', 'Pred Virginica'])
+    cm = sns.heatmap(df_cm, annot=True)
+    fig = cm.get_figure()
+
+    # We will now upload our model to the Hopsworks Model Registry. First get an object for the model registry.
+    mr = project.get_model_registry()
+    
+    # The contents of the 'iris_model' directory will be saved to the model registry. Create the dir, first.
+    model_dir="iris_model"
+    if os.path.isdir(model_dir) == False:
+        os.mkdir(model_dir)
+
+    # Save both our model and the confusion matrix to 'model_dir', whose contents will be uploaded to the model registry
+    joblib.dump(model, model_dir + "/iris_model.pkl")
+    fig.savefig(model_dir + "/confusion_matrix.png")    
+
+
+    # Specify the schema of the model's input/output using the features (X_train) and labels (y_train)
+    input_schema = Schema(X_train)
+    output_schema = Schema(y_train)
+    model_schema = ModelSchema(input_schema, output_schema)
+
+    # Create an entry in the model registry that includes the model's name, desc, metrics
+    iris_model = mr.python.create_model(
+        name="iris_modal", 
+        metrics={"accuracy" : metrics['accuracy']},
+        model_schema=model_schema,
+        description="Iris Flower Predictor"
+    )
+    
+    # Upload the model to the model registry, including all files in 'model_dir'
+    iris_model.save(model_dir)
+    
+if __name__ == "__main__":
+    if LOCAL == True :
+        g()
+    else:
+        with stub.run():
+            f()

requirements.txt

@@ -0,0 +1,5 @@
+hopsworks
+joblib
+scikit-learn
+seaborn
+dataframe-image