titanic_training_pipeline.py

import os
import modal

LOCAL=True

if LOCAL == False:
   stub = modal.Stub()
   image = modal.Image.debian_slim().apt_install(["libgomp1"]).pip_install(["hopsworks", "seaborn", "joblib", "scikit-learn"])

   @stub.function(image=image, schedule=modal.Period(days=1), secret=modal.Secret.from_name("HOPSWORKS_API_KEY"))
   def f():
       g()


def g():
    import hopsworks
    import pandas as pd
    from sklearn.neighbors import KNeighborsClassifier
    from sklearn.linear_model import LogisticRegression
    from sklearn.metrics import accuracy_score
    from sklearn.metrics import confusion_matrix
    from sklearn.metrics import classification_report
    import seaborn as sns
    from matplotlib import pyplot
    from hsml.schema import Schema
    from hsml.model_schema import ModelSchema
    import joblib

    # You have to set the environment variable 'HOPSWORKS_API_KEY' for login to succeed
    # 您必须设置环境变量 'HOPSWORKS_API_KEY' 才能成功登录
    project = hopsworks.login()
    # fs is a reference to the Hopsworks Feature Store
    # fs 是对 Hopsworks Feature Store 的引用
    fs = project.get_feature_store()

    # The feature view is the input set of features for your model. The features can come from different feature groups.  
    # feature view 是模型的输入特征集。 这些特征可以来自不同的特征组。
    # You can select features from different feature groups and join them together to create a feature view
    # 您可以从不同的特征组中选择特征并将它们连接在一起以创建 feature view 
    try: 
        feature_view = fs.get_feature_view(name="titanic_modal", version=1)
    except:
        titanic_fg = fs.get_feature_group(name="titanic_modal", version=1)
        query = titanic_fg.select_all()
        feature_view = fs.create_feature_view(name="titanic_modal",
                                          version=1,
                                          description="Read from Titanic survival dataset",
                                          labels=["Survived"],
                                          query=query)    

    # You can read training data, randomly split into train/test sets of features (X) and labels (y) 
    # 你可以读取训练数据，随机分成特征（X）和标签（y）的训练/测试集         
    X_train, X_test, y_train, y_test = feature_view.train_test_split(0.2)




################################# K-Neighbor算法 ##############################
    # Train our model with the Scikit-learn K-nearest-neighbors algorithm using our features (X_train) and labels (y_train)
    # # 使用我们的特征 (X_train) 和标签 (y_train) 使用 Scikit-learn K 最近邻算法训练我们的模型
    '''
    model = KNeighborsClassifier(n_neighbors=20)
    # n_neighbors: 默认情况下kneighbors查询使用的邻居数。就是k-NN的k的值，选取最近的k个点。
    model.fit(X_train, y_train.values.ravel())

    # Evaluate model performance using the features from the test set (X_test)
    # 使用测试集 (X_test) 中的特征评估模型性能
    y_pred = model.predict(X_test)
    '''
################################# K-Neighbor算法 ##############################


############################ Logistic Regression 算法 #########################
    model = LogisticRegression()
    model.fit(X_train, y_train.values.ravel())
    y_pred = model.predict(X_test)
############################ Logistic Regression 算法 #########################

    # Compare predictions (y_pred) with the labels in the test set (y_test)
    # 将预测 (y_pred) 与测试集中的标签 (y_test) 进行比较
    metrics = classification_report(y_test, y_pred, output_dict=True)
    results = confusion_matrix(y_test, y_pred)

    # Create the confusion matrix as a figure, we will later store it as a PNG image file
    # 将混淆矩阵创建为图形，稍后我们将其存储为 PNG 图像文件
    df_cm = pd.DataFrame(results, ['True survived', 'True died'],
                         ['Pred survived', 'Pred died'])
    cm = sns.heatmap(df_cm, annot=True)
    fig = cm.get_figure()

    # We will now upload our model to the Hopsworks Model Registry. First get an object for the model registry.
    mr = project.get_model_registry()
    
    # The contents of the 'titanic_model' directory will be saved to the model registry. Create the dir, first.
    model_dir="titanic_model"
    if os.path.isdir(model_dir) == False:
        os.mkdir(model_dir)

    # We will now upload our model to the Hopsworks Model Registry. First get an object for the model registry.
    # 我们现在将我们的模型上传到 Hopsworks 模型注册表。 首先为模型注册表获取一个对象。
    joblib.dump(model, model_dir + "/titanic_model.pkl")
    fig.savefig(model_dir + "/confusion_matrix.png")    


    # Specify the schema of the model's input/output using the features (X_train) and labels (y_train)
    # 使用特征 (X_train) 和标签 (y_train) 指定模型输入/输出的模式
    input_schema = Schema(X_train)
    output_schema = Schema(y_train)
    model_schema = ModelSchema(input_schema, output_schema)
    
    # Create an entry in the model registry that includes the model's name, desc, metrics
    # 在模型注册表中创建一个条目，其中包括模型的名称、desc、指标
    titanic_model = mr.python.create_model(
        name="titanic_modal", 
      metrics={"accuracy" : metrics['accuracy']},
        model_schema=model_schema,
        description="Titanic survival Predictor"
    )
     
    # Upload the model to the model registry, including all files in 'model_dir'
    # 将模型上传到模型注册表，包括“model_dir”中的所有文件
    titanic_model.save(model_dir)
    
if __name__ == "__main__":
    if LOCAL == True :
        g()
    else:
        with stub.run():
            f()