I am working on the file "Code/1_data_ingestion.ipynb".
The first task is:
The figure shows one month worth of telemetry sensor data for one machine. Each sensor is shown in it's own panel.
Next, we convert the errors data to a Spark dataframe, and verify the data types have converted correctly.
Can you write Python code for it?

# The data was read in using a Pandas data frame. We'll convert
# it to pyspark to ensure it is in a Spark usable form for later
# manipulations.
# This line takes about 9.5 minutes to run.
telemetry_spark = spark.createDataFrame(telemetry, verifySchema=False)

# We no longer need the pandas dataframes, so we can release that memory.
del telemetry
del plt_data
del plot_df

# Check data type conversions.
telemetry_spark.printSchema()


I am working on the file "Code/1_data_ingestion.ipynb".
The first task is:
The following histogram details the distribution of the failure records obtained from failure log. This log was built originally from component replacements the maintenance log file.
Can you write Python code for it?

# Plot failures
ggplot(aes(x="failure"), failures) + geom_bar(fill="blue", color="black")


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Errors data set
Load the errors data set from your Azure blob.
Can you write Python code for it?

if not os.path.exists(ERROR_LOCAL_DIRECT):
    os.makedirs(ERROR_LOCAL_DIRECT)
    print('DONE creating a local directory!')

# Connect to blob storage container
for blob in az_blob_service.list_blobs(CONTAINER_NAME):
    if ERROR_DATA in blob.name:
        local_file = os.path.join(ERROR_LOCAL_DIRECT, os.path.basename(blob.name))
        az_blob_service.get_blob_to_path(CONTAINER_NAME, blob.name, local_file)

# Read in the data
errors = spark.read.parquet(ERROR_LOCAL_DIRECT)

print(errors.count())
errors.printSchema()
errors.limit(5).toPandas().head()


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Failures data set
Load the failures data set from your Azure blob.
Can you write Python code for it?

# create a local path  to store the data.
if not os.path.exists(FAILURES_LOCAL_DIRECT):
    os.makedirs(FAILURES_LOCAL_DIRECT)
    print('DONE creating a local directory!')

# download the entire parquet result folder to local path for a new run
for blob in az_blob_service.list_blobs(CONTAINER_NAME):
    if FAILURE_DATA in blob.name:
        local_file = os.path.join(FAILURES_LOCAL_DIRECT, os.path.basename(blob.name))
        az_blob_service.get_blob_to_path(CONTAINER_NAME, blob.name, local_file)

failures = spark.read.parquet(FAILURES_LOCAL_DIRECT).dropDuplicates(['machineID', 'datetime'])

print(failures.count())
failures.limit(5).toPandas().head()


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Machines data set
Now, we load the machines data set from your Azure blob.
Can you write Python code for it?

# create a local path  to store the data.
if not os.path.exists(MACH_LOCAL_DIRECT):
    os.makedirs(MACH_LOCAL_DIRECT)
    print('DONE creating a local directory!')

# Connect to blob storage container
for blob in az_blob_service.list_blobs(CONTAINER_NAME):
    if MACH_DATA in blob.name:
        local_file = os.path.join(MACH_LOCAL_DIRECT, os.path.basename(blob.name))
        az_blob_service.get_blob_to_path(CONTAINER_NAME, blob.name, local_file)

# Read in the data
machines = spark.read.parquet(MACH_LOCAL_DIRECT)

print(machines.count())
machines.limit(5).toPandas().head()


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Next, we convert the labels from text to numeric values. In the end, this will transform the problem from boolean of 'healthy'/'impending failure' to a multiclass 'healthy'/'component n impending failure'.
Can you write Python code for it?

# map the failure data to final feature matrix
labeled_features = (final_feat.join(fail_diff,
                                    ((final_feat['machineID'] == fail_diff['machineID'])
                                     & (final_feat['dt_truncated'] == fail_diff['dt_truncated'])), "left")
                    .drop(fail_diff.machineID).drop(fail_diff.dt_truncated)
                    .withColumn('failure', F.when(col('failure') == "comp1", 1.0).otherwise(col('failure')))
                    .withColumn('failure', F.when(col('failure') == "comp2", 2.0).otherwise(col('failure')))
                    .withColumn('failure', F.when(col('failure') == "comp3", 3.0).otherwise(col('failure')))
                    .withColumn('failure', F.when(col('failure') == "comp4", 4.0).otherwise(col('failure'))))

labeled_features = (labeled_features.withColumn("failure",
                                                labeled_features.failure.cast(DoubleType()))
                    .fillna(0))

print(labeled_features.count())
labeled_features.limit(10).toPandas().head(10)


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Now, we join the four component replacement tables together. Once joined, align the data by tumbling the average across 12 hour observation windows.
Can you write Python code for it?

# Join component 3 and 4
comp3_4 = (comp3.join(comp4, ((comp3['machineID'] == comp4['machineID'])
                              & (comp3['datetime_tel'] == comp4['datetime_tel'])), "left")
           .drop(comp4.machineID).drop(comp4.datetime_tel))

# Join component 2 to 3 and 4
comp2_3_4 = (comp2.join(comp3_4, ((comp2['machineID'] == comp3_4['machineID'])
                                  & (comp2['datetime_tel'] == comp3_4['datetime_tel'])), "left")
             .drop(comp3_4.machineID).drop(comp3_4.datetime_tel))

# Join component 1 to 2, 3 and 4
comps_feat = (comp1.join(comp2_3_4, ((comp1['machineID'] == comp2_3_4['machineID'])
                                      & (comp1['datetime_tel'] == comp2_3_4['datetime_tel'])), "left")
               .drop(comp2_3_4.machineID).drop(comp2_3_4.datetime_tel)
               .groupBy("machineID", "datetime_tel")
               .agg(F.max('sincelastcomp1').alias('sincelastcomp1'),
                    F.max('sincelastcomp2').alias('sincelastcomp2'),
                    F.max('sincelastcomp3').alias('sincelastcomp3'),
                    F.max('sincelastcomp4').alias('sincelastcomp4'))
               .fillna(0))

# Choose the time_val hour timestamps to align the data
dt_truncated = ((round(unix_timestamp(col("datetime_tel")) / time_val) * time_val).cast("timestamp"))

# Collect data
maint_feat = (comps_feat.withColumn("dt_truncated", dt_truncated)
              .groupBy("machineID","dt_truncated")
              .agg(F.mean('sincelastcomp1').alias('comp1sum'),
                   F.mean('sincelastcomp2').alias('comp2sum'),
                   F.mean('sincelastcomp3').alias('comp3sum'),
                   F.mean('sincelastcomp4').alias('comp4sum')))

print(maint_feat.count())
maint_feat.limit(10).toPandas().head(10)


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Replacement features are then created by tracking the number of days between each component replacement. We'll repeat these calculations for each of the four components and join them together into a maintenance feature table.
First component number 1 (comp1):
Can you write Python code for it?

# We want to align the component information on telemetry features timestamps.
telemetry_times = (telemetry_feat.select(telemetry_feat.machineID, telemetry_feat.dt_truncated)
                   .withColumnRenamed('dt_truncated','datetime_tel'))

# Grab component 1 records
maint_comp1 = (maint_replace.where(col("comp1sum") == '1').withColumnRenamed('datetime','datetime_maint')
               .drop('comp2sum', 'comp3sum', 'comp4sum'))

# Within each machine, get the last replacement date for each timepoint
maint_tel_comp1 = (telemetry_times.join(maint_comp1,
                                        ((telemetry_times ['machineID']== maint_comp1['machineID'])
                                         & (telemetry_times ['datetime_tel'] > maint_comp1['datetime_maint'])
                                         & ( maint_comp1['comp1sum'] == '1')))
                   .drop(maint_comp1.machineID))

# Calculate the number of days between replacements
comp1 = (maint_tel_comp1.withColumn("sincelastcomp1",
                                    datediff(maint_tel_comp1.datetime_tel, maint_tel_comp1.datetime_maint))
         .drop(maint_tel_comp1.datetime_maint).drop(maint_tel_comp1.comp1sum))

print(comp1.count())
comp1.filter(comp1.machineID == '625').orderBy(comp1.datetime_tel).limit(20).toPandas().head(20)


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Telemetry
Load the telemetry data set from your Azure blob.
Can you write Python code for it?

# create a local path  to store the data.
if not os.path.exists(TELEMETRY_LOCAL_DIRECT):
    os.makedirs(TELEMETRY_LOCAL_DIRECT)
    print('DONE creating a local directory!')

# Connect to blob storage container
for blob in az_blob_service.list_blobs(CONTAINER_NAME):
    if TELEMETRY_DATA in blob.name:
        local_file = os.path.join(TELEMETRY_LOCAL_DIRECT, os.path.basename(blob.name))
        az_blob_service.get_blob_to_path(CONTAINER_NAME, blob.name, local_file)

# Read in the data
telemetry = spark.read.parquet(TELEMETRY_LOCAL_DIRECT)

print(telemetry.count())
telemetry.limit(5).toPandas().head()


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Then component 2 (comp2):
Can you write Python code for it?

# Grab component 2 records
maint_comp2 = (maint_replace.where(col("comp2sum") == '1').withColumnRenamed('datetime','datetime_maint')
               .drop('comp1sum', 'comp3sum', 'comp4sum'))

# Within each machine, get the last replacement date for each timepoint
maint_tel_comp2 = (telemetry_times.join(maint_comp2,
                                        ((telemetry_times ['machineID']== maint_comp2['machineID'])
                                         & (telemetry_times ['datetime_tel'] > maint_comp2['datetime_maint'])
                                         & ( maint_comp2['comp2sum'] == '1')))
                   .drop(maint_comp2.machineID))

# Calculate the number of days between replacements
comp2 = (maint_tel_comp2.withColumn("sincelastcomp2",
                                    datediff(maint_tel_comp2.datetime_tel, maint_tel_comp2.datetime_maint))
         .drop(maint_tel_comp2.datetime_maint).drop(maint_tel_comp2.comp2sum))

print(comp2.count())
comp2.filter(comp2.machineID == '625').orderBy(comp2.datetime_tel).limit(5).toPandas().head(5)


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
Then component 3 (comp3):
Can you write Python code for it?

# Grab component 3 records
maint_comp3 = (maint_replace.where(col("comp3sum") == '1').withColumnRenamed('datetime','datetime_maint')
               .drop('comp1sum', 'comp2sum', 'comp4sum'))

# Within each machine, get the last replacement date for each timepoint
maint_tel_comp3 = (telemetry_times.join(maint_comp3, ((telemetry_times ['machineID']==maint_comp3['machineID'])
                                                      & (telemetry_times ['datetime_tel'] > maint_comp3['datetime_maint'])
                                                      & ( maint_comp3['comp3sum'] == '1')))
                   .drop(maint_comp3.machineID))

# Calculate the number of days between replacements
comp3 = (maint_tel_comp3.withColumn("sincelastcomp3",
                                    datediff(maint_tel_comp3.datetime_tel, maint_tel_comp3.datetime_maint))
         .drop(maint_tel_comp3.datetime_maint).drop(maint_tel_comp3.comp3sum))

print(comp3.count())
comp3.filter(comp3.machineID == '625').orderBy(comp3.datetime_tel).limit(5).toPandas().head(5)


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
To now, we have labels as failure events. To convert to impending failure, we over label over the previous 7 days as failed.
Can you write Python code for it?

# lag values to manually backfill label (bfill =7)
my_window = Window.partitionBy('machineID').orderBy(labeled_features.dt_truncated.desc())

# Create the previous 7 days
labeled_features = (labeled_features.withColumn("prev_value1",
                                                F.lag(labeled_features.failure).
                                                over(my_window)).fillna(0))
labeled_features = (labeled_features.withColumn("prev_value2",
                                                F.lag(labeled_features.prev_value1).
                                                over(my_window)).fillna(0))
labeled_features = (labeled_features.withColumn("prev_value3",
                                                F.lag(labeled_features.prev_value2).
                                                over(my_window)).fillna(0))
labeled_features = (labeled_features.withColumn("prev_value4",
                                                F.lag(labeled_features.prev_value3).
                                                over(my_window)).fillna(0))
labeled_features = (labeled_features.withColumn("prev_value5",
                                                F.lag(labeled_features.prev_value4).
                                                over(my_window)).fillna(0))
labeled_features = (labeled_features.withColumn("prev_value6",
                                                F.lag(labeled_features.prev_value5).
                                                over(my_window)).fillna(0))
labeled_features = (labeled_features.withColumn("prev_value7",
                                                F.lag(labeled_features.prev_value6).
                                                over(my_window)).fillna(0))

# Create a label features
labeled_features = (labeled_features.withColumn('label', labeled_features.failure +
                                                labeled_features.prev_value1 +
                                                labeled_features.prev_value2 +
                             ...


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
To verify the label construction, we plot a sample of four machines over the data set life time. We expect the labels to cluster for each component, since there are 7 day windows of "fail". We have omitted the healthy labels, as they are uninformative. Since the labels are actually classes, the plot as four distinct values on the y-axis.
Can you write Python code for it?

plt_dta = (labeled_features.filter(labeled_features.label_e > 0)
           .where(col("machineID").isin({"65", "558", "222", "965"}))
           .select(labeled_features.machineID, labeled_features.dt_truncated, labeled_features.label_e)
           .toPandas())

# format datetime field which comes in as string
plt_dta['dt_truncated'] = pd.to_datetime(plt_dta['dt_truncated'], format="%Y-%m-%d %H:%M:%S")
plt_dta.label_e = plt_dta.label_e.astype(int)

ggplot(aes(x="dt_truncated", y="label_e", color="label_e"), plt_dta) +\
    geom_point()+\
    xlab("Date") + ylab("Component Number") +\
    scale_x_date(labels=date_format('%m-%d')) +\
    scale_color_brewer(type = 'seq', palette = 'BuGn') +\
    facet_grid('machineID')


I am working on the file "Code/2_feature_engineering.ipynb".
The first task is:
and component 4 (comp4):
Can you write Python code for it?

# Grab component 4 records
maint_comp4 = (maint_replace.where(col("comp4sum") == '1').withColumnRenamed('datetime','datetime_maint')
               .drop('comp1sum', 'comp2sum', 'comp3sum'))

# Within each machine, get the last replacement date for each timepoint
maint_tel_comp4 = telemetry_times.join(maint_comp4, ((telemetry_times['machineID']==maint_comp4['machineID'])
                                                     & (telemetry_times['datetime_tel'] > maint_comp4['datetime_maint'])
                                                     & (maint_comp4['comp4sum'] == '1'))).drop(maint_comp4.machineID)

# Calculate the number of days between replacements
comp4 = (maint_tel_comp4.withColumn("sincelastcomp4",
                                    datediff(maint_tel_comp4.datetime_tel, maint_tel_comp4.datetime_maint))
         .drop(maint_tel_comp4.datetime_maint).drop(maint_tel_comp4.comp4sum))

print(comp4.count())
comp4.filter(comp4.machineID == '625').orderBy(comp4.datetime_tel).limit(5).toPandas().head(5)


I am working on the file "Code/3_model_building.ipynb".
The first task is:
Load feature data set
We have previously created the labeled feature data set in the Code\2_feature_engineering.ipynb Jupyter notebook. Since the Azure Blob storage account name and account key are not passed between notebooks, you'll need your credentials here again.
Can you write Python code for it?

# Enter your Azure blob storage details here
ACCOUNT_NAME = "<your blob storage account name>"

# You can find the account key under the _Access Keys_ link in the
# [Azure Portal](portal.azure.com) page for your Azure storage container.
ACCOUNT_KEY = "<your blob storage account key>"

#-------------------------------------------------------------------------------------------
# The data from the feature engineering note book is stored in the feature engineering container.
CONTAINER_NAME = CONTAINER_NAME = "featureengineering"

# Connect to your blob service
az_blob_service = BlockBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY)

# We will store and read each of these data sets in blob storage in an
# Azure Storage Container on your Azure subscription.
# See https://github.com/Azure/ViennaDocs/blob/master/Documentation/UsingBlobForStorage.md
# for details.

# This is the final feature data file.
FEATURES_LOCAL_DIRECT = 'featureengineering_files.parquet'

# This is where we store the final model data file.
LOCAL_DIRECT = 'model_result.parquet'


I am working on the file "Code/3_model_building.ipynb".
The first task is:
Prepare the Training/Testing data
A fundamental practice in machine learning is to calibrate and test your model parameters on data that has not been used to train the model. Evaluation of the model requires splitting the available data into a training portion, a calibration portion and an evaluation portion. Typically, 80% of data is used to train the model and 10% each to calibrate any parameter selection and evaluate your model.
In general random splitting can be used, but since time series data have an inherent correlation between observations. For predictive maintenance problems, a time-dependent spliting strategy is often a better approach to estimate performance. For a time-dependent split, a single point in time is chosen, the model is trained on examples up to that point in time, and validated on the examples after that point. This simulates training on current data and score data collected in the future data after the splitting point is not known. However, care must be taken on labels near the split point. In this case, feature records within 7 days of the split point can not be labeled as a failure, since that is unobserved data.
In the following code blocks, we split the data at a single point to train and evaluate this model.
Can you write Python code for it?

# define list of input columns for downstream modeling

# We'll use the known label, and key variables.
label_var = ['label_e']
key_cols =['machineID','dt_truncated']

# Then get the remaing feature names from the data
input_features = feat_data.columns

# We'll use the known label, key variables and
# a few extra columns we won't need.
remove_names = label_var + key_cols + ['failure','model_encoded','model' ]

# Remove the extra names if that are in the input_features list
input_features = [x for x in input_features if x not in set(remove_names)]

input_features


I am working on the file "Code/3_model_building.ipynb".
The first task is:
The confusion matrix lists each true component failure in rows and the predicted value in columns. Labels numbered 0.0 corresponds to no component failures. Labels numbered 1.0 through 4.0 correspond to failures in one of the four components in the machine. As an example, the third number in the top row indicates how many days we predicted component 2 would fail, when no components actually did fail. The second number in the second row, indicates how many days we correctly predicted a component 1 failure within the next 7 days.
We read the confusion matrix numbers along the diagonal as correctly classifying the component failures. Numbers above the diagonal indicate the model incorrectly predicting a failure when non occured, and those below indicate incorrectly predicting a non-failure for the row indicated component failure.
When evaluating classification models, it is convenient to reduce the results in the confusion matrix into a single performance statistic. However, depending on the problem space, it is impossible to always use the same statistic in this evaluation. Below, we calculate four such statistics.

Accuracy: reports how often we correctly predicted the labeled data. Unfortunatly, when there is a class imbalance (a large number of one of the labels relative to others), this measure is biased towards the largest class. In this case non-failure days.

Because of the class imbalance inherint in predictive maintenance problems, it is better to look at the remaining statistics instead. Here positive predictions indicate a failure.

Precision: Precision is a measure of how well the model classifies the truely positive samples. Precision depends on falsely classifying negative days as positive.

Recall: Recall is a measure of how well the model can find the positive samples. Recall depends on falsely classifying positive days as negative.

F1: F1 considers both the precision and the recall. F1 score is the harmonic average of precision and recall. An F1 score reaches its best value at 1 (perfect...
Can you write Python code for it?

# select (prediction, true label) and compute test error
# select (prediction, true label) and compute test error
# True positives - diagonal failure terms
tp = confuse['1.0'][1]+confuse['2.0'][2]+confuse['3.0'][3]+confuse['4.0'][4]

# False positves - All failure terms - True positives
fp = np.sum(np.sum(confuse[['1.0', '2.0','3.0','4.0']])) - tp

# True negatives
tn = confuse['0.0'][0]

# False negatives total of non-failure column - TN
fn = np.sum(np.sum(confuse[['0.0']])) - tn

# Accuracy is diagonal/total
acc_n = tn + tp
acc_d = np.sum(np.sum(confuse[['0.0','1.0', '2.0','3.0','4.0']]))
acc = acc_n/acc_d

# Calculate precision and recall.
prec = tp/(tp+fp)
rec = tp/(tp+fn)

# Print the evaluation metrics to the notebook
print("Accuracy = %g" % acc)
print("Precision = %g" % prec)
print("Recall = %g" % rec )
print("F1 = %g" % (2.0 * prec * rec/(prec + rec)))
print("")

# logger writes information back into the AML Workbench run time page.
# Each title (i.e. "Model Accuracy") can be shown as a graph to track
# how the metric changes between runs.
logger.log("Model Accuracy", (acc))
logger.log("Model Precision", (prec))
logger.log("Model Recall", (rec))
logger.log("Model F1", (2.0 * prec * rec/(prec + rec)))

importances = model_pipeline.stages[2].featureImportances

importances


I am working on the file "Code/3_model_building.ipynb".
The first task is:
To evaluate this model, we predict the component failures over the test data set. Since the test set has been created from data the model has not been seen before, it simulates future data. The evaluation then can be generalize to how the model could perform when operationalized and used to score the data in real time.
Can you write Python code for it?

# make predictions. The Pipeline does all the same operations on the test data
predictions = model_pipeline.transform(testing)

# Create the confusion matrix for the multiclass prediction results
# This result assumes a decision boundary of p = 0.5
conf_table = predictions.stat.crosstab('indexedLabel', 'prediction')
confuse = conf_table.toPandas()
confuse.head()


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
Comparing the output of this to the actual value indicates a mismatch in the failure prediction.
Model assets
Next we package the model assets into a zip file and store them to azure blob deployment into an operationalization environment. First write out the tested assets to local storage.
Can you write Python code for it?

# save the schema file for deployment
out = json.dumps(json_schema)
with open(os.environ['AZUREML_NATIVE_SHARE_DIRECTORY'] + 'service_schema.json', 'w') as f:
    f.write(out)


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
Create schema file
The deployment requires a schema file to define the incoming data.
Can you write Python code for it?

# We'll use the known label, key variables and
# a few extra columns we won't need. (machineID is required)
key_cols =['label_e','dt_truncated', 'failure','model_encoded','model' ]

# Then get the remaining feature names from the data
input_features = fedata.columns
# Remove the extra stuff if it's in the input_df
input_features = [x for x in input_features if x not in set(key_cols)]

# define the input data frame
inputs = {"input_df": SampleDefinition(DataTypes.SPARK,
                                       fedata.select(input_features))}

json_schema = generate_schema(run_func=run, inputs=inputs, filepath='service_schema.json')


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
Define deployment functions
The init() function initializes your web service, loading in any data or models that you need to score your inputs. In the example below, we load in the trained model. This command is run when the Docker container containing your service initializes.
The run() function defines what is executed on a scoring call. In our simple example, we simply load in the input as a data frame, and run our pipeline on the input, and return the prediction.
Start by defining the init() and run() functions, test them with example data. Then write them to the score.py file for deployment.
Can you write Python code for it?

# Initialize the deployment environment
def init():
    # read in the model file
    from pyspark.ml import PipelineModel
    global pipeline

    pipeline = PipelineModel.load(os.environ['AZUREML_NATIVE_SHARE_DIRECTORY']+'pdmrfull.model')

# Run the model and return the scored result.
def run(input_df):
    import json
    response = ''
    try:
        #Get prediction results for the dataframe

        # We'll use the known label, key variables and
        # a few extra columns we won't need.
        key_cols =['label_e','machineID','dt_truncated', 'failure','model_encoded','model' ]

        # Then get the remaing feature names from the data
        input_features = input_df.columns

        # Remove the extra stuff if it's in the input_df
        input_features = [x for x in input_features if x not in set(key_cols)]

        # Vectorize as in model building
        va = VectorAssembler(inputCols=(input_features), outputCol='features')
        data = va.transform(input_df).select('machineID','features')
        score = pipeline.transform(data)
        predictions = score.collect()

        #Get each scored result
        preds = [str(x['prediction']) for x in predictions]
        response = ",".join(preds)
    except Exception as e:
        print("Error: {0}",str(e))
        return (str(e))

    # Return results
    print(json.dumps(response))
    return json.dumps(response)


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
The deployment requires first initializing (init()) the environment, then running the model with the supplied data fields (run(<data>)). The run() function returns the predicted label, 0.0 indicates a healthy record, other values correspond to the component predicted to fail within the next 7 days (1.0, 2.0, 3.0, 4.0).
Can you write Python code for it?

# test init() in local notebook
init()

# test run() in local notebook
run(smple)


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
The model returned a 0.0, indicating a healthy prediction. Comparing this to the actual value of the label_e variable for this record would determine how the model actually did. However we did not include this feature in the sampled data, as it would not be available in the production environment.
In the following code block, we use the filter function to select 10 records with a specific failure label (4.0) indicating a failure for component 4 is probable within the next 7 days. You can see this by scrolling to the right to find the label_e variable.
Can you write Python code for it?

smple_f = fedata.filter(fedata.label_e == 4.0).sample(False, .8).limit(10)
smple_f.toPandas().head()


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
These files are stored in the ['AZUREML_NATIVE_SHARE_DIRECTORY'] location on the kernel host machine with the model stored in the 3_model_building.ipynb notebook. In order to share these assets and operationalize the model, we create a new blob container and store a compressed file containing those assets for later retrieval from the deployment location.
Can you write Python code for it?

# Compress the operationalization assets for easy blob storage transfer
MODEL_O16N = shutil.make_archive('o16n', 'zip', os.environ['AZUREML_NATIVE_SHARE_DIRECTORY'])

# Create a new container if necessary, otherwise you can use an existing container.
# This command creates the container if it does not already exist. Else it does nothing.
az_blob_service.create_container(MODEL_CONTAINER,
                                 fail_on_exist=False,
                                 public_access=PublicAccess.Container)

# Transfer the compressed operationalization assets into the blob container.
az_blob_service.create_blob_from_path(MODEL_CONTAINER, "o16n.zip", str(MODEL_O16N) )

# Time the notebook execution.
# This will only make sense if you "Run All" cells
toc = time.time()
print("Full run took %.2f minutes" % ((toc - tic)/60))

logger.log("Operationalization Run time", ((toc - tic)/60))


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
We need to load the feature data set from memory to construct the operationalization schema. We again will require your storage account name and account key to connect to the blob storage.
Can you write Python code for it?

# Enter your Azure blob storage details here
ACCOUNT_NAME = "<your blob storage account name>"

# You can find the account key under the _Access Keys_ link in the
# [Azure Portal](portal.azure.com) page for your Azure storage container.
ACCOUNT_KEY = "<your blob storage account key>"

#-------------------------------------------------------------------------------------------
# We will create this container to hold the results of executing this notebook.
# If this container name already exists, we will use that instead, however
# This notebook will ERASE ALL CONTENTS.
CONTAINER_NAME = "featureengineering"
FE_DIRECTORY = 'featureengineering_files.parquet'

MODEL_CONTAINER = 'modeldeploy'

# Connect to your blob service
az_blob_service = BlockBlobService(account_name=ACCOUNT_NAME, account_key=ACCOUNT_KEY)

# Create a new container if necessary, otherwise you can use an existing container.
# This command creates the container if it does not already exist. Else it does nothing.
az_blob_service.create_container(CONTAINER_NAME,
                                 fail_on_exist=False,
                                 public_access=PublicAccess.Container)

# create a local path where to store the results later.
if not os.path.exists(FE_DIRECTORY):
    os.makedirs(FE_DIRECTORY)

# download the entire parquet result folder to local path for a new run
for blob in az_blob_service.list_blobs(CONTAINER_NAME):
    if CONTAINER_NAME in blob.name:
        local_file = os.path.join(FE_DIRECTORY, os.path.basename(blob.name))
        az_blob_service.get_blob_to_path(CONTAINER_NAME, blob.name, local_file)

fedata = spark.read.parquet(FE_DIRECTORY)

fedata.limit(5).toPandas().head(5)


I am working on the file "Code/4_operationalization.ipynb".
The first task is:
We will use %%writefile meta command to save the init() and run() functions to the pdmscore.py file. Because of how the %%writefile command works, we have to copy these functions from the tested versions above into this code block.
Can you write Python code for it?

%%writefile {os.environ['AZUREML_NATIVE_SHARE_DIRECTORY']}/pdmscore.py

import json
from pyspark.ml import Pipeline
from pyspark.ml.classification import RandomForestClassifier, DecisionTreeClassifier

# for creating pipelines and model
from pyspark.ml.feature import StringIndexer, VectorAssembler, VectorIndexer

def init():
    # read in the model file
    from pyspark.ml import PipelineModel
    # read in the model file
    global pipeline
    pipeline = PipelineModel.load('pdmrfull.model')

def run(input_df):
    response = ''
    try:

        # We'll use the known label, key variables and
        # a few extra columns we won't need.
        key_cols =['label_e','machineID','dt_truncated', 'failure','model_encoded','model' ]

        # Then get the remaing feature names from the data
        input_features = input_df.columns

        # Remove the extra stuff if it's in the input_df
        input_features = [x for x in input_features if x not in set(key_cols)]

        # Vectorize as in model building
        va = VectorAssembler(inputCols=(input_features), outputCol='features')
        data = va.transform(input_df).select('machineID','features')
        score = pipeline.transform(data)
        predictions = score.collect()

        #Get each scored result
        preds = [str(x['prediction']) for x in predictions]
        response = ",".join(preds)
    except Exception as e:
        print("Error: {0}",str(e))
        return (str(e))

    # Return results
    print(json.dumps(response))
    return json.dumps(response)

if __name__ == "__main__":
    init()
   ...


I am working on the file "Code/AUP_LCA_evelynegroen.ipynb".
The first task is:
Step 1: Calculate partial derivatives
NB: this is a vectorized implementation of the MatLab code that was originally written by Reinout Heijungs & Sangwong Suh
Can you write Python code for it?

s = A_det.I * f                                 #scaling vector s: inv(A_det)*f
Lambda = B_det * A_det.I;                       #B_det*inv(A)

dgdA = -(s * Lambda).T                          #Partial derivatives A-matrix
Gamma_A = np.multiply((A_det/g_LCA), dgdA)      #For free: the multipliers of the A-matrix
print("The multipliers of the A-matrix are:")
print(Gamma_A)

dgdB = s.T                                      #Partial derivatives B-matrix
Gamma_B = np.multiply((B_det/g_LCA), dgdB)      #For free too: the multipliers of the B-matrix
print("The multipliers of the B-matrix are:")
print(Gamma_B)


I am working on the file "Code/AUP_LCA_evelynegroen.ipynb".
The first task is:
Step 2: Determine output variance
Can you write Python code for it?

CV = 0.05                             #Coefficient of variation set to 5% (CV = sigma/mu)
var_A = np.power(abs(CV*A_det),2)     #Variance of the A-matrix (var =sigma^2)
var_B = np.power(abs(CV*B_det),2)     #Variance of the B-matrix

P = np.concatenate((np.reshape(dgdA, 4), dgdB), axis=1)       #P contains partial derivatives of both A and B
var_P = np.concatenate((np.reshape(var_A, 4), var_B), axis=1) #var_P contains all variances of each parameter in A and B

var_g = np.sum(np.multiply(np.power(P, 2), var_P))             #Total output variance (first order Taylor)

print("The total output variance equals:", var_g)


I am working on the file "Code/AlexNet_Experiments.ipynb".
The first task is:
Define some variables
Can you write Python code for it?

batch_size = 16
input_size = (3,227,227)
nb_classes = 2
mean_flag = True # if False, then the mean subtraction layer is not prepended


I am working on the file "Code/AlexNet_Experiments.ipynb".
The first task is:
Discussion
The standard AlexNet, trained from scratch obtains ~84% accuracy. The accuracy and loss plots shown above suggest overfitting. In the next section, we will try to overcome this problem through fine-tuning.
Task 2 : Fine-Tuning Alexnet
We will follow the strategy as suggested in this paper - http://ieeexplore.ieee.org/abstract/document/7426826/?reload=true
The basic idea is to perform the training layerwise. What this means is - consider a 5 layer CNN with layers {L1,L2,L3,L4,L5}. In the first round of training, we freeze layers L1-L4 and finetune layer L5 for some epochs. In the next round, we train layers L4,L5 for some epochs. In the next round, we train L3,L4,L5 for some epochs. Essentially, the training percolates to shallow layers.
Can you write Python code for it?

alexnet = get_alexnet(input_size,nb_classes,mean_flag)
alexnet.load_weights('../convnets-keras/weights/alexnet_weights.h5', by_name=True)

print alexnet.summary()

layers = ['dense_3_new','dense_2','dense_1','conv_5_1','conv_4_1','conv_3','conv_2_1','conv_1']
epochs = [10,10,10,10,10,10,10,10]
lr = [1e-2,1e-3,1e-3,1e-3,1e-3,1e-3,1e-3,1e-3]

history_finetune = []

for i,layer in enumerate(layers):

    alexnet = unfreeze_layer_onwards(alexnet,layer)

    sgd = SGD(lr=lr[i], decay=1e-6, momentum=0.9, nesterov=True)
    alexnet.compile(loss='mse',
              optimizer=sgd,
              metrics=['accuracy'])

    for epoch in range(epochs[i]):
        h = alexnet.fit_generator(train_generator,
                                        samples_per_epoch=2000,
                                        validation_data=validation_generator,
                                        nb_val_samples=800,
                                        nb_epoch=1,
                                        verbose=1)

        history_finetune = append_history(history_finetune,h)

plot_performance(history_finetune)


I am working on the file "Code/AlexNet_Experiments.ipynb".
The first task is:
Discussion
We get a test accuracy of ~89%. This is almost a 5% jump over the training from scratch case. This is a huge jump in test accuracy! One should not expect such a bug jump usually. We should bear in mind that the ImageNet dataset, on which AlexNet was pre-trained already contained dogs and cats among its classes. However, many papers report benefits of fine-tuning over training from scratch.
Task 3 : Using AlexNet as a feature extractor
We treat the activation maps from outout of the last convolutional layer as dense feature maps. These are flattened and fed into a single layer ANN consisting of 256 neurons.
In the cell below, we get a sliced CNN model (alexnet_convolutional_only) which will enable us to extract features from any layer specified by its name ("convpool_5" in our case)
Can you write Python code for it?

alexnet = get_alexnet(input_size,nb_classes,mean_flag)
alexnet.load_weights('../convnets-keras/weights/alexnet_weights.h5', by_name=True)

alexnet_convolutional_only = Model(input=alexnet.input, output=alexnet.get_layer('convpool_5').output)


I am working on the file "Code/AlexNet_Experiments.ipynb".
The first task is:
Get the output of the sliced CNN model
Can you write Python code for it?

#code ported from https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html

import numpy as np

generator = train_datagen.flow_from_directory(
        '../Data/Train',
        target_size=(227, 227),
        batch_size=batch_size,
        class_mode=None,  # this means our generator will only yield batches of data, no labels
        shuffle=False)  # our data will be in order, so all first 1000 images will be cats, then 1000 dogs
# the predict_generator method returns the output of a model, given
# a generator that yields batches of numpy data
train_data = alexnet_convolutional_only.predict_generator(generator, 2000)
train_labels = np.array([[1, 0]] * 1000 + [[0, 1]] * 1000)

generator = test_datagen.flow_from_directory(
        '../Data/Test',
        target_size=(227, 227),
        batch_size=batch_size,
        class_mode=None,
        shuffle=False)

validation_data = alexnet_convolutional_only.predict_generator(generator, 800)
validation_labels = np.array([[1,0]] * 400 + [[0,1]] * 400)


I am working on the file "Code/AlexNet_Experiments.ipynb".
The first task is:
In the cell below, we define generators which are neat ways to perform realtime data augmentation. We use only basic data augmentation such as shear, zoom and flipping.
Can you write Python code for it?

#code ported from https://blog.keras.io/building-powerful-image-classification-models-using-very-little-data.html

# this is the augmentation configuration we will use for training
train_datagen = ImageDataGenerator(
        shear_range=0.2,
        zoom_range=0.2,
        horizontal_flip=True)

test_datagen = ImageDataGenerator()

train_generator = train_datagen.flow_from_directory(
        '../Data/Train',
        batch_size=batch_size,
        shuffle=True,
        target_size=input_size[1:],
        class_mode='categorical')

validation_generator = test_datagen.flow_from_directory(
        '../Data/Test',
        batch_size=batch_size,
        target_size=input_size[1:],
        shuffle=True,
        class_mode='categorical')


I am working on the file "Code/AlexNet_Experiments.ipynb".
The first task is:
Task 1 : Training AlexNet from scratch
Ontain the AlexNet model object. The get_alexnet function abstracts two things
    1. Implements real-time mean subtraction through prepending a custom layer
    2. Specifies the initialization parameter of each layer to he_normal
Can you write Python code for it?

alexnet = get_alexnet(input_size,nb_classes,mean_flag)

print alexnet.summary()


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
It's a good idea to close every file you open. It is recommended that you deal with files using the with keyword which ensures that the file is closed after use.
Can you write Python code for it?

with open('withFile.txt', 'w') as f:
    f.write("A better way of opening a file.\n")


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
Notice how Are you... still here? is all on a single line even though it was written using two separate calls to write. This happened because the "Are you... " string didn't terminate in \n (known as the newline character).
You can also read each line of a file into a list directly.
Can you write Python code for it?

with open('basicOutput.txt', 'r') as f:
    lines = f.readlines()
    print(lines)


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
Opening a file with 'w' will overwrite the existing contents of a file. If you want to append more information to the file use 'a'.
Can you write Python code for it?

with open("basicOutput.txt", 'a') as f:
    f.write("Are you... ")
    f.write("still here?\n")


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
The above writes the time and temperature data to a file, along with a header describing the data. We can verify this by opening and reading the file
Can you write Python code for it?

with open("temperatureData.csv", 'r') as f:
    print(f.read())


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
The skiprows keyword tells loadtxt to skip the first line (which contains the header). It's also possible to read the columns in directly to variables.
Can you write Python code for it?

timeFromFile, temperatureFromFile = np.loadtxt("temperatureData.csv", delimiter=",", skiprows=1,
                                        unpack=True)
timeFromFile, temperatureFromFile


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
This last method is useful when dealing with very large files. In such cases reading the whole file into Python may be very slow.
Comma Separate Values (CSV)
CSV files are a very basic way of of storing data. For example the following mock file has two columns, one for Time and another for Temperature. Each row represents an entry, with commas used to separate values.
Time (s), Temperature (K)
0,300
10,314
20,323
30,331
Python has a csv library that can be used to read and write csv files. It's a very general purpose library and works in many different scenarios. For our purposes though it's more straightforward to use two Numpy functions np.savetxt and np.readtxt.
Can you write Python code for it?

import numpy as np

time = [0, 10, 20, 30]
temperature = [300, 314, 323, 331]
zippedData = zip(time, temperature)

np.savetxt('temperatureData.csv', zippedData,
           delimiter=',', header="Time (s), Temperature (K)")
zippedData


I am working on the file "Code/An Interlude on Input and Output/1 - Reading and Writing Data.ipynb".
The first task is:
You can read the contents of a file as a single string using the read function.
Can you write Python code for it?

with open('basicOutput.txt', 'r') as f:  # Use 'r' for reading.
    contents = f.read()
    print(contents)


I am working on the file "Code/Assignment-10/GenderDistinguished_ADHD_Bipolar_Fix&Tune_YatingJing.ipynb".
The first task is:
Classification
Can you write Python code for it?

from sklearn.cross_validation import KFold

def train_test_clf(clf, clf_name, X, y, dr_method, k=10, n_comps=10):
    '''
    Train and test a classifier using K-fold cross validation
    Args:
        clf: sklearn classifier
        clf_name: classifier name (for printing)
        X: training data (2D numpy matrix)
        y: labels (1D vector)
        dr_method: dimensionality reduction method used to construct the features
        k: number of folds (default=10)
        n_comps: number of principal components to keep (default=10)
    '''
    X, y = np.array(X), np.array(y)
    kf = KFold(len(X), n_folds=k)
    scores = list()

    for i, (train_index, test_index) in enumerate(kf):
        # print 'Fold', i, '(10-fold CV)'
        # Train test split
        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]

        X_train_redu, X_test_redu = [], []

        # Dimensionality Reduction
        if dr_method == 'PCA':
            # PCA fit
            pca, X_train_redu = pca_fit(X_train, n_comps)
            # PCA transform
            X_test_redu = pca_transform(pca, X_test, n_comps)
        elif dr_method == 'LLE':
            # LLE fit
            lle, X_train_redu = lle_fit(X_train, n_comps)
            # LLE transform
            X_test_redu = lle_transform(lle, X_test, n_comps)

        # Classification
        clf.fit(X_train_redu, y_train)
        score = clf.score(X_test_redu, y_test)
        scores.append(score)

    scores = np.array(scores)
    acc, acc_std = scores.mean(), scores.std()

    print clf_name + ' accuracy is %0.4f (+/- %0.3f)' % (acc, acc_std)
    return acc, acc_std

%matplotlib inline
import matplotlib.pyplot as plt

from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.lda import LDA
from sklearn.qda import QDA
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble...


I am working on the file "Code/Assignment-10/GenderDistinguished_ADHD_Bipolar_Fix&Tune_YatingJing.ipynb".
The first task is:
Gender 2 ADHD v.s. Bipolar Analysis
Can you write Python code for it?

# Classification using PCA features
print 'Using PCA features:\n'
X2 = df2_all.values
classify(X2, y2, 'Gender 2', 'PCA', 10)

# Classification using LLE features
print 'Using LLE features:\n'
X2 = df2_all.values
classify(X2, y2, 'Gender 2', 'LLE', 15)


I am working on the file "Code/Assignment-10/GenderDistinguished_ADHD_Bipolar_Fix&Tune_YatingJing.ipynb".
The first task is:
Locally Linear Embedding
Can you write Python code for it?

from sklearn import preprocessing
from sklearn.manifold import LocallyLinearEmbedding

# Compute explained variance ratio of transformed data
def compute_explained_variance_ratio(transformed_data):
    explained_variance = np.var(transformed_data, axis=0)
    explained_variance_ratio = explained_variance / np.sum(explained_variance)
    explained_variance_ratio = np.sort(explained_variance_ratio)[::-1]
    return explained_variance_ratio

def lle_fit(X, n=10):
    # LLE
    lle = LocallyLinearEmbedding(n_neighbors=25, n_components=n, method='ltsa')
    # Scale
    X_scaled = preprocessing.scale(X)

    pc = lle.fit_transform(X_scaled)
    ex_var_ratio = compute_explained_variance_ratio(pc)

    # print 'Explained Variance Ratios:'
    # print ex_var_ratio

    return lle, pc

def lle_transform(lle, X, n=10):
    X_scaled = preprocessing.scale(X)
    pc = lle.transform(X_scaled)
    return pc


I am working on the file "Code/Assignment-10/GenderDistinguished_ADHD_Bipolar_Fix&Tune_YatingJing.ipynb".
The first task is:
Machine Learning Utilities
Principal Component Analysis
Can you write Python code for it?

from sklearn import preprocessing
from sklearn.decomposition import PCA

# Plot explained variance ratio
def plot_evr(ex_var_ratio):
    fig = plt.figure()
    plt.title('Explained Variance Ratios by PCA')
    plt.plot(ex_var_ratio)
    plt.ylabel('Explained Variance Ratio')
    plt.xlabel('Principal Component')

def pca_fit(X, n=10):
    # PCA
    pca = PCA(n_components=n)

    # Scale
    X_scaled = preprocessing.scale(X)

    # Fit
    pc = pca.fit_transform(X_scaled)

    # print '\nExplained Variance Ratios:'
    # print pca.explained_variance_ratio_

    # print 'Sum of Explained Variance Ratios of the first', n, 'components is',
    # print np.sum(pca.explained_variance_ratio_)

    # plot_evr(pca.explained_variance_ratio_)

    return pca, pc

def pca_transform(pca, X, n=10):
    X_scaled = preprocessing.scale(X)
    pc = pca.transform(X_scaled)
    return pc


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (Disorder data).ipynb".
The first task is:
As is evident from the above 2 experiments, no clear clustering is apparent.But there is some significant overlap and there 2 clear groups
Classification Experiments
Let's experiment with a bunch of classifiers
Can you write Python code for it?

ADHD_men_iso = pd.DataFrame(ADHD_men_iso)
BP_men_iso = pd.DataFrame(BP_men_iso)

ADHD_cauc_iso = pd.DataFrame(ADHD_cauc_iso)
BP_cauc_iso = pd.DataFrame(BP_cauc_iso)

BP_men_iso['ADHD-Bipolar'] = 0
ADHD_men_iso['ADHD-Bipolar'] = 1

BP_cauc_iso['ADHD-Bipolar'] = 0
ADHD_cauc_iso['ADHD-Bipolar'] = 1

data1 = pd.concat([ADHD_men_iso, BP_men_iso])
data2 = pd.concat([ADHD_cauc_iso, BP_cauc_iso])
class_labels1 = data1['ADHD-Bipolar']
class_labels2 = data2['ADHD-Bipolar']
data1 = data1.drop(['ADHD-Bipolar'], axis = 1, inplace = False)
data2 = data2.drop(['ADHD-Bipolar'], axis = 1, inplace = False)
data1 = data1.get_values()
data2 = data2.get_values()

# Leave one Out cross validation
def leave_one_out(classifier, values, labels):
    leave_one_out_validator = LeaveOneOut(len(values))
    classifier_metrics = cross_validation.cross_val_score(classifier, values, labels, cv=leave_one_out_validator)
    accuracy = classifier_metrics.mean()
    deviation = classifier_metrics.std()
    return accuracy, deviation

rf = RandomForestClassifier(n_estimators = 22)
qda = QDA()
lda = LDA()
gnb = GaussianNB()
classifier_accuracy_list = []
classifiers = [(rf, "Random Forest"), (lda, "LDA"), (qda, "QDA"), (gnb, "Gaussian NB")]
for classifier, name in classifiers:
    accuracy, deviation = leave_one_out(classifier, data1, class_labels1)
    print '%s accuracy is %0.4f (+/- %0.3f)' % (name, accuracy, deviation)
    classifier_accuracy_list.append((name, accuracy))

for classifier, name in classifiers:
    accuracy, deviation = leave_one_out(classifier, data2, class_labels2)
    print '%s accuracy is %0.4f (+/- %0.3f)' % (name, accuracy, deviation)
    classifier_accuracy_list.append((name, accuracy))


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (Disorder data).ipynb".
The first task is:
Dimensionality reduction
Manifold Techniques
ISOMAP
Can you write Python code for it?

combined1 = pd.concat([ADHD_men, BP_men])
combined2 = pd.concat([ADHD_cauc, BP_cauc])

print combined1.shape
print combined2.shape

combined1 = preprocessing.scale(combined1)
combined2 = preprocessing.scale(combined2)

combined1 = manifold.Isomap(20, 20).fit_transform(combined1)
ADHD_men_iso = combined1[:1326]
BP_men_iso = combined1[1326:]

combined2 = manifold.Isomap(20, 20).fit_transform(combined2)
ADHD_cauc_iso = combined2[:1379]
BP_cauc_iso = combined2[1379:]


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data with baseline).ipynb".
The first task is:
Clustering and other grouping experiments
K-Means clustering - iso
Can you write Python code for it?

data1 = pd.concat([pd.DataFrame(ADHD_men_iso), pd.DataFrame(BP_men_iso)])
data2 = pd.concat([pd.DataFrame(ADHD_cauc_iso), pd.DataFrame(BP_cauc_iso)])

print data1.shape
print data2.shape

kmeans = KMeans(n_clusters=2)
kmeans.fit(data1.get_values())
labels1 = kmeans.labels_
centroids1 = kmeans.cluster_centers_
print('Estimated number of clusters: %d' % len(centroids1))

for label in [0, 1]:
    ds = data1.get_values()[np.where(labels1 == label)]
    plt.plot(ds[:,0], ds[:,1], '.')
    lines = plt.plot(centroids1[label,0], centroids1[label,1], 'o')

kmeans = KMeans(n_clusters=2)
kmeans.fit(data2.get_values())
labels2 = kmeans.labels_
centroids2 = kmeans.cluster_centers_
print('Estimated number of clusters: %d' % len(centroids2))

for label in [0, 1]:
    ds2 = data2.get_values()[np.where(labels2 == label)]
    plt.plot(ds2[:,0], ds2[:,1], '.')
    lines = plt.plot(centroids2[label,0], centroids2[label,1], 'o')


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data with baseline).ipynb".
The first task is:
Dimensionality reduction
Manifold Techniques
ISOMAP
Can you write Python code for it?

combined1 = pd.concat([ADHD_men, BP_men])
combined2 = pd.concat([ADHD_cauc, BP_cauc])

print combined1.shape
print combined2.shape

combined1 = preprocessing.scale(combined1)
combined2 = preprocessing.scale(combined2)

combined1 = manifold.Isomap(20, 20).fit_transform(combined1)
ADHD_men_iso = combined1[:946]
BP_men_iso = combined1[946:]

combined2 = manifold.Isomap(20, 20).fit_transform(combined2)
ADHD_cauc_iso = combined2[:992]
BP_cauc_iso = combined2[992:]


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data with baseline).ipynb".
The first task is:
Extracting the samples we are interested in
Can you write Python code for it?

# Let's extract ADHd and Bipolar patients (mutually exclusive)

ADHD_men = X.loc[X['ADHD'] == 1]
ADHD_men = ADHD_men.loc[ADHD_men['Bipolar'] == 0]

BP_men = X.loc[X['Bipolar'] == 1]
BP_men = BP_men.loc[BP_men['ADHD'] == 0]

ADHD_cauc = Y.loc[Y['ADHD'] == 1]
ADHD_cauc = ADHD_cauc.loc[ADHD_cauc['Bipolar'] == 0]

BP_cauc = Y.loc[Y['Bipolar'] == 1]
BP_cauc = BP_cauc.loc[BP_cauc['ADHD'] == 0]

print ADHD_men.shape
print BP_men.shape

print ADHD_cauc.shape
print BP_cauc.shape

# Keeping a backup of the data frame object because numpy arrays don't play well with certain scikit functions
ADHD_men = pd.DataFrame(ADHD_men.drop(['Patient_ID', 'Gender_id', 'ADHD', 'Bipolar'], axis = 1, inplace = False))
BP_men = pd.DataFrame(BP_men.drop(['Patient_ID', 'Gender_id', 'ADHD', 'Bipolar'], axis = 1, inplace = False))

ADHD_cauc = pd.DataFrame(ADHD_cauc.drop(['Patient_ID', 'race_id', 'ADHD', 'Bipolar'], axis = 1, inplace = False))
BP_cauc = pd.DataFrame(BP_cauc.drop(['Patient_ID', 'race_id', 'ADHD', 'Bipolar'], axis = 1, inplace = False))


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data).ipynb".
The first task is:
As is evident from the above 2 experiments, no clear clustering is apparent.But there is some significant overlap and there 2 clear groups
Classification Experiments
Let's experiment with a bunch of classifiers
Can you write Python code for it?


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data).ipynb".
The first task is:
Clustering and other grouping experiments
K-Means clustering - iso
Can you write Python code for it?


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data).ipynb".
The first task is:
Dimensionality reduction
Manifold Techniques
ISOMAP
Can you write Python code for it?

combined1 = pd.concat([ADHD_men, BP_men])
combined2 = pd.concat([ADHD_cauc, BP_cauc])

print combined1.shape
print combined2.shape

combined1 = preprocessing.scale(combined1)
combined2 = preprocessing.scale(combined2)

combined1 = manifold.Isomap(20, 20).fit_transform(combined1)
ADHD_men_iso = combined1[:1056]
BP_men_iso = combined1[1056:]

combined2 = manifold.Isomap(20, 20).fit_transform(combined2)
ADHD_cauc_iso = combined2[:1110]
BP_cauc_iso = combined2[1110:]


I am working on the file "Code/Assignment-10/SubjectSelectionExperiments (rCBF data).ipynb".
The first task is:
Extracting the samples we are interested in
Can you write Python code for it?


I am working on the file "Code/Assignment-11/AdvancedFeatureSelection.ipynb".
The first task is:
Classifiers
Can you write Python code for it?

# Leave one Out cross validation
def leave_one_out(classifier, values, labels):
    leave_one_out_validator = LeaveOneOut(len(values))
    classifier_metrics = cross_validation.cross_val_score(classifier, values, labels, cv=leave_one_out_validator)
    accuracy = classifier_metrics.mean()
    deviation = classifier_metrics.std()
    return accuracy, deviation

rf = RandomForestClassifier(n_estimators = 22)
qda = QDA()
lda = LDA()
gnb = GaussianNB()
classifier_accuracy_list = []
classifiers = [(rf, "Random Forest"), (lda, "LDA"), (qda, "QDA"), (gnb, "Gaussian NB")]
for classifier, name in classifiers:
    accuracy, deviation = leave_one_out(classifier, combined1_recf, clabels1)
    print '%s accuracy is %0.4f (+/- %0.3f)' % (name, accuracy, deviation)
    classifier_accuracy_list.append((name, accuracy))

for classifier, name in classifiers:
    accuracy, deviation = leave_one_out(classifier, combined1_kpercentile, clabels1)
    print '%s accuracy is %0.4f (+/- %0.3f)' % (name, accuracy, deviation)
    classifier_accuracy_list.append((name, accuracy))

for classifier, name in classifiers:
    accuracy, deviation = leave_one_out(classifier, combined1_trees, clabels1)
    print '%s accuracy is %0.4f (+/- %0.3f)' % (name, accuracy, deviation)
    classifier_accuracy_list.append((name, accuracy))


I am working on the file "Code/Assignment-11/AdvancedFeatureSelection.ipynb".
The first task is:
Feature Selection
We are now going to explore Some feature selection procedures, the output of this will then be sent to a classifier

Recursive elimination with cross validation
Simple best percentile features
Tree based feature selection

The output from this is then sent to  the following classifiers

1. Random Forrests - Good ensemble technique
2. QDA - Other experiments with this classifier have been successful
3. LDA - A good simple technique
4. Gaussian Naive Bayes - Experiments with this classifier have proven successful in the past
Can you write Python code for it?

from sklearn.svm import SVC
from sklearn.cross_validation import StratifiedKFold
from sklearn.feature_selection import RFECV
from sklearn.feature_selection import SelectFromModel
from sklearn.ensemble import ExtraTreesClassifier

# Make the Labels vector
clabels1 = [1] * 946 + [0] * 223

# Concatenate and Scale
combined1 = pd.concat([ADHD_men, BP_men])
combined1 = pd.DataFrame(preprocessing.scale(combined1))

# Recursive Feature elimination with cross validation
svc = SVC(kernel="linear")
rfecv = RFECV(estimator=svc, step=1, cv=StratifiedKFold(clabels1, 2),
              scoring='accuracy')
rfecv.fit(combined1, clabels1)
combined1_recf = rfecv.transform(combined1)

combined1_recf = pd.DataFrame(combined1_recf)
print combined1_recf.head()

# Percentile base feature selection

from sklearn.feature_selection import SelectPercentile, f_classif
selector = SelectPercentile(f_classif, percentile=5)
combined_kpercentile = selector.fit_transform(combined1, clabels1)

combined1_kpercentile = pd.DataFrame(combined_kpercentile)
print combined1_kpercentile.head()

# Tree based selection
from sklearn.ensemble import ExtraTreesClassifier

clf = ExtraTreesClassifier()
clf = clf.fit(combined1, clabels1)
combined1_trees = SelectFromModel(clf, prefit=True).transform(combined1)

combined1_trees = pd.DataFrame(combined1_trees)
print combined1_trees.head()


I am working on the file "Code/Assignment-11/GridSearch_YatingJing.ipynb".
The first task is:
Gender 1 ADHD v.s. Bipolar Analysis
Can you write Python code for it?

print 'Gender 1 Analysis using LLE features:\n'
X1 = df1_all.values
lle_clf_pipe(X1, y1, 'Gender 1', 15, k=10)


I am working on the file "Code/Assignment-11/GridSearch_YatingJing.ipynb".
The first task is:
Gender 2 ADHD v.s. Bipolar Analysis
Can you write Python code for it?

print 'Gender 2 Analysis using LLE features:\n'
X2 = df2_all.values
lle_clf_pipe(X2, y2, 'Gender 2', 15, k=10)


I am working on the file "Code/Assignment-3/Descriptive_Exploratory_Answers_2.ipynb".
The first task is:
Additional prediction goals: ( in the future )
-whether or not certain disorders are correlated with certain baseline values (or delta values from concentration to baseline)
-if different parts of the brain are affected with different kinds of ADHD
Exploratory

What is the sole metric that can be used to separate healthy people from unhealthy people?

As shown in the label construction procedure above, our sole metric for identifying a person as healthy is: the record (row) for such person has 0 values for all disorders (across all disorder columns).

What is the range of values nominal features can take?
Can you write Python code for it?

# Read full dataset
df_all = pd.DataFrame.from_csv('Data_Adults_1.csv')

# Extract non-numerical features
non_num_keys = [key for key in dict(df_all.dtypes) if dict(df_all.dtypes)[key] not in ['float', 'int']]

print 'Nominal features are', non_num_keys

print 'Unique values for the nominal features:\n'
print np.unique(df_all['RaceName'])
print np.unique(df_all['Age_Group'])
print np.unique(df_all['STUDY_NAME'])
print np.unique(df_all['BSC_Respondent'])
print np.unique(df_all['ADHD_Type'])
print np.unique(df_all['locationname'])
print np.unique(df_all['LDS_Respondent'])
print np.unique(df_all['GSC_Respondent'])
print np.unique(df_all['group_name'])
print np.unique(df_all['Gendername'])


I am working on the file "Code/Assignment-3/Descriptive_Exploratory_Answers_2.ipynb".
The first task is:
Descriptive

What do the features in the vector indicate?

Current guess:
This data is likely from a study of people with ADHD, as it is looking at brain activity during a concentration activity. We could maybe try to parse out the areas of the brain associated with different types of ADHD from parts that are activated due to a different disorder based on comparing those with one disorder to some without.
Also the trends of different kinds of ADHD could have different locations in the brain for the concentration task

Why are certain types of baseline values not applicable to certain individuals (NaN values)?

It is impossible to know for sure why certain baseline values are not applicable to certain individuals. However, we speculate a few possibilities. Perhaps baseline data was not recorded, either by design or by error (e.g. measurement utensil was not calibrated appropriately). Alternatively, the baseline data may not have been collected originally, and then later found necessary to compare to the concentration values. The cost of measuring brain activity could also have factored in to the decision to only record the baseline values of some of the participants. Nonetheless, as stated above, we have eliminated the missing values from our training data because without a baseline value, the concentration value is not as meaningful. If we find any consistent trends within the data, we may be able to estimate the baseline values for those with missing information.

What kind of labels are we going to extract? What will be yi?

For starters, a simple learning goal would be to separate healthy participants from the unhealthy ones.
Here we identify those who are diagnosed with no disorders as the healty partipants, and assign such patient records with label of value 1, the remaining records are assigned label 0.
Therefore to answer the question above, our labels at the initial stage are binary, indicating whether a participant is healthy (has no disorders) or not.
Can you write Python code for it?

# Ignore different types of ADHD for now
df_disorder_results = df_disorders.drop('ADHD_Type', inplace=False, axis=1)

# Find records that has zero values across all the columns (disorders)
# Extract a list of Patient_IDs corresponding to healthy participants
healthy_ids = df_disorder_results[(df_disorder_results.T==0).all()].index.tolist()

print 'There are %d healthy participants.\n' % len(healthy_ids)
print 'Their Patient_IDs are', healthy_ids
print '\nFor the records above, label y_i=1 (healthy), for all the other records, y_i=0 (mentally disordered).'

# Now constuct the label vector y
y = pd.Series([0] * len(df_disorders), index=patient_ids)
y[healthy_ids] = 1

print 'Finish constructing label vector for healthy/unhealthy.'


I am working on the file "Code/Assignment-3/Descriptive_Exploratory_Answers_2.ipynb".
The first task is:
In order not to comfuse our future model, here for certain types of features, we avoid using one value to denote different categories a variable can take, instead, we want to encode them into small one-hot vectors.
Note that we do not take null values into consideration for now.
Range of values for the nominal features:
1) RaceName: This can be encoded into a one-hot vector of length 16. For example, 'African American' will be encoded into [1]+[0]*15 (python grammar), with the value at index 0 set to 1, the rest of 15 elements of the vector are 0. 'Asian' takes value of 1 at index 2, 'Unknown' takes value of 1 at index 15, etc.
2) Age_Group: This can be represented by scales, where Adult=2, Geriatric=3, Pediatric=1.
3) STUDY_NAME: Only one name, can be removed from the dataset.
4) BSC_Respondent, ADHD_Type, locationname, LDS_Respondent, GSC_Respondent can be encoded in the same way as RaceName, except that their vector lengths will be 4, 7, 11, 4, 4 respectively.
5) group_name: We are not sure if this is just random group names or control group versus experimental group, this has to be decided after we get to look at data documentation. For the former case, it can be ignored. For the latter case, we will represent it using a binary indicator (0-1).
6) Gender: Female=0, Male=1, Unknown=0.5 (maybe).

Are the features correlated?

For feature correlation, since we do not have complete information about all column headers at this point, only correlation within baseline values and concentration values are analyzed.
Can you write Python code for it?

# Get baseline and concenctration data
df_base = pd.DataFrame.from_csv('baseline.csv')
df_concen = pd.DataFrame.from_csv('concentration.csv')

# Use numpy matrix format (numerical)
df_base_vals = df_base.values
df_concen_vals = df_concen.values

def check_perfect_corr(coeff):
    # Fill diagonal with 0 (not comparing to oneself)
    np.fill_diagonal(coeff, 0)
    # Perfect correlation: 1 or -1
    return coeff.max()==1 or coeff.min()==-1

# Compute Pearson product-moment correlation coefficients
# Check for perfect correlation row-wise and column-wise
def pearson_corr_test(x):
    # rowvar = 1: row-wise
    # rowvar = 0: column-wise
    row_coeff = np.corrcoef(x, y=None, rowvar=1)
    col_coeff = np.corrcoef(x, y=None, rowvar=0)

    # Check for perfect correlation row-wise
    row_corr = check_perfect_corr(row_coeff)
    # Check for perfect correlation column-wise
    col_corr = check_perfect_corr(col_coeff)

    return row_corr and col_corr

print 'Perfect correlation exists in baselines?', pearson_corr_test(df_base_vals)
print 'Perfect correlation exists in concentrations?', pearson_corr_test(df_concen_vals)


I am working on the file "Code/Assignment-3/Exploratory_Kmeans_PCA.ipynb".
The first task is:
Can we find k most important features to construct our training data, where k is significantly smaller than m?

Given the explained variance ratios printed and plotted above, if we can settle for capturing 90% of variance, then we can find a k such that k = 10 << m = 736.
Can you write Python code for it?

# Looking at what columns are favored by the first two principal dimensions
print '\nColumns favored by the first principal component:'
pc = pd.DataFrame(pca.components_, columns=X.iloc[:, :].columns).T
pc.sort_values(0, ascending=False)[:6]

print '\nColumns favored by the second principal component:'
pc.sort_values(1, ascending=False)[:6]


I am working on the file "Code/Assignment-3/Exploratory_Kmeans_PCA.ipynb".
The first task is:
K-Means Clustering
Can you write Python code for it?

from sklearn.cluster import KMeans

k = 4
data = X.get_values().T
kmeans = KMeans(n_clusters=k)
kmeans.fit(data)

labels = kmeans.labels_
centroids = kmeans.cluster_centers_

for i in range(k):
    # Extract observations within each cluster
    ds = data[np.where(labels==i)]
    # Plot the observations with symbol o
    plt.plot(ds[:,0], ds[:,1], 'o')
    # Plot the centroids with simbol x
    lines = plt.plot(centroids[i,0], centroids[i,1], 'x')
    plt.setp(lines, ms=8.0)
    plt.setp(lines, mew=2.0)


I am working on the file "Code/Assignment-4/Inferential_Depressed.ipynb".
The first task is:
Compute p-value on your real data
Can you write Python code for it?

p_vals = list()
for a, b in zip(depr, not_depr):
    p_vals.append(round(ks_2samp(a, b)[1], 5))
print p_vals


I am working on the file "Code/Assignment-4/Inferential_Depressed.ipynb".
The first task is:
Note: Generate random samples and plot power vs n on null set based on Greg's code.
Random Sampling Setup
Can you write Python code for it?

import itertools

np.random.seed(123456789) # for reproducibility, set random seed
alpha = 0.05
r = 20  # number of rois
N = 100 # number of samples at each iteration

# define number of subjects per class
S = np.array((4, 6, 8, 10, 14, 18, 20, 26, 30, 40,
              50, 60, 70, 80, 100, 120, 150, 200, 250,
              300, 400, 500, 750, 1000, 1500, 2000,
              3000, 5000))


I am working on the file "Code/Assignment-4/Inferential_Depressed.ipynb".
The first task is:
Sample data from null
Can you write Python code for it?

pow_null = np.array((), dtype=np.dtype('float64'))

# compute this statistic for various sizes of datasets
for s in S:
    s0 = s/2
    s1 = s - s0

    # compute this many times for each operating point to get average
    pval = np.array((), dtype=np.dtype('float64'))
    for _ in itertools.repeat(None,N):
        g0 = 1 * (np.random.rand(r, r, s0) > 0.5) # (null), 0.52 (classes)
        g1 = 1 * (np.random.rand(r, r, s1) > 0.5) # (null), 0.48 (classes)

        # compute feature of data
        pbar0 = 1.0*np.sum(g0, axis=(0,1))/(r**2 * s0)
        pbar1 = 1.0*np.sum(g1, axis=(0,1))/(r**2 * s1)

        # compute K-S test on feature
        pval = np.append(pval, ks_2samp(pbar0, pbar1)[1])

    # record average p value at operating point
    pow_null = np.append(pow_null, np.sum(1.0*(pval < alpha))/N)


I am working on the file "Code/Assignment-5/Classification.ipynb".
The first task is:
1. State assumptions about your data

X are i.i.d. random variables.
y ~ Bern(p)

2. Formally define classification problem
Feature matrix X: Rn x m
Each sample Xi ∈ Rm, i ∈ [1, n]
Label yi ∈ {0, 1}, i ∈ [1, n]
g(X) → y
G := { g: R → {0, 1} }
Goal: g* = argming ∈ G E[L(g(X), y)], where L denotes loss function.
The loss function L differs for different classifiers and is speficied in the classfication context below:

Multinomial Naive Bayes: negative joint likelihood

L = -log p(X, y)

Logistic Regression:  logistic loss (cross-entropy loss)

L = -log P(y|g(X)) = -(y · log(g(X)) + (1 - y) · log(1 - g(X))

K Nearest Neighbors

L = ∑i D(Xi|yi=1, X|y=1) + ∑i D(Xi|yi=0, X|y=0)

D(a, b) = (a - b)2

Support Vector Machine: squared hinge loss

L = (max{0, 1 − y · g(x)})2

Random Forest

L = ∑i (g(Xi) - yi)2

3. Provide algorithm for solving problem (including choosing hyperparameters as appropriate)

Multinomial Naive Bayes

alpha: default =1.0 (Laplace smoothing parameter)

fit_prior: default = True (learn class prior probabilities)

Logistic Regression

penalty = 'l1' (l1 norm for penalization)

K Nearest Neighbors

n_neighbors = 10

Support Vector Machine (Linear Kernel)

C: default = 1.0 (penalty parameter of the error term)

Random Forest

n_estimators = 60 (number of trees)

criterion: default = 'gini'

4. Sample data from a simulation setting inspired by your data
Can you write Python code for it?

'''
Data Preparation
'''
from sklearn.cross_validation import train_test_split

# Split the simulated data into training set and test set
# Randomly sample 20% data as the test set
train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.2, random_state=42)

print 'Training set size is', train_X.shape
print 'Testing set size is', test_X.shape

max_n = len(train_X)

# Train the given classifier
def train_clf(clf, train_feats, train_labels):
    # Supervised training
    clf.fit(train_feats, train_labels)

# Test the given classifier anc calculate accuracy
def test_clf(clf, test_feats, test_labels):
    # Predict using test set
    predicted = clf.predict(test_feats)
    # Compute accuracy
    acc = np.mean(predicted == test_labels)
    return predicted, acc


I am working on the file "Code/Assignment-5/Classification.ipynb".
The first task is:
5. Compute accuracy
Can you write Python code for it?

# Compute accuracy of a model trained with a specific number of samples
def compute_acc(clf, n):
    train_clf(clf, train_X[:n], train_y[:n])
    predict_y, acc = test_clf(clf, test_X, test_y)
    return acc

from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier

[acc_NB, acc_LG, acc_KNN, acc_SVM, acc_RF] = [[]  for i in xrange(5)]

for n in xrange(100, max_n, 100):
    # Multinomial Naive Bayes
    multiNB = MultinomialNB()
    acc_NB.append(compute_acc(multiNB, n))

    # Logistic Regression
    lg = LogisticRegression(penalty='l1')
    acc_LG.append(compute_acc(lg, n))

    # K Nearest Neighbors
    knn = KNeighborsClassifier(n_neighbors=10)
    acc_KNN.append(compute_acc(knn, n))

    # Support Vector Machine
    svc = LinearSVC()
    acc_SVM.append(compute_acc(svc, n))

    # Random Forest
    rf = RandomForestClassifier(n_estimators=60)
    acc_RF.append(compute_acc(rf, n))


I am working on the file "Code/Assignment-5/Classification.ipynb".
The first task is:
6. Plot accuracy vs. sample size in simulation
Can you write Python code for it?

%matplotlib inline
import matplotlib.pyplot as plt

sizes = range(100, max_n, 100)

fig = plt.figure(1)
fig.set_size_inches(9, 6.5)

plt.plot(sizes, acc_NB, label='Multinomial NB')
plt.plot(sizes, acc_LG, label='Logistic Regression')
plt.plot(sizes, acc_KNN, label='K Nearest Neighbors')
plt.plot(sizes, acc_SVM, label='Support Vector Machine')
plt.plot(sizes, acc_RF, label='Random Forest')

plt.legend(loc='best')
plt.xlabel('sample size')
plt.ylabel('Simulation Accuracy')


I am working on the file "Code/Assignment-5/Classification.ipynb".
The first task is:
7. Apply method directly on real data
Can you write Python code for it?

'''
Train models with all the training data
Evaluate using the test data
'''
# Multinomial Naive Bayes
multiNB = MultinomialNB()
train_clf(multiNB, train_X, train_y)
predict_y, acc_nb = test_clf(multiNB, test_X, test_y)

# Logistic Regression
lg = LogisticRegression(penalty='l1')
train_clf(lg, train_X, train_y)
predict_y, acc_lg = test_clf(lg, test_X, test_y)

# K Nearest Neighbors
knn = KNeighborsClassifier(n_neighbors=10)
train_clf(knn, train_X, train_y)
predict_y, acc_knn = test_clf(knn, test_X, test_y)

# Support Vector Machine
svc = LinearSVC()
train_clf(svc, train_X, train_y)
predict_y, acc_svc = test_clf(svc, test_X, test_y)

# Random Forest
rf = RandomForestClassifier(n_estimators=60)
train_clf(rf, train_X, train_y)
predict_y, acc_rf = test_clf(rf, test_X, test_y)

print 'Multinomial Naive Bayes accuracy is', acc_nb
print 'Logistic Regression accuracy is', acc_lg
print 'K Nearest Neighbors accuracy is', acc_knn
print 'Support Vector Machine (Linear Kernel) accuracy is', acc_svc
print 'Random Forest accuracy is', acc_rf

# Visualize classifier performance
x = range(5)
y = [acc_nb, acc_lg, acc_knn, acc_svc, acc_rf]
clf_names = ['Multinomial Naive Bayes', 'Logistic Regression', \
             'K Nearest Neighbors', 'Support Vector Machine', 'Random Forest']

width = 0.6/1.2
plt.bar(x, y, width)

plt.title('Classifier Performance')
plt.xticks(x, clf_names, rotation=25)
plt.ylabel('Accuracy')


I am working on the file "Code/Assignment-5/Classification_Simulation.ipynb".
The first task is:
1. State assumptions about your data

X has size: n samples by m features.

X are i.i.d. random variables.

Features Xij (j = 1,...,m) are not identically distributed.

2. Formally define classification problem
Feature matrix X: Rn x m
Each sample Xi ∈ Rm, i ∈ [1, n]
Label yi ∈ {0, 1}, i ∈ [1, n]
g(X) → y
G := { g: R → {0, 1} }
Goal: g* = argming ∈ G E[L(g(X), y)], where L denotes loss function.
The loss function L differs for different classifiers and is speficied in the classfication context below:

Multinomial Naive Bayes: negative joint likelihood

L = -log p(X, y)

Logistic Regression:  logistic loss (cross-entropy loss)

L = -log P(y|g(X)) = -(y · log(g(X)) + (1 - y) · log(1 - g(X))

K Nearest Neighbors

L = ∑i D(Xi|yi=1, X|y=1) + ∑i D(Xi|yi=0, X|y=0)

D(a, b) = (a - b)2

Support Vector Machine: squared hinge loss

L = (max{0, 1 − y · g(x)})2

Random Forest

L = ∑i (g(Xi) - yi)2

Quadratic Discriminant Analysis

L = max{0, 1 − y · g(x)}

3. Provide algorithm for solving problem (including choosing hyperparameters as appropriate)

Logistic Regression

penalty = 'l1' (l1 norm for penalization)

K Nearest Neighbors

n_neighbors = 7

Support Vector Machine (Linear Kernel)

C: default = 1.0 (penalty parameter of the error term)

Random Forest

n_estimators = 20 (number of trees)

criterion: default = 'gini'

Quadratic Discriminant Analysis

None

Note: Generate random samples and plot accuracies based on Greg's code.
Simulation Setup
Can you write Python code for it?

np.random.seed(12345678) # for reproducibility, set random seed
r = 27  # define number of rois
N = 100 # number of samples at each iteration
p0 = 0.10
p1 = 0.15

# define number of subjects per class
S = np.array((8, 16, 20, 32, 40, 64, 80, 100, 120, 200, 320, 400, 800, 1000))


I am working on the file "Code/Assignment-5/Classification_Simulation.ipynb".
The first task is:
4 & 5. Sample data from a simulation setting & Compute Accuracy
Can you write Python code for it?

from sklearn.cross_validation import train_test_split
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import LinearSVC
from sklearn.ensemble import RandomForestClassifier
from sklearn.qda import QDA

[acc_LG, acc_KNN, acc_SVM, acc_RF, acc_QDA] = [[]  for i in xrange(5)]
[err_LG, err_KNN, err_SVM, err_RF, err_QDA] = [[]  for i in xrange(5)] # accuracy standard deviation

for idx1, s in enumerate(S):
    s0=s/2
    s1=s/2

    g0 = 1 * (np.random.rand(r, r, s0) > 1-p0)
    g1 = 1 * (np.random.rand(r, r, s1) > 1-p1)
    mbar0 = 1.0 * np.sum(g0, axis=(0,1))
    mbar1 = 1.0 * np.sum(g1, axis=(0,1))

    X = np.array((np.append(mbar0, mbar1), np.append(mbar0/(r**2), mbar1/(r**2)))).T
    y = np.append(np.zeros(s0), np.ones(s1))

    # Split the simulated data into training set and test set
    # Randomly sample 20% data as the test set
    train_X, test_X, train_y, test_y = cross_validation.train_test_split(X, y, test_size=0.2, random_state=42)

    # Logistic Regression
    lg = LogisticRegression(penalty='l1')
    acc, acc_std = loo_cv(lg, X, y)
    acc_LG.append(acc)
    err_LG.append(acc_std)

    # K Nearest Neighbors
    knn = KNeighborsClassifier(n_neighbors=7)
    acc, acc_std = loo_cv(knn, X, y)
    acc_KNN.append(acc)
    err_KNN.append(acc_std)

    # Support Vector Machine
    svc = LinearSVC()
    acc, acc_std = loo_cv(svc, X, y)
    acc_SVM.append(acc)
    err_SVM.append(acc_std)

    # Random Forest
    rf = RandomForestClassifier(n_estimators=20)
    acc, acc_std = loo_cv(rf, X, y)
    acc_RF.append(acc)
    err_RF.append(acc_std)

    # Quadratic Discriminant Analysis
    qda = QDA()
    acc, acc_std = loo_cv(qda, X, y)
    acc_QDA.append(acc)
    err_QDA.append(acc_std)


I am working on the file "Code/Assignment-5/Classification_Simulation.ipynb".
The first task is:
7. Apply method directly on real data
Can you write Python code for it?

# Not needed in this notebook, only used in an ealier verion, kept in case
def clean_negs(X):
    # Get indices of columns that contain negative values
    neg_col_inds = np.unique(np.where(X<0)[1])
    # Subtract minimum negative for each column
    for neg_i in neg_col_inds:
        neg_col = X[:, neg_i]
        min_neg = np.min(neg_col)
        new_col = [c - min_neg for c in neg_col]
        X[:, neg_i] = new_col
    return X

'''
Data Preparation
'''
from sklearn.cross_validation import train_test_split

real_X = df_feats.get_values()
real_y = df_labels.get_values()

print 'Dataset size is', real_X.shape

# Logistic Regression
lg = LogisticRegression(penalty='l1')
acc_lg, acc_std_lg = loo_cv(lg, real_X, real_y)

# K Nearest Neighbors
knn = KNeighborsClassifier(n_neighbors=7)
acc_knn, acc_std_knn = loo_cv(knn, real_X, real_y)

# Support Vector Machine
svc = LinearSVC()
acc_svm, acc_std_svm = loo_cv(svc, real_X, real_y)

# Random Forest
rf = RandomForestClassifier(n_estimators=20)
acc_rf, acc_std_rf = loo_cv(rf, real_X, real_y)

# Quadratic Discriminant Analysis
qda = QDA()
acc_qda, acc_std_qda = loo_cv(qda, real_X, real_y)

print 'Logistic Regression accuracy is %0.4f (+/- %0.3f)' % (acc_lg, acc_std_lg)
print 'K Nearest Neighbors accuracy is %0.4f (+/- %0.3f)' % (acc_knn, acc_std_knn)
print 'Support Vector Machine (Linear Kernel) accuracy is %0.4f (+/- %0.3f)' % (acc_svm, acc_std_svm)
print 'Random Forest accuracy is %0.4f (+/- %0.3f)' % (acc_rf, acc_std_rf)
print 'Quadratic Discriminant Analysis accuracy is %0.4f (+/- %0.3f)' % (acc_qda, acc_std_qda)

# Visualize classifier performance
x = range(5)
y = [acc_lg, acc_knn, acc_svm, acc_rf, acc_qda]
clf_names = ['Logistic Regression', 'K Nearest Neighbors', \
             'Support Vector Machine', 'Random Forest', 'Quadratic Discriminant Analysis']

width = 0.6/1.2
plt.bar(x, y, width)

plt.title('Classifier Performance')
plt.xticks(x, clf_names, rotation=25)
plt.ylabel('Accuracy')


I am working on the file "Code/Assignment-5/Classification_Simulation.ipynb".
The first task is:
Training & Test Utilities
Can you write Python code for it?

from sklearn import cross_validation
from sklearn.cross_validation import LeaveOneOut

# Train the given classifier
def train_clf(clf, train_feats, train_labels):
    # Supervised training
    clf.fit(train_feats, train_labels)

# Test the given classifier anc calculate accuracy
def test_clf(clf, test_feats, test_labels):
    # Predict using test set
    predicted = clf.predict(test_feats)
    # Compute accuracy
    acc = np.mean(predicted == test_labels)
    return predicted, acc

# Compute accuracy of a model trained with a specific number (n) of samples
def compute_acc(clf, n):
    train_clf(clf, train_X[:n], train_y[:n])
    predict_y, acc = test_clf(clf, test_X, test_y)
    return acc

# Leave one out cross validation
def loo_cv(clf, X, y):
    loo = LeaveOneOut(len(X))
    scores = cross_validation.cross_val_score(clf, X, y, cv=loo)
    return scores.mean(), scores.std()


I am working on the file "Code/Assignment-6/test_assumptions.ipynb".
The first task is:
Likely number of dependent entries for depressed set: 34/1205

Likely number of dependent entries for not depressed set: 64/1591

Given this info, it might be safe to say that the data entries are by and large independent
Indentical Samples Assumption

Gaussian Mixture Model for samples with reduced features
Can you write Python code for it?

import itertools
import matplotlib as mpl
from scipy import linalg
from sklearn.mixture import GMM

%matplotlib inline
import matplotlib.pyplot as plt

def plot_BIS(X, title):
    lowest_bic = np.infty
    bic = []
    n_components_range = range(1, 6)
    cv_types = ['spherical', 'tied', 'diag', 'full']
    for cv_type in cv_types:
        for n_components in n_components_range:
            # Fit a mixture of Gaussians with EM
            gmm = GMM(n_components=n_components, covariance_type=cv_type)
            gmm.fit(X)
            bic.append(gmm.bic(X))
            if bic[-1] < lowest_bic:
                lowest_bic = bic[-1]
                best_gmm = gmm

    bic = np.array(bic)
    color_iter = itertools.cycle(['k', 'r', 'g', 'b', 'c', 'm', 'y'])
    clf = best_gmm
    bars = []

    # Plot the BIC scores
    fig = plt.figure()
    fig.suptitle(title, fontsize=14)
    fig.set_size_inches(12, 4)
    plt.subplots_adjust(top=0.85)
    spl = plt.subplot(2, 1, 1)

    for i, (cv_type, color) in enumerate(zip(cv_types, color_iter)):
        xpos = np.array(n_components_range) + .2 * (i - 2)
        bars.append(plt.bar(xpos, bic[i * len(n_components_range):
                                      (i + 1) * len(n_components_range)],
                            width=.2, color=color))
    plt.xticks(n_components_range)
    plt.ylim([bic.min() * 1.01 - .01 * bic.max(), bic.max()])
    plt.title('BIC score per model')
    xpos = np.mod(bic.argmin(), len(n_components_range)) + .65 +\
        .2 * np.floor(bic.argmin() / len(n_components_range))
    plt.text(xpos, bic.min() * 0.97 + .03 * bic.max(), '*', fontsize=14)
    spl.set_xlabel('Number of components')
    spl.legend([b[0] for b in bars], cv_types, loc='best', prop={'size':10})

# Reduced Feature BIC
plot_BIS(red_depr, 'Reduced Depressed Features')
plot_BIS(red_not_depr, 'Reduced Non-Depressed Features')


I am working on the file "Code/Assignment-6/test_assumptions.ipynb".
The first task is:
Over here our reasoning is this, the probability for 2 lines to be selected twice in 2 differrent sets of 27 from 1205 candidates is pretty low. But when repeated 100 times there is a high chance for double counting. Allowing for this double counting if the number is still less than 5% of the total number of patients we can safely say that our data is mostly independent
Following a similar technique for the set of reduced not depressed people
Can you write Python code for it?

import numpy as np

# We don't want to ensure reproducibility, we want the sampling to be random everytime, this is because only then
# will iterative coverage of the rows of the large matrix be max

# Before we perform any identical-ness determination let's have a look at the shape of the matix
print "Shape of reduced features: " + str(red_not_depr.shape)

# variable to hold dependent count
dep_count = 0

for loop in range(0,100):

    # The number of entries far outnumber the number of reduced features. This almost guarantees linear dependence
    # To combat the let us random sample 27 values
    randrows = np.random.randint(1205,size=27)
    double_red_not_depr = red_not_depr[randrows]
    # print "Shape of double reduced features: " + str(double_red_depr.shape)

    # Now, let's calculate the rank of these matrix, if the rank is less than 27, then there are atleast 2 rows
    # that are dependent
    rank = np.linalg.matrix_rank(double_red_not_depr)
    if rank != 27:
        # print "Interdependent lines with matrank: " + str(rank)
        dep_count += 1
print "likely lower bound on dependent patients " + str(dep_count)


I am working on the file "Code/Assignment-6/test_assumptions.ipynb".
The first task is:
Plot BIC (Bayesian information criterion) scores to decide which covariance type to use.
The model with the lowest negation of the BIC (highest BIC score above) is preferred, therefore spherical covariance type is selected.
The definition on wiki leads to some confusion, the explanation can be found here: http://forum.hugin.com/index.php?topic=188.0.
Estimate clusters based on Greg's code.
Can you write Python code for it?

def evaluate_clusters(X):
    i = np.linspace(1, 16, 16, dtype='int')
    bic = np.array(())

    for idx in i:
        print "Fitting and evaluating model with " + str(idx) + " clusters."
        gmm = GMM(n_components=idx, n_iter=1000, covariance_type='spherical')
        gmm.fit(X)
        bic = np.append(bic, gmm.bic(X))
    plt.figure(figsize=(7, 7))
    plt.plot(i, 1.0/bic)
    plt.title('BIC')
    plt.ylabel('score')
    plt.xlabel('number of clusters')
    plt.show()

    print bic

evaluate_clusters(red_depr)

evaluate_clusters(red_not_depr)


I am working on the file "Code/Assignment-8/Independent Analysis.ipynb".
The first task is:
As is evident from the above 2 experiments, no clear clustering is apparent.But there is some significant overlap and there 2 clear groups
Classification Experiments
Let's experiment with a bunch of classifiers
Can you write Python code for it?

# support methods
# Train the given classifier
def train_clf(clf, train_feats, train_labels):
    # Supervised training
    clf.fit(train_feats, train_labels)

# Test the given classifier anc calculate accuracy
def test_clf(clf, test_feats, test_labels):
    # Predict using test set
    predicted = clf.predict(test_feats)
    # Compute accuracy
    acc = np.mean(predicted == test_labels)
    return predicted, acc

# Compute accuracy of a model trained with a specific number (n) of samples
def compute_acc(clf, n):
    train_clf(clf, train_X[:n], train_y[:n])
    predict_y, acc = test_clf(clf, test_X, test_y)
    return acc

# Leave one out cross validation
def loo_cv(clf, X, y):
    loo = LeaveOneOut(len(X))
    scores = cross_validation.cross_val_score(clf, X, y, cv=loo)
    return scores.mean(), scores.std()

BP_reduced_df['ADHD-Bipolar'] = 0
ADHD_reduced_df['ADHD-Bipolar'] = 1

data = pd.concat([ADHD_reduced_df, BP_reduced_df])
class_labels = data['ADHD-Bipolar']
data = data.drop(['ADHD-Bipolar'], axis = 1, inplace = False)
print data.shape
data = data.get_values()

# Logistic Regression
lg = LogisticRegression(penalty='l2')
acc_lg, acc_std_lg = loo_cv(lg, data, class_labels)

# K Nearest Neighbors
knn = KNeighborsClassifier(n_neighbors=7)
acc_knn, acc_std_knn = loo_cv(knn, data, class_labels)

# Support Vector Machine
svc = SVC(gamma=2, C=1)
acc_svm, acc_std_svm = loo_cv(svc, data, class_labels)

# Random Forest
rf = RandomForestClassifier(n_estimators=20)
acc_rf, acc_std_rf = loo_cv(rf, data, class_labels)

# Random Forest
dt = DecisionTreeClassifier(max_depth = 20)
acc_dt, acc_std_dt = loo_cv(dt, data, class_labels)

# Quadratic Discriminant Analysis
qda = QDA()
acc_qda, acc_std_qda = loo_cv(qda, data, class_labels)

# Gaussian Naive Bayes
gnb = GaussianNB()
acc_gnb, acc_std_gnb = loo_cv(gnb, data, class_labels)

print 'Logistic Regression accuracy is %0.4f (+/- %0.3f)' % (acc_lg, acc_std_lg)
print 'K Nearest Neighbors accuracy is %0.4f (+/- %0.3f)' % (acc_knn, acc_std_knn)
print 'Support Vector...


I am working on the file "Code/Assignment-8/Independent Analysis.ipynb".
The first task is:
Though I'm not sure how to tweak the hyper-parameters of the bandwidth estimation function, there doesn't seem to be much difference. Minute variations to the bandwidth result in large cluster differences. Perhaps the data isn't very suitable for a contrived clustering technique like Mean-Shift. Therefore let us attempt something more naive and simplistic like K-Means
K-Means clustering
Can you write Python code for it?

kmeans = KMeans(n_clusters=2)
kmeans.fit(data.get_values())
labels = kmeans.labels_
centroids = kmeans.cluster_centers_
print('Estimated number of clusters: %d' % len(centroids))
print data.shape

for i in range(2):
    # Extract observations within each cluster
    ds = data.get_values()[np.where(labels==i)]
    # Plot the observations with symbol o
    plt.plot(ds[:,0], ds[:,1], 'o')
    # Plot the centroids with simbol x
    lines = plt.plot(centroids[i,0], centroids[i,1], 'x')
    plt.setp(lines, ms=8.0)
    plt.setp(lines, mew=2.0)


I am working on the file "Code/Assignment-8/Independent Analysis.ipynb".
The first task is:
We see here that most of the variance is preserved with just 24 features.
Clustering and other grouping experiments
Mean-Shift
Can you write Python code for it?

ADHD_clust = pd.DataFrame(ADHD)
BP_clust = pd.DataFrame(BP)

# This is a consequence of how we dropped columns, I apologize for the hacky code
data = pd.concat([ADHD_clust, BP_clust])

# Let's see what happens with Mean Shift clustering
bandwidth = estimate_bandwidth(data.get_values(), quantile=0.9, n_samples=1430) * 1.1
ms = MeanShift(bandwidth=bandwidth)
ms.fit(data.get_values())
labels = ms.labels_

cluster_centers = ms.cluster_centers_
labels_unique = np.unique(labels)
n_clusters_ = len(labels_unique)
print('Estimated number of clusters: %d' % n_clusters_)

for i in range(n_clusters_):
    # Extract observations within each cluster
    ds = data.get_values()[np.where(labels==i)]
    # Plot the observations with symbol o
    plt.plot(ds[:,0], ds[:,1], 'o')
    # Plot the centroids with simbol x
    lines = plt.plot(cluster_centers[i,0], cluster_centers[i,1], 'x')
    plt.setp(lines, ms=8.0)
    plt.setp(lines, mew=2.0)


I am working on the file "Code/Assignment-8/Independent Analysis.ipynb".
The first task is:
we see here that there 1110 people who have ADHD but are not Bipolar and 329 people who are Bipolar but do not have ADHD
Dimensionality reduction
Can you write Python code for it?

pca = PCA(n_components = 24, whiten = "True").fit(ADHD)
ADHD = pca.transform(ADHD)
print sum(pca.explained_variance_ratio_)

pca = PCA(n_components = 24, whiten = "True").fit(BP)
BP = pca.transform(BP)
print sum(pca.explained_variance_ratio_)

print ADHD.shape
print BP.shape

ADHD_reduced_df = pd.DataFrame(ADHD)

BP_reduced_df = pd.DataFrame(BP)

ADHD_reduced_df['Patient_ID'] = 123
BP_reduced_df['Patient_ID'] = 123

print ADHD_reduced_df.shape
print BP_reduced_df.shape

# resorting to some hacky crap, that I am ashamed to write, but pandas is refusing to cooperate
z = []
for x in BP_df['Patient_ID']:
    z.append(x)
BP_reduced_df['Patient_ID'] = z

z = []
for x in ADHD_df['Patient_ID']:
    z.append(x)
ADHD_reduced_df['Patient_ID'] = z


I am working on the file "Code/Assignment-8/Independent Analysis.ipynb".
The first task is:
we've now dropped the last of the discrete numerical inexplicable data, and removed children from the mix
Extracting the samples we are interested in
Can you write Python code for it?

# Let's extract ADHd and Bipolar patients (mutually exclusive)

ADHD = X.loc[X['ADHD'] == 1]
ADHD = ADHD.loc[ADHD['Bipolar'] == 0]

BP = X.loc[X['Bipolar'] == 1]
BP = BP.loc[BP['ADHD'] == 0]

# Keeping a backup of the data frame object because numpy arrays don't play well with certain scikit functions
ADHD_df = ADHD
BP_df = BP
ADHD = ADHD.drop(['Patient_ID'], axis = 1, inplace = False)
BP = BP.drop(['Patient_ID'], axis = 1, inplace = False)


I am working on the file "Code/Assignment-8/extended_analysis_ADHD_Bipolar_yatbear.ipynb".
The first task is:
As we can see from the plotted power curves above, ten features (with large p-values) obviously follow the null distribution. For those whose p-values are slightly smaller than the alpha power(0.05), it is hard to be tell whether the corresponding powers will eventually approximate 1.0 due to the lack of data, but we can still see the trend of increasing.
Predictive Analysis
Predict ADHD disorder against Bipolar disorder using 10-fold Cross Validation.
Since we have 1040 ADHD samples out of 1347 samples, the chance is 77.2%, we want the prediction accuracy much larger than the chance.
Q6 - Q14. What are the performances of different classification methods on the current features?

Logistic Regression Classifier
K Nearest Neighbors Classifier
Support Vector Machine (linear kernel)
Linear Discriminant Analysis
Quadratic Discriminant Analysis
Random Forest Classifier
Gradient Boosting Classifier
Extra Trees Classifier
AdaBoost Classifier
Can you write Python code for it?

from sklearn import cross_validation
from sklearn.cross_validation import LeaveOneOut
from sklearn.cross_validation import KFold

# Train the given classifier
def train_clf(clf, train_feats, train_labels):
    # Supervised training
    clf.fit(train_feats, train_labels)

# Test the given classifier and calculate accuracy
def test_clf(clf, test_feats, test_labels):
    # Predict using test set
    predicted = clf.predict(test_feats)
    # Compute accuracy
    acc = np.mean(predicted == test_labels)
    return predicted, acc

# Compute accuracy of a model trained with a specific number(n) of samples
def compute_acc(clf, n):
    train_clf(clf, train_X[:n], train_y[:n])
    predict_y, acc = test_clf(clf, test_X, test_y)
    return acc

# Leave one out cross validation
def loo_cv(clf, X, y):
    loo = LeaveOneOut(len(X))
    scores = cross_validation.cross_val_score(clf, X, y, cv=loo)
    return scores.mean(), scores.std()

# K-fold cross validation
def kf_cv(clf, X, y, k=10):
    kf = KFold(len(X), n_folds=k)
    scores = cross_validation.cross_val_score(clf, X, y, cv=kf)
    return scores.mean(), scores.std()

df_all = pd.concat([df_adhd, df_bipolar], axis=0)
X = df_all.values
y = [0] * df_adhd.shape[0] + [1] * df_bipolar.shape[0]

print 'Double check data size:', X.shape, len(y)

from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import LinearSVC, SVC
from sklearn.lda import LDA
from sklearn.qda import QDA
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import AdaBoostClassifier

# Logistic Regression
lg = LogisticRegression(penalty='l2')
acc_lg, acc_std_lg = kf_cv(lg, X, y)

# K Nearest Neighbors
knn = KNeighborsClassifier(n_neighbors=7)
acc_knn, acc_std_knn = kf_cv(knn, X, y)

# Support Vector Machine - Linear Kernel
svc = LinearSVC()
acc_svm, acc_std_svm = kf_cv(svc, X, y)

# Linear Discriminant...


I am working on the file "Code/Assignment-8/extended_analysis_ADHD_Bipolar_yatbear.ipynb".
The first task is:
Compute p-values on the reduced data
Can you write Python code for it?

p_vals = list()
for a, b in zip(adhd, bipolar):
    p_vals.append(round(ks_2samp(a, b)[1], 5))
print p_vals