Kaggle competition: OSIC Pulmonary Fibrosis Progression
predictions = vclf.predict(test_X)
 predictions = lb.inverse_transform(predictions)
 predictions_final = pd.DataFrame({'cuisine' : predictions , 'id' : test_data.id }, columns=['id', 'cuisine'])
 predictions_final.to_csv('Final_submission.csv', index = False )<set_options>
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import linear_model
import statsmodels.api as sm
init_notebook_mode(connected=True)
 %matplotlib inline
color = sns.color_palette() 
<load_from_csv>
BASE_PATH = '.. /input/osic-pulmonary-fibrosis-progression'
data_train_dir = f'{BASE_PATH}/train'
data_test_dir = f'{BASE_PATH}/test'
train = pd.read_csv(f'{BASE_PATH}/train.csv')
 test = pd.read_csv(f'{BASE_PATH}/test.csv' )
train = pd.read_json('.. /input/train.json')
 test = pd.read_json('.. /input/test.json')
 sub = pd.read_csv('.. /input/sample_submission.csv' )<sort_values>
sample_submission = pd.read_csv(f'{BASE_PATH}/sample_submission.csv' )
total = train.isnull().sum().sort_values(ascending = False)
 percent =(train.isnull().sum() /train.isnull().count() *100 ).sort_values(ascending = False)
 missing_train_data = pd.concat([total, percent], axis=1, keys=['Total missing', 'Percent missing'])
 missing_train_data.head(20 )<sort_values>
patient_dict = {}
def init_fvc(row):
if row['Patient'] not in patient_dict.keys() :
patient_dict[row['Patient']] = row['FVC']
return row['FVC']
else:
return patient_dict[row['Patient']]
train['InitFVC'] = train.apply(lambda row: init_fvc(row), axis=1)
 train.head(20 )
total = test.isnull().sum().sort_values(ascending = False)
 percent =(test.isnull().sum() /test.isnull().count() *100 ).sort_values(ascending = False)
 missing_test_data = pd.concat([total, percent], axis=1, keys=['Total missing', 'Percent missing'])
 missing_test_data.head(20 )<load_from_csv>
patient_dict = {}
def init_week(row):
if row['Patient'] not in patient_dict.keys() :
patient_dict[row['Patient']] = row['Weeks']
return row['Weeks']
else:
return patient_dict[row['Patient']]
train['InitWeeks'] = train.apply(lambda row: init_week(row), axis=1)
 train.head(20 )
def read_dataset(path):
	return json.load(open(path))
train = read_dataset('.. /input/train.json')
 test = read_dataset('.. /input/test.json')
 def generate_text(data):
	text_data = [" ".join(doc['ingredients'] ).lower() for doc in data]
	return text_data
train_text = generate_text(train)
 test_text = generate_text(test)
 target = [doc['cuisine'] for doc in train] 
<feature_engineering>
patient_dict = {}
def init_percent(row):
if row['Patient'] not in patient_dict.keys() :
patient_dict[row['Patient']] = row['Percent']
return row['Percent']
else:
return patient_dict[row['Patient']]
train['InitPercent'] = train.apply(lambda row: init_percent(row), axis=1)
 train.head(20 )
tfidf = TfidfVectorizer(binary=True)
 def tfidf_features(txt, flag):
if flag == "train":
	x = tfidf.fit_transform(txt)
 else:
	 x = tfidf.transform(txt)
 x = x.astype('float16')
 return x
X = tfidf_features(train_text, flag="train")
 X_test = tfidf_features(test_text, flag="test" )<categorify>
train_df = pd.get_dummies(train, columns=['Sex', 'SmokingStatus'], prefix=['Sex', 'SmokingStatus'] )
lb = LabelEncoder()
y = lb.fit_transform(target)
 classifier = SVC(C=100,
	 			 kernel='rbf',
	 			 degree=3,
	 			 gamma=1,
	 			 coef0=1,
	 			 shrinking=True,
	 			 tol=0.001,
	 		 probability=False,
	 		 cache_size=200,
	 		 class_weight=None,
	 		 verbose=False,
	 		 max_iter=-1,
		 decision_function_shape=None,
		 random_state=None)
 model = OneVsRestClassifier(classifier, n_jobs=4)
 model.fit(X, y)
 y_test = model.predict(X_test)
 y_pred = lb.inverse_transform(y_test)
 test_id = [doc['id'] for doc in test]
sub = pd.DataFrame({'id': test_id, 'cuisine': y_pred}, columns=['id', 'cuisine'])
 sub.to_csv('sub1.csv', index=False )<import_modules>
data = []
for i in range(-12, 133+1):
for index, row in test.iterrows() :
new_cols = list(test.columns)
 new_cols.append('InitWeeks')
 new_vals = [row['Patient'], i, row['FVC'], row['Percent'],row['Age'],row['Sex'],row['SmokingStatus'], row['Weeks']]
data.append(dict(zip(new_cols, new_vals)))
 test_df = pd.DataFrame(data)
 test_df.head(10 )
random_state = 7
def read_dataset(path):
return json.load(open(path, encoding='utf-8')) 
<prepare_x_and_y>
X = train[['Weeks']]
Y = train['FVC']
regr = linear_model.LinearRegression()
regr.fit(X, Y)
 print('Intercept:
', regr.intercept_)
 print('Coefficients:
', regr.coef_)
 model = sm.OLS(Y, X ).fit()
print(model.summary() )
print("Read Dataset...")
 df_train_raw=pd.read_json(os.path.join(challenge_path,'train.json')).set_index('id')
 df_train_raw['invalid']=[False] *(len(df_train_raw))
x_field='ingredients'
y_field='cuisine'
df_train_raw=df_train_raw.loc[df_train_raw.cuisine !=None]
df_train_raw.head()<string_transform>
X_test = test[['Weeks']]
Y_test = test['FVC']
print('Predicted FVCs:
', regr.predict(X_test))
print('Actual FVCs:
', Y_test )
lemmatizer = WordNetLemmatizer()
def preprocess(ingredients):
ingredients_text = ' '.join(ingredients)
 ingredients_text = ingredients_text.lower()
ingredients_text = ingredients_text.replace('-', ' ')
 words = []
for word in ingredients_text.split() :
if re.findall('[0-9]', word): continue
if len(word)<= 2: continue
if '’' in word: continue
word = lemmatizer.lemmatize(word)
 if len(word)> 0: words.append(word)
 return ' '.join(words)
 ing=df_train_raw[x_field].values.tolist()
cou=df_train_raw[y_field].values.tolist()
find_duplicated=False
dup=0
texts=[]
for i in range(len(ing)) :
if df_train_raw['invalid'].values[i]==False:
if len(ing[i])<1 or pd.isna(cou[i]):
df_train_raw['invalid'].values[i]=True
elif find_duplicated:
if ing[i] in ing[i+1:]:
dup+=1
print("found dup at {}".format(i))
df_train_raw['invalid'].values[i]=True
newtext=preprocess(df_train_raw[x_field].values[i])
 texts.append(newtext)
 df_train_raw['texts']=texts<feature_engineering>
X = train[['Weeks', 'InitFVC']]
Y = train['FVC']
regr = linear_model.LinearRegression()
regr.fit(X, Y)
 print('Intercept:
', regr.intercept_)
 print('Coefficients:
', regr.coef_)
 model = sm.OLS(Y, X ).fit()
print(model.summary() )
len_orig=[len(x)for x in df_train_raw[x_field].values.tolist() ]
len_proc=[len(x.split(' ')) for x in texts]
df_train_raw['len_orig']=len_orig
df_train_raw['len_proc']=len_proc
df_train_raw.head()<count_duplicates>
X_test = test_df[['Weeks', 'FVC']]
Y_test = test_df['FVC']
Y_pred = regr.predict(X_test)
 ', Y_pred)
 ', Y_test )
df_train=df_train_raw.loc[df_train_raw.invalid == False]
dup,len(df_train),len(df_train_raw )<load_from_disk>
X = train[['Weeks', 'InitFVC', 'InitWeeks']]
Y = train['FVC']
regr = linear_model.LinearRegression()
regr.fit(X, Y)
 print('Intercept:
', regr.intercept_)
 print('Coefficients:
', regr.coef_)
 model = sm.OLS(Y, X ).fit()
print(model.summary() )
df_submission = pd.read_json(os.path.join(challenge_path,'test.json')).set_index('id')
 df_submission.head()<categorify>
X_test = test_df[['Weeks', 'FVC', 'InitWeeks']]
Y_test = test_df['FVC']
Y_pred = regr.predict(X_test)
 ', Y_pred)
 ', Y_test )
y_j=df_train[y_field].values
x_j=df_train['texts'].values
for i in range(len(df_submission)) :
df_submission[x_field].values[i]=preprocess(df_submission[x_field].values[i])
 x_j_sub = df_submission[x_field].values<categorify>
X = train[['Weeks', 'InitFVC','InitWeeks']]
Y = train['FVC']
regr = linear_model.LinearRegression()
regr.fit(X, Y)
 print('Intercept:
', regr.intercept_)
 print('Coefficients:
', regr.coef_)
 model = sm.OLS(Y, X ).fit()
print(model.summary() )
y_lbr=LabelBinarizer()
y_j_conv=y_lbr.fit_transform(y_j)
 y_classes=y_lbr.classes_
y_classes_transformed=y_lbr.transform(y_classes )<normalization>
X = train[['Weeks', 'InitFVC', 'InitWeeks', 'Age']]
Y = train['FVC']
regr = linear_model.LinearRegression()
regr.fit(X, Y)
 print('Intercept:
', regr.intercept_)
 print('Coefficients:
', regr.coef_)
 model = sm.OLS(Y, X ).fit()
print(model.summary() )
class_weights = class_weight.compute_class_weight('balanced', y_classes, y_j)
 class_weights={i:class_weights[i] for i in range(len(y_lbr.classes_)) }
class_weights<feature_engineering>
X_test = test_df[['Weeks', 'FVC', 'InitWeeks', 'Age']]
Y_test = test_df['FVC']
Y_pred = regr.predict(X_test)
 ', Y_pred)
 ', Y_test )
def tokenize(text):
tokens = nltk.word_tokenize(text)
 return tokens
x_cvr = TfidfVectorizer(analyzer='word', tokenizer=tokenize, token_pattern=None, stop_words='english')
 x_cvr.fit(x_j)
 x_analyze = x_cvr.build_analyzer()
x_j_conv =x_cvr.transform(x_j ).toarray()
x_j_sub_conv=x_cvr.transform(x_j_sub ).toarray() 
<split>
X = train_df[['Weeks', 'InitFVC', 'InitWeeks', 'Age', 'SmokingStatus_Currently smokes']]
Y = train_df['FVC']
regr = linear_model.LinearRegression()
regr.fit(X, Y)
 print('Intercept:
', regr.intercept_)
 print('Coefficients:
', regr.coef_)
 model = sm.OLS(Y, X ).fit()
print(model.summary() )
xt,xv,yt,yv,yjt,yjv=train_test_split(x_j_conv,y_j_conv,y_j, test_size=test_size, stratify=y_j, random_state=7, shuffle=True )<choose_model_class>
data = []
for i in range(test_df.shape[0]):
new_cols = ['Patient_Week', 'FVC', 'Confidence']
new_vals = [test_df.iloc[i]['Patient']+"_"+str(test_df.iloc[i]['Weeks']), Y_pred[i], 244]
data.append(dict(zip(new_cols, new_vals)))
 submission = pd.DataFrame(data)
 submission.head(95)
K.clear_session()
np.random.seed(7)
 do=0.50
i = Input(shape=xt.shape[1:])
 x=Dropout(do-0.1 )(i)
 x=Dense(640 )(x)
 x=BatchNormalization()(x)
 x=Activation('relu' )(x)
 x=Dropout(do )(x)
 o=Dense(yt.shape[1],activation='linear' )(x)
 o=Dense(yt.shape[1],activation='softmax' )(o)
 model=Model(inputs=[i,], outputs=[o,])
 model.summary()<choose_model_class>
submission.to_csv('submission.csv', index=False )
<load_pretrained><EOS> 
submission.to_csv('submission.csv', index=False )