Kaggle competition: AI Academy Intermediate Class Competition 1
<SOS> metric: categorizationaccuracy Kaggle data source: ai-academy-intermediate-class-competition-1<compute_train_metric>
data = pd.read_csv(".. /input/ai-academy-intermediate-class-competition-1/BBC News Train.csv")
 data = data[["Text", "Category"]]
data
def logistic_curve2(x, k1, k2, x_0, ymax):
return ymax /(1 + np.exp(-k1*(x-x_0)) + np.exp(-k2*(x-x_0)) )<data_type_conversions>
vectorizer = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 X = vectorizer.fit_transform(data["Text"])
 print(len(vectorizer.get_feature_names()))
print(X.shape )
list_countries = dftrain[dftrain['Date'] == '2020-01-22']['Country_Region'].tolist()
list_states = dftrain[dftrain['Date'] == '2020-01-22']['Province_State'].tolist()
datenow = datetime.now()<data_type_conversions>
data["category_id"]=data["Category"].factorize() [0]
list_date_pand = [] ; list_maxcases = []; list_maxfat = []
for country, state in list(zip(list_countries, list_states)) :
df2 = dftrain.loc[(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)].fillna('None')
 maxcases, maxfat = df2['ConfirmedCases'].max() , df2['Fatalities'].max()
date_pand2 = []
date_pand = df2[df2['ConfirmedCases'] > 100.]['Date'].tolist()
try:
list_date_pand.append(pd.to_datetime(date_pand[0]))
except:
list_date_pand.append(pd.to_datetime(datenow))
list_maxcases.append(maxcases); list_maxfat.append(maxfat)
 dfstartpand = pd.DataFrame(np.array([list_countries, list_states, list_date_pand, list_maxcases, list_maxfat] ).T,
columns=['Country_Region', 'Province_State', 'Start_Pandemic', 'ConfirmedCases', 'Fatalities'])
 dfstartpand['Start_Pandemic'] = dfstartpand['Start_Pandemic'].dt.date<sort_values>
data = data[["Text", "category_id", "Category"]]
data
category_id_data = data[['Category', 'category_id']].drop_duplicates().sort_values('category_id')
 category_to_id = dict(category_id_data.values)
 id_to_category = dict(category_id_data[['category_id', 'Category']].values)
 category_id_data
id_to_category
dfstartpand_ordered = dfstartpand.sort_values(by=['Start_Pandemic', 'ConfirmedCases', 'Fatalities'], ascending=[True, False, False])
 country_state_ordered = list(zip(dfstartpand_ordered['Country_Region'].tolist() , dfstartpand_ordered['Province_State']))
datetrain = dftrain['Date'].unique()
datetest = dftest['Date'].unique()<feature_engineering>
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', encoding='latin-1', ngram_range=(1, 2), stop_words='english')
 features = tfidf.fit_transform(data.Text ).toarray()
print(features)
 labels = data.category_id
print(labels)
 features.shape
dftest['ConfirmedCases_logreg'] = 0.0 ; dftrain['ConfirmedCases_logreg'] = 0.0
dftest['Fatalities_logreg'] = 0.0 ; dftrain['Fatalities_logreg'] = 0.0
p0 = 1
for country, state in country_state_ordered:
masktrain =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)
 masktrain2 =(dftrain['Country_Region'] == country)&(dftrain['Province_State'] == state)& \ 
(dftrain['Date'] <= '2020-04-07')&(dftrain['Date'] >= starttest)
 masktest =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)
 masktest2 =(dftest['Country_Region'] == country)&(dftest['Province_State'] == state)& \ 
(dftest['Date'] <= '2020-04-07')
 df2plot = dftrain[masktrain].set_index('Date')
 X = np.arange(len(df2plot))
X_test =(np.timedelta64(datetest[0]-datetrain[0], 'D')).astype(float)+np.arange(0,len(datetest))
try:
y = df2plot['ConfirmedCases']
p0_cases = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_cases,bounds=([0,0,0],np.inf), maxfev=1000)
 k_cases, x_0_cases, ymax_cases = popt
cases_train_fc = pd.Series(logistic_curve(X, k_cases, x_0_cases, ymax_cases),index=df2plot.index)
 cases_test_fc = pd.Series(logistic_curve(X_test, k_cases, x_0_cases, ymax_cases),index=datetest)
 dftest.loc[masktest,'ConfirmedCases_logreg'] = cases_test_fc.tolist()
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = cases_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the confirmed cases')
 dftest.loc[masktest,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases'].iloc[-1]
dftrain.loc[masktrain,'ConfirmedCases_logreg'] = dftrain.loc[masktrain,'ConfirmedCases']
try:
y = df2plot['Fatalities']
p0_deaths = [1/(len(X)/2.) , X[-1], y.max() ]
popt, pcov = curve_fit(logistic_curve, X, y, p0=p0_deaths,bounds=([0,0,0],np.inf), maxfev=1000)
 k_deaths, x_0_deaths, ymax_deaths = popt
deaths_train_fc = pd.Series(logistic_curve(X, k_deaths, x_0_deaths, ymax_deaths),index=datetrain)
 deaths_test_fc = pd.Series(logistic_curve(X_test, k_deaths, x_0_deaths, ymax_deaths),index=datetest)
 dftest.loc[masktest,'Fatalities_logreg'] = deaths_test_fc.tolist()
dftrain.loc[masktrain,'Fatalities_logreg'] = deaths_train_fc.tolist()
except:
print(country+' '+state+' Unable to fit the fatalities')
 dftest.loc[masktest,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities'].iloc[-1]
dftrain.loc[masktrain,'Fatalities_logreg'] = dftrain.loc[masktrain,'Fatalities']
dftest.loc[masktest2,'ConfirmedCases_logreg'] = dftrain.loc[masktrain2,'ConfirmedCases'].tolist()
dftest.loc[masktest2,'Fatalities_logreg'] = dftrain.loc[masktrain2,'Fatalities'].tolist()<data_type_conversions>
N = 5
for category, category_id in sorted(category_to_id.items()):
features_chi2 = chi2(features, labels == category_id)
 indices = np.argsort(features_chi2[0])
 feature_names = np.array(tfidf.get_feature_names())[indices]
unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
print("
print(".Most correlated unigrams: 
.{}".format(' 
.'.join(unigrams[-N:])))
 print(".Most correlated bigrams: 
.{}".format(' 
.'.join(bigrams[-N:])) )
dfsubm['ConfirmedCases'] = dftest['ConfirmedCases_logreg']
dfsubm['Fatalities'] = dftest['Fatalities_logreg']<save_to_csv>
X_train, X_test, y_train, y_test, indices_train, indices_test = train_test_split(features, labels, data.index, test_size=0.20, random_state=0)
 for model in models:
model.fit(X_train, y_train)
 y_pred_proba = model.predict_proba(X_test)
 y_pred = model.predict(X_test)
 accuracy = accuracy_score(y_test, y_pred)
 print(accuracy )
dfsubm.to_csv('submission.csv', index=False )<load_from_csv>
model = models[2]
model.fit(features, labels)
 model.coef_
pd.set_option('display.max_columns', 500)
 pd.set_option('display.max_rows', 500)
 df_train = pd.read_csv('.. /input/titanic/train.csv')
 df_test = pd.read_csv('.. /input/titanic/test.csv')
 df_sub = pd.read_csv('.. /input/titanic/gender_submission.csv' )<concatenate>
test_data = pd.read_csv(".. /input/bbc-test-3/BBC News Test.csv")
 test_data
df = pd.concat([df_train, df_test],sort=False)
 df.reset_index(drop=True,inplace=True )<count_missing_values>
test_data.Text.tolist()
test_features = tfidf.transform(test_data.Text.tolist())
 Y_pred = model.predict(test_features)
 Y_pred
submission = []
for pred in Y_pred:
submission.append(id_to_category[pred])
 submission
df.isnull().sum()<filter>
submission = pd.DataFrame({
"ArticleId": test_data["ArticleId"],
"Category": submission
})
 submission
df[df['Fare'].isnull() ]<correct_missing_values>
submission.to_csv('submission.csv', index=False )
df_dropna = df.dropna(subset=['Fare'] )<count_values>
submission.to_csv('submission.csv', index=False )