Kaggle competition: Titanic - Machine Learning from Disaster
<SOS> metric: categorizationaccuracy Kaggle data source: titanic-machine-learning-from-disaster<load_from_csv>
style.use('https://raw.githubusercontent.com/JoseGuzman/minibrain/master/minibrain/paper.mplstyle')
 train = pd.read_csv(".. /input/titanic/train.csv", index_col='PassengerId')
 test = pd.read_csv('.. /input/titanic/test.csv', index_col='PassengerId' )
corona_data=pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 choro_map=px.choropleth(corona_data,
locations="Country/Region",
locationmode = "country names",
color="Confirmed",
hover_name="Country/Region",
animation_frame="ObservationDate" 
)
 choro_map.update_layout(
 title_text = 'Global Spread of Coronavirus',
title_x = 0.5,
geo=dict(
 showframe = False,
showcoastlines = False, 
))
choro_map.show()<load_from_csv>
mydf = train.groupby('Survived' ).mean()
mydf
zone=pd.read_csv('/kaggle/input/covid-19-india-zone-classification/lockdownindiawarningzones.csv' )<load_from_csv>
train.groupby('Sex')['Survived'].mean()
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 statewise_cases = pd.DataFrame(covid_India_cases.groupby(['State'])['Confirmed', 'Deaths', 'Recovered'].max().reset_index())
 statewise_cases["Country"] = "India"
fig = px.treemap(statewise_cases, path=['Country','State'], values='Confirmed',
color='Confirmed', hover_data=['State'],
color_continuous_scale='Rainbow')
 fig.show()<load_from_csv>
from sklearn.pipeline import Pipeline
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 statewise_cases = pd.DataFrame(covid_India_cases.groupby(['State'])['Confirmed', 'Deaths', 'Recovered'].max().reset_index())
 last=statewise_cases
pos=pd.read_csv('.. /input/utm-of-india/UTM ZONES of INDIA.csv')
 ind_grp=last.merge(pos , left_on='State', right_on='State / Union Territory' )<load_from_csv>
class FeaturesTransformer() :

 def __init__(self, create_family=True, dissect_cabin = True):

 self.create_family = create_family
self.dissect_cabin = dissect_cabin
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
mydf['Title'] = X.Name.str.extract('([A-Za-z]+)\.', expand=False)
 mydf.drop(['Name'], axis = 1, inplace = True)
 mydf['Title'].replace(['Ms','Lady', 'Countess','Dona'],'Mrs', inplace=True)
 mydf['Title'].replace(['Mme','Mlle'], 'Miss', inplace = True)
 mydf['Title'].replace(['Major', 'Sir', 'Jonkheer', 'Dr','Col','Don', 'Capt','Rev'],'Mr', inplace=True)
 if self.create_family:
mydf['FamilySize'] = X['SibSp'] + X['Parch'] + 1
mydf.drop(['SibSp', 'Parch'], axis=1, inplace=True)
 if self.dissect_cabin:
mydf['Cabin'].fillna('N', inplace=True)
 mydf['Room'] = X['Cabin'].str.extract("([0-9]+)", expand=False ).astype("float")
 mydf['nRooms'] = mydf.Cabin.apply(lambda x: len(str(x ).split()))
mydf.loc[mydf.Cabin == 'N', 'nRooms'] = 0
mydf['CabinDeck'] = mydf.Cabin.apply(lambda x: str(x[0]))
mydf['CabinDeck'].replace(['T'], 'A', inplace=True)
 del mydf['Cabin']
return mydf
ind_map=pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 pos=pd.read_csv('.. /input/utm-of-india/UTM ZONES of INDIA.csv')
 ind_map1=ind_map.merge(pos , left_on='State/UnionTerritory', right_on='State / Union Territory')
<set_options>
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True))
])
 mytrain = preprocess.fit_transform(train )
warnings.filterwarnings('ignore')
 %matplotlib inline<load_from_csv>
mytrain.groupby('Survived')['Title'].value_counts()
age_details = pd.read_csv('.. /input/covid19-in-india/AgeGroupDetails.csv')
 india_covid_19 = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 hospital_beds = pd.read_csv('.. /input/covid19-in-india/HospitalBedsIndia.csv')
 individual_details = pd.read_csv('.. /input/covid19-in-india/IndividualDetails.csv')
 ICMR_labs = pd.read_csv('.. /input/covid19-in-india/ICMRTestingLabs.csv')
 state_testing = pd.read_csv('.. /input/covid19-in-india/StatewiseTestingDetails.csv')
<load_from_csv>
class FillerTransformer() :

 def __init__(self, mean=True):

 self.mean = mean
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
mydf['Embarked'] = mydf['Embarked'].fillna('C')
 mydf['Fare'].fillna(mydf['Fare'].median() , inplace = True)
 if self.mean:
mydf['Age'] = mydf.groupby(['Pclass','Title'])['Age'].apply(lambda x: x.fillna(x.mean()))
else:
mydf['Age'] = mydf.groupby(['Pclass','Title'])['Age'].apply(lambda x: x.fillna(x.median()))
return mydf
confirmed_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_confirmed_global.csv')
 deaths_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_deaths_global.csv')
 recovered_df = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_time_series/time_series_covid19_recovered_global.csv')
 latest_data = pd.read_csv('https://raw.githubusercontent.com/CSSEGISandData/COVID-19/master/csse_covid_19_data/csse_covid_19_daily_reports/04-04-2020.csv' )<data_type_conversions>
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=True))
])
 mytrain = preprocess.fit_transform(train)
 mytrain.Age.isnull().sum()
india_covid_19['Date'] = pd.to_datetime(india_covid_19['Date'])
 state_testing['Date'] = pd.to_datetime(state_testing['Date'] )<data_type_conversions>
class ColumnsDelete() :

 def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
if self.columns:
mydf.drop(self.columns, axis=1, inplace=True)
 return mydf
dates = list(confirmed_df.columns[4:])
 dates = list(pd.to_datetime(dates))
dates_india = dates[8:]<feature_engineering>
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=True)) , 
('DeleteColumns', ColumnsDelete(['Room', 'Ticket']))
])
 mytrain = preprocess.fit_transform(train)
 mytest = preprocess.fit_transform(test)
 mytrain.info() , mytest.info()
df = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 data = df.copy()
data['Date'] = data['Date'].apply(pd.to_datetime)
 data.drop(['Sno', 'Time'],axis=1,inplace=True)
 data_apr = data[data['Date'] > pd.Timestamp(date(2020,4,12)) ]
state_cases = data_apr.groupby('State/UnionTerritory')['Confirmed','Deaths','Cured'].max().reset_index()
state_cases['Active'] = state_cases['Confirmed'] -(state_cases['Deaths']- state_cases['Cured'])
 state_cases["Death Rate(per 100)"] = np.round(100*state_cases["Deaths"]/state_cases["Confirmed"],2)
 state_cases["Cure Rate(per 100)"] = np.round(100*state_cases["Cured"]/state_cases["Confirmed"],2 )<load_from_csv>
class CategoryEncoder() :

 def __init__(self, columns=None):
self.columns = columns
def fit(self, X, y=None, **fit_params):

 return self
def transform(self, X, **transform_params):

 mydf = X.copy()
mydf['Sex'] = mydf['Sex'].map({'male': 1, 'female': 0})
 if self.columns:
mydf = pd.get_dummies(mydf, columns=self.columns, prefix=self.columns)
 return mydf
state_testing = pd.read_csv('.. /input/covid19-in-india/StatewiseTestingDetails.csv')
 state_testing<load_from_csv>
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=True)) , 
('DeleteColumns', ColumnsDelete(['Room', 'Ticket'])) , 
('CategoEncoder', CategoryEncoder(['Embarked', 'CabinDeck', 'Title'])) ,
])
 mytrain = preprocess.fit_transform(train)
 mytrain
labs = pd.read_csv(".. /input/covid19-in-india/ICMRTestingLabs.csv")
 fig = px.treemap(labs, path=['state','city'],
color='city', hover_data=['lab','address'],
color_continuous_scale='reds')
 fig.show()<load_from_csv>
class myZScaler(BaseEstimator, TransformerMixin):

 def __init__(self, columns = all, **init_params):
self.columns = columns
self.scaler = StandardScaler(**init_params)
 def fit(self, X, y=None):

 self.scaler.fit(X[self.columns], y)
 return self
def transform(self, X: pd):

 if self.columns is all:
self.columns = X.columns
init_col_order = X.columns
X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns, index = X.index)
 X_not_scaled = X.loc[:,~X.columns.isin(self.columns)]
return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]
myscaler = myZScaler(['Age', 'Fare'], copy=True)
 myfoo = myscaler.fit_transform(mytrain)
 myfoo.head()
zone=pd.read_csv('/kaggle/input/covid-19-india-zone-classification/lockdownindiawarningzones.csv')
 zone.style.set_properties(**{'background-color': 'black',
'color': 'lawngreen',
'border-color': 'white'} )<load_from_csv>
preprocess = Pipeline([ 
('Preprocessing', FeaturesTransformer(create_family=True)) , 
('FillingValues', FillerTransformer(mean=False)) , 
('DeleteColumns', ColumnsDelete(['Room', 'Ticket'])) , 
('CategoEncoder', CategoryEncoder(['Embarked', 'CabinDeck', 'Title'])) , 
('z-score' , myZScaler(['Age', 'Fare', 'FamilySize', 'Pclass'], copy=True))
])
 train.loc[797,'Name'] = "Leader, Ms.Alice(Farnham)"
mytrain = preprocess.fit_transform(train )
country_codes = pd.read_csv('https://raw.githubusercontent.com/plotly/datasets/master/2014_world_gdp_with_codes.csv')
 country_codes = country_codes.drop('GDP(BILLIONS)', 1)
 country_codes.rename(columns={'COUNTRY': 'Country', 'CODE': 'Code'}, inplace=True )<load_from_csv>
mytrain = preprocess.fit_transform(train)
 mytest = preprocess.fit_transform(test )
virus_data = pd.read_csv('/kaggle/input/novel-corona-virus-2019-dataset/covid_19_data.csv')
 prev_index = 0
first_time = False
tmp = 0
for i, row in virus_data.iterrows() :
if(virus_data.loc[i,'SNo'] < 1342 and virus_data.loc[i,'Province/State']=='Hubei'):
if(first_time):
tmp = virus_data.loc[i,'Confirmed']
prev_index = i
virus_data.loc[i,'Confirmed'] = virus_data.loc[i,'Confirmed'] + 593
first_time = False
else:
increment = virus_data.loc[i,'Confirmed'] - tmp
tmp = virus_data.loc[i,'Confirmed']
virus_data.loc[i,'Confirmed'] = virus_data.loc[prev_index,'Confirmed'] + increment + 593
prev_index = i
virus_data.rename(columns={'Country/Region': 'Country', 'ObservationDate': 'Date'}, inplace=True)
 virus_data = virus_data.fillna('unknow')
 virus_data['Country'] = virus_data['Country'].str.replace('US','United States')
 virus_data['Country'] = virus_data['Country'].str.replace('UK','United Kingdom')
 virus_data['Country'] = virus_data['Country'].str.replace('Mainland China','China')
 virus_data['Country'] = virus_data['Country'].str.replace('South Korea','Korea, South')
 virus_data['Country'] = virus_data['Country'].str.replace('North Korea','Korea, North')
 virus_data['Country'] = virus_data['Country'].str.replace('Macau','China')
 virus_data['Country'] = virus_data['Country'].str.replace('Ivory Coast','Cote d'Ivoire')
 virus_data = pd.merge(virus_data,country_codes,on=['Country'])
<load_from_csv>
target = mytrain['Survived']
mytrain.drop(['Survived'], axis=1, inplace=True )
covid_India_cases = pd.read_csv('.. /input/covid19-in-india/covid_19_india.csv')
 covid_India_cases=covid_India_cases.dropna()
covid_India_cases.rename(columns={'State/UnionTerritory': 'State', 'Cured': 'Recovered', 'Confirmed': 'Confirmed'}, inplace=True)
 covid_India_cases = covid_India_cases.fillna('unknow')
 top_country = covid_India_cases.loc[covid_India_cases['Date'] == covid_India_cases['Date'].iloc[-1]]
top_country = top_country.groupby(['State'])['Confirmed'].sum().reset_index()
top_country = top_country.sort_values('Confirmed', ascending=False)
 top_country = top_country[:30]
top_country_codes = top_country['State']
top_country_codes = list(top_country_codes)
 countries = covid_India_cases[covid_India_cases['State'].isin(top_country_codes)]
countries_day = countries.groupby(['Date','State'])['Confirmed','Deaths','Recovered'].sum().reset_index()
exponential_line_x = []
exponential_line_y = []
for i in range(16):
exponential_line_x.append(i)
 exponential_line_y.append(i)
 Maharashtra = countries_day.loc[countries_day['State']=='Maharashtra']
Maharashtra=Maharashtra.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Maharashtra = []
new_confirmed_cases_Maharashtra.append(list(Maharashtra['Confirmed'])[0] - list(Maharashtra['Deaths'])[0]
- list(Maharashtra['Recovered'])[0])
 for i in range(1,len(Maharashtra)) :
new_confirmed_cases_Maharashtra.append(list(Maharashtra['Confirmed'])[i] -
list(Maharashtra['Deaths'])[i] -
list(Maharashtra['Recovered'])[i])
 Gujarat = countries_day.loc[countries_day['State']=='Gujarat']
Gujarat=Gujarat.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Gujarat = []
new_confirmed_cases_Gujarat.append(list(Gujarat['Confirmed'])[0] - list(Gujarat['Deaths'])[0]
- list(Gujarat['Recovered'])[0])
 for i in range(1,len(Gujarat)) :
new_confirmed_cases_Gujarat.append(list(Gujarat['Confirmed'])[i] -
list(Gujarat['Deaths'])[i] -
list(Gujarat['Recovered'])[i])
 Delhi = countries_day.loc[countries_day['State']=='Delhi']
Delhi=Delhi.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Delhi = []
new_confirmed_cases_Delhi.append(list(Delhi['Confirmed'])[0] - list(Delhi['Deaths'])[0]
- list(Delhi['Recovered'])[0])
 for i in range(1,len(Delhi)) :
new_confirmed_cases_Delhi.append(list(Delhi['Confirmed'])[i] -
list(Delhi['Deaths'])[i] -
list(Delhi['Recovered'])[i])
 Madhya_Pradesh = countries_day.loc[countries_day['State']=='Madhya Pradesh']
Madhya_Pradesh=Madhya_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Madhya_Pradesh = []
new_confirmed_cases_Madhya_Pradesh.append(list(Madhya_Pradesh['Confirmed'])[0] - list(Madhya_Pradesh['Deaths'])[0]
- list(Madhya_Pradesh['Recovered'])[0])
 for i in range(1,len(Madhya_Pradesh)) :
new_confirmed_cases_Madhya_Pradesh.append(list(Madhya_Pradesh['Confirmed'])[i] -
list(Madhya_Pradesh['Deaths'])[i] -
list(Madhya_Pradesh['Recovered'])[i])
 Rajasthan = countries_day.loc[countries_day['State']=='Rajasthan']
Rajasthan=Rajasthan.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Rajasthan = []
new_confirmed_cases_Rajasthan.append(list(Rajasthan['Confirmed'])[0] - list(Rajasthan['Deaths'])[0]
- list(Rajasthan['Recovered'])[0])
 for i in range(1,len(Rajasthan)) :
new_confirmed_cases_Rajasthan.append(list(Rajasthan['Confirmed'])[i] -
list(Rajasthan['Deaths'])[i] -
list(Rajasthan['Recovered'])[i])
 Uttar_Pradesh = countries_day.loc[countries_day['State']=='Uttar Pradesh']
Uttar_Pradesh=Uttar_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Uttar_Pradesh = []
new_confirmed_cases_Uttar_Pradesh.append(list(Uttar_Pradesh['Confirmed'])[0] - list(Uttar_Pradesh['Deaths'])[0]
- list(Uttar_Pradesh['Recovered'])[0])
 for i in range(1,len(Uttar_Pradesh)) :
new_confirmed_cases_Uttar_Pradesh.append(list(Uttar_Pradesh['Confirmed'])[i] -
list(Uttar_Pradesh['Deaths'])[i] -
list(Uttar_Pradesh['Recovered'])[i])
 Tamil_Nadu = countries_day.loc[countries_day['State']=='Tamil Nadu']
Tamil_Nadu=Tamil_Nadu.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Tamil_Nadu = []
new_confirmed_cases_Tamil_Nadu.append(list(Tamil_Nadu['Confirmed'])[0] - list(Tamil_Nadu['Deaths'])[0]
- list(Tamil_Nadu['Recovered'])[0])
 for i in range(1,len(Tamil_Nadu)) :
new_confirmed_cases_Tamil_Nadu.append(list(Tamil_Nadu['Confirmed'])[i] -
list(Tamil_Nadu['Deaths'])[i] -
list(Tamil_Nadu['Recovered'])[i])
 Andhra_Pradesh = countries_day.loc[countries_day['State']=='Andhra Pradesh']
Andhra_Pradesh=Andhra_Pradesh.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_Andhra_Pradesh = []
new_confirmed_cases_Andhra_Pradesh.append(list(Andhra_Pradesh['Confirmed'])[0] - list(Andhra_Pradesh['Deaths'])[0]
- list(Andhra_Pradesh['Recovered'])[0])
 for i in range(1,len(Andhra_Pradesh)) :
new_confirmed_cases_Andhra_Pradesh.append(list(Andhra_Pradesh['Confirmed'])[i] -
list(Andhra_Pradesh['Deaths'])[i] -
list(Andhra_Pradesh['Recovered'])[i])
 West_Bengal = countries_day.loc[countries_day['State']=='West Bengal']
West_Bengal=West_Bengal.sort_values('Confirmed',ascending=True)
 new_confirmed_cases_West_Bengal = []
new_confirmed_cases_West_Bengal.append(list(West_Bengal['Confirmed'])[0] - list(West_Bengal['Deaths'])[0]
- list(West_Bengal['Recovered'])[0])
 for i in range(1,len(West_Bengal)) :
new_confirmed_cases_West_Bengal.append(list(West_Bengal['Confirmed'])[i] -
list(West_Bengal['Deaths'])[i] -
list(West_Bengal['Recovered'])[i])
 p1 = figure(plot_width=800, plot_height=550, title="Trajectory of Covid-19")
 p1.grid.grid_line_alpha=0.3
p1.ygrid.band_fill_color = "olive"
p1.ygrid.band_fill_alpha = 0.1
p1.xaxis.axis_label = 'Total number of detected cases(Log scale)'
p1.yaxis.axis_label = 'New confirmed cases(Log scale)'
p1.line(exponential_line_x, exponential_line_y, line_dash="4 4", line_width=0.5)
 p1.line(np.log(list(Maharashtra['Confirmed'])) , np.log(new_confirmed_cases_Maharashtra), color='
legend_label='Maharashtra', line_width=1)
 p1.circle(np.log(list(Maharashtra['Confirmed'])[-1]), np.log(new_confirmed_cases_Maharashtra[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Gujarat['Confirmed'])) , np.log(new_confirmed_cases_Gujarat), color='
legend_label='Gujarat', line_width=1)
 p1.circle(np.log(list(Gujarat['Confirmed'])[-1]), np.log(new_confirmed_cases_Gujarat[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Delhi['Confirmed'])) , np.log(new_confirmed_cases_Delhi), color='
legend_label='Delhi', line_width=1)
 p1.circle(np.log(list(Delhi['Confirmed'])[-1]), np.log(new_confirmed_cases_Delhi[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Madhya_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Madhya_Pradesh), color='
legend_label='Madhya Pradesh', line_width=1)
 p1.circle(np.log(list(Madhya_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Madhya_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Rajasthan['Confirmed'])) , np.log(new_confirmed_cases_Rajasthan), color='
legend_label='Rajasthan', line_width=1)
 p1.circle(np.log(list(Rajasthan['Confirmed'])[-1]), np.log(new_confirmed_cases_Rajasthan[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Uttar_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Uttar_Pradesh), color='
legend_label='Uttar Pradesh', line_width=1)
 p1.circle(np.log(list(Uttar_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Uttar_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Tamil_Nadu['Confirmed'])) , np.log(new_confirmed_cases_Tamil_Nadu), color='
legend_label='Tamil Nadu', line_width=1)
 p1.circle(np.log(list(Tamil_Nadu['Confirmed'])[-1]), np.log(new_confirmed_cases_Tamil_Nadu[-1]), fill_color="white", size=5)
 p1.line(np.log(list(Andhra_Pradesh['Confirmed'])) , np.log(new_confirmed_cases_Andhra_Pradesh), color='
legend_label='Andhra Pradesh', line_width=1)
 p1.circle(np.log(list(Andhra_Pradesh['Confirmed'])[-1]), np.log(new_confirmed_cases_Andhra_Pradesh[-1]), fill_color="white", size=5)
 p1.line(np.log(list(West_Bengal['Confirmed'])) , np.log(new_confirmed_cases_West_Bengal), color='
legend_label='West Bengal', line_width=1)
 p1.circle(np.log(list(West_Bengal['Confirmed'])[-1]), np.log(new_confirmed_cases_West_Bengal[-1]), fill_color="white", size=5)
 p1.legend.location = "bottom_right"
output_file("coronavirus.html", title="coronavirus.py")
 show(p1 )<load_from_csv>
RFCmodel = RandomForestClassifier(random_state=42)
 RFCmodel.fit(mytrain, target)
 print(f'Accuracy: {RFCmodel.score(mytrain,target)*100:.2f}%' )
plt.style.use('fivethirtyeight')
 train=pd.read_csv('/kaggle/input/coronavirus-2019ncov/covid-19-all.csv' )<groupby>
myprediction = RFCmodel.predict(mytrain)
 print(f'Accuracy : {accuracy_score(myprediction,target)*100:.2f}%')
 print(f'Precission: {precision_score(myprediction, target)*100:.2f}%')
 print(f'Recall : {recall_score(myprediction, target)*100:.2f}%')
 print(f'F1-score : {f1_score(myprediction, target)*100:.2f}%' )
in_df = train[train['Country/Region']=='India'].groupby('Date')['Confirmed','Deaths','Recovered'].sum().reset_index(False)
 in_df['Active']=in_df['Confirmed']-in_df['Deaths']-in_df['Recovered']
in_df = in_df[in_df.Active>=100]<prepare_x_and_y>
def evaluate_model(model):

 accuracy = cross_val_score(model, mytrain, target, cv=5, scoring='accuracy' ).mean()
return accuracy
models = {
'Logistic regression':LogisticRegression(random_state = 42),
'Decision tree':DecisionTreeClassifier(random_state = 42),
'Random forest':RandomForestClassifier(random_state = 42)
 }
for key, mymodel in models.items() :
accuracy = evaluate_model(model = mymodel)
 print('{:20}: {:2.2f}%'.format(key, accuracy*100))
in_df['day_count'] = list(range(1,len(in_df)+1))
in_df['increase'] =(in_df.Active-in_df.Active.shift(1))
in_df['rate'] =(in_df.Active-in_df.Active.shift(1)) /in_df.Active
def sigmoid(x,c,a,b):
y = c*1 /(1 + np.exp(-a*(x-b)))
 return y
xdata = np.array(list(in_df.day_count)[::2])
 ydata = np.array(list(in_df.Active)[::2])
 population=1.332*10**9 
<load_from_csv>
from sklearn.model_selection import GridSearchCV
datas = []
for dirname, _, filenames in os.walk('/kaggle/input'):
for filename in filenames:
print(os.path.join(dirname, filename))
datas.append(pd.read_csv(os.path.join(dirname, filename)))
 datas[1] = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/train.csv')
 datas[2] = pd.read_csv('/kaggle/input/covid19-global-forecasting-week-4/test.csv')
 population = pd.read_csv('/kaggle/input/covid19-global-forecasting-locations-population/locations_population.csv')
 population = population.rename(columns = {'Province.State':'Province_State', 'Country.Region':'Country_Region'})
<feature_engineering>
models = [
{
'name':'Logistic regression',
'estimator':LogisticRegression(random_state = 42),
'hyperparameters':{
'solver': ['newton-cg', 'lbfgs', 'liblinear']
}
},
{
'name':'Decision tree',
'estimator':DecisionTreeClassifier(random_state = 42),
'hyperparameters':{
'criterion':['entropy','gini'],
'splitter':['best','random'],
'max_depth':range(4,9),
'max_features':['auto','sqrt','log2'],
'min_samples_split':[3,4,5]
}
}, {
'name':'RandomForestClassifier',
'estimator':RandomForestClassifier(random_state = 42),
'hyperparameters':{
'n_estimators' : [10, 15, 18, 19, 20, 21, 22],
'criterion':['entropy','gini'],
'max_depth':range(4,9),
'max_features':['auto','sqrt','log2'],
'min_samples_split':[3,4,5]
}
}
]
def tune_models(models):
for model in models:
print(model['name'])
 print('-' * len(model['name']))
grid = GridSearchCV(model['estimator'], param_grid=model['hyperparameters'], cv=5, scoring='accuracy')
 grid.fit(mytrain, target)
 model['best_score'] = grid.best_score_
model['best_params'] = grid.best_params_
model['best_model'] = grid.best_estimator_
print('Best accuracy: {:.2f}%'.format(model['best_score']*100))
print('Best parameters: {}
'.format(model['best_params']))
return models
models = tune_models(models )
tmp_country_label = []
for idata in datas[1].itertuples() :
try:
tmp_country_label.append(idata.Country_Region + '_' + idata.Province_State)
 except:
tmp_country_label.append(idata.Country_Region)
 datas[1]['country_label'] = tmp_country_label
np.isnan(datas[1].loc[0].Province_State)
 country_list = datas[1].country_label.unique().tolist()<compute_test_metric>
mymodel = RandomForestClassifier(criterion='entropy', max_depth=7, max_features = 'auto', min_samples_split = 4, n_estimators = 18, random_state=42 )
def rmsle(pred,true):
assert pred.shape[0]==true.shape[0]
return K.sqrt(K.mean(K.square(K.log(pred+1)- K.log(true+1))))
def attention_mechanism(days, input_):
x = Dense(256, activation='sigmoid' )(input_)
 x = Dense(days, activation='softmax' )(x)
 return x
def attention_model(input_size, days=21, batch_size=32, epochs=200, lr=1e-3):
country_input = Input(shape=(313,), name='country_onehot')
 inputs = Input(shape=(None, input_size), name='encoder_input')
 target_number = Input(shape=(1,), name='target_input')
 flag_input = Input(shape=(1,), name='flag_input')
 x = Masking(mask_value=0, input_shape=(None, input_size))(inputs)
 x = GRU(128, name='GRU_layer1', return_sequences=True )(x)
 attention_x = attention_mechanism(days, x)
 gru_out = Permute(( 2, 1))(x)
 attention_mul = K.batch_dot(gru_out, attention_x)
 attention_mul = Permute(( 2, 1))(attention_mul)
 x = GRU(128, name='GRU_layer2', return_sequences=True )(attention_mul)
 gru_x = Flatten()(x)
 x = Dense(32, activation='relu' )(gru_x)
 x = Dense(128, activation='relu' )(x)
 x = Dense(256, activation='relu' )(x)
 outputs = Dense(1, activation='sigmoid' )(x)
 print(outputs.shape, flag_input.shape, target_number.shape)
 outputs = target_number *(flag_input + outputs)
 optimizer = Adam(lr=lr, name='adam')
 model = Model([inputs, country_input, target_number, flag_input], outputs, name='gru_network')
 model.compile(optimizer=optimizer, loss=rmsle)
 return model<init_hyperparams>
mymodel.fit(mytrain, target)
 myprediction = mymodel.predict(mytrain)
 print(f'Precision: {precision_score(myprediction, target)*100:.2f}%')
 print(f'Accuracy: {accuracy_score(myprediction, target)*100:.2f}%')
class corona19_predict:
def __init__(self, df, population, days=21, batch_size=8, epochs=200):
self.days = days
self.batch_size = batch_size
self.epochs = epochs
self.confirmed_cases_model = attention_model(2, days, lr=1e-4)
 self.fatalities_model = attention_model(5, days, lr=1e-6)
 self.cal_increase_rate(df, population)
 def cal_increase_rate(self, df, population):
pre_ccd = 0
pre_fd = 0
confirmed_cases_diff = []
fatalities_diff = []
for idata in df.itertuples() :
if idata.ConfirmedCases < pre_ccd:
pre_ccd = 0
pre_fd = 0
confirmed_cases_diff.append(idata.ConfirmedCases - pre_ccd)
 fatalities_diff.append(idata.Fatalities - pre_fd)
 pre_ccd = idata.ConfirmedCases
pre_fd = idata.Fatalities
df['ConfirmedCases_diff'] = confirmed_cases_diff
df['Fatalities_diff'] = fatalities_diff
df['Fatalities_diff'] = df['Fatalities_diff'].clip(0)
 df['ConfirmedCases_diff_percent'] = df['ConfirmedCases_diff'].values /(df['ConfirmedCases'].values + 1.0e-10)
 df['Fatalities_diff_percent'] = df['Fatalities_diff'].values /(df['Fatalities'].values + 1.0e-10)
 tmp_country_label = []
for idata in df.itertuples() :
try:
tmp_country_label.append(idata.Country_Region + '_' + idata.Province_State)
 except:
tmp_country_label.append(idata.Country_Region)
 df['country_label'] = tmp_country_label
tmp_country_label = []
for idata in population.itertuples() :
try:
tmp_country_label.append(idata.Country_Region + '_' + idata.Province_State)
 except:
tmp_country_label.append(idata.Country_Region)
 population['country_label'] = tmp_country_label
self.target_df = df
self.population_df = population
self.country_list = df['country_label'].unique().tolist()
return df
def get_country_onehot(self, country_str):
country_onehot = np.zeros(len(self.country_list))
country_onehot[self.country_list.index(country_str)] = 1
return country_onehot
def encoded_data(self, country_onehot, target_date_str, country_population):
date_list = target_date_str.split('-')
 delta = timedelta(days=1)
 date = datetime(int(date_list[0]), int(date_list[1]), int(date_list[2])) - delta
day_list = []
for i in range(self.days):
day_list.append(date.strftime('%Y-%m-%d'))
date -= delta
day_list = day_list[::-1]
confirmed_cases = 0
fatalities = 0
encoded_data = []
if self.country_df.country_label.values[0] != self.country_list[np.argmax(country_onehot)]:
self.country_df = self.target_df[self.target_df.country_label == self.country_list[np.argmax(country_onehot)]]
for date_str in day_list:
tmp_data_df = self.country_df[self.country_df.Date == date_str]
if len(tmp_data_df)== 0:
'train data not exist'
else:
confirmed_cases_diff = tmp_data_df.ConfirmedCases_diff.values[0]
fatalities_diff = tmp_data_df.Fatalities_diff.values[0]
confirmed_cases = tmp_data_df.ConfirmedCases.values[0]
fatalities = tmp_data_df.Fatalities.values[0]
confirmed_cases_diff_percent = tmp_data_df.ConfirmedCases_diff_percent.values[0]
fatalities_diff_percent = tmp_data_df.Fatalities_diff_percent.values[0]
encoded_data.append([confirmed_cases, fatalities, confirmed_cases_diff, fatalities_diff, confirmed_cases_diff_percent, fatalities_diff_percent, country_population])
 return encoded_data
def make_train_data(self):
train_data = {'country_onehot': [], 'encoder_input': []}
train_label = []
p_country = ''
for idata in tqdm(self.target_df.itertuples() , total=len(self.target_df), position=0):
if p_country != idata.country_label:
self.country_df = self.target_df[self.target_df.country_label == idata.country_label]
country_population = self.population_df[self.population_df.country_label == idata.country_label].Population.values[0]
p_country = idata.country_label
if idata.Date > self.target_df.iloc[self.days + 1].Date:
tmp_onehot_data = self.get_country_onehot(idata.country_label)
 tmp_encoded_data = self.encoded_data(tmp_onehot_data, idata.Date, country_population)
 try:
if np.sum(np.array(tmp_encoded_data)[:, :])!= 0:
train_data['country_onehot'].append(tmp_onehot_data)
 train_data['encoder_input'].append(tmp_encoded_data)
 train_label.append([idata.ConfirmedCases, idata.Fatalities, idata.ConfirmedCases_diff, idata.Fatalities_diff, idata.ConfirmedCases_diff_percent, idata.Fatalities_diff_percent, country_population])
 except:
print(idata.country_label, country_population)
 print(tmp_ecoded_data, idata.country_label, country_population)
 return [np.array(train_data['encoder_input']), np.array(train_data['country_onehot'])], np.array(train_label)
 def train_data_fatalities(self):
try:
with gzip.open('encoded_data.dat', 'rb')as f:
X_train, y_train = pickle.load(f)
 except:
X_train, y_train = self.make_train_data()
with gzip.open('encoded_data.dat', 'wb')as f:
pickle.dump([X_train, y_train], f)
 x_train = X_train.copy()
x_train[0] = np.concatenate([\ 
(x_train[0][:, :, 0] / x_train[0][:, :, 6] ).reshape(list(x_train[0].shape[:-1])+ [1]), \ 
(x_train[0][:, :, 2] / x_train[0][:, :, 6] ).reshape(list(x_train[0].shape[:-1])+ [1]), \ 
(x_train[0][:, :, 1] / x_train[0][:, :, 6] ).reshape(list(x_train[0].shape[:-1])+ [1]), \ 
(x_train[0][:, :, 3] / x_train[0][:, :, 6] ).reshape(list(x_train[0].shape[:-1])+ [1]), \ 
(x_train[0][:, :, 1] /(x_train[0][:, :, 0] + 1e-8)).reshape(list(x_train[0].shape[:-1])+ [1])\
], axis=2)
 death_rate_index = np.where(y_train[:, 1] /(y_train[:, 0] + 1e-8)< 0.15)[0]
tb_hist = tensorflow.keras.callbacks.TensorBoard(log_dir='./graph_gru_attention', histogram_freq=1, write_graph=True, write_images=True)
 model_path = './fatalities_gru_attention.h5'
cb_checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=1, save_best_only=True)
 early_stopping = EarlyStopping(patience=10)
 history = self.fatalities_model.fit({'encoder_input':x_train[0][death_rate_index], 'country_onehot':x_train[1][death_rate_index], 'target_input': X_train[0][death_rate_index][:, -1, 1].reshape(list(X_train[0][death_rate_index].shape[:-2])+ [1]), 'flag_input': np.ones(len(x_train[0][death_rate_index])) }, y_train[:, 1][death_rate_index], batch_size=self.batch_size, epochs=self.epochs, verbose=1, shuffle=True,
validation_split=0.2,
callbacks=[tb_hist, cb_checkpoint, early_stopping])
 y_predict = self.fatalities_model.predict({'encoder_input':x_train[0], 'country_onehot':x_train[1], 'target_input': X_train[0][:, -1, 0].reshape(list(X_train[0].shape[:-2])+ [1]), 'flag_input': np.zeros(len(x_train[0])) })
 return y_predict, y_train
def train_data_confirmed_cases(self):
try:
with gzip.open('encoded_data.dat', 'rb')as f:
X_train, y_train = pickle.load(f)
 except:
X_train, y_train = self.make_train_data()
with gzip.open('encoded_data.dat', 'wb')as f:
pickle.dump([X_train, y_train], f)
 x_train = X_train.copy()
x_train[0] = np.concatenate([(x_train[0][:, :, 0] / x_train[0][:, :, 6] ).reshape(list(x_train[0].shape[:-1])+ [1]),(x_train[0][:, :, 2] / x_train[0][:, :, 6] ).reshape(list(x_train[0].shape[:-1])+ [1])], axis=2)
 tb_hist = tensorflow.keras.callbacks.TensorBoard(log_dir='./graph_gru_attention', histogram_freq=1, write_graph=True, write_images=True)
 model_path = './confirmed_cases_gru_attention.h5'
cb_checkpoint = ModelCheckpoint(filepath=model_path, monitor='val_loss', verbose=1, save_best_only=True)
 early_stopping = EarlyStopping(patience=10)
 history = self.confirmed_cases_model.fit({'encoder_input':x_train[0], 'country_onehot':X_train[1], 'target_input': X_train[0][:,-1,0].reshape(list(X_train[0].shape[:-2])+ [1]), 'flag_input':np.ones(( len(x_train[0]), 1)) }, y_train[:, 0], batch_size=self.batch_size, epochs=self.epochs, verbose=1, shuffle=True,
validation_split=0.2,
callbacks=[tb_hist, cb_checkpoint, early_stopping])
 y_predict = self.confirmed_cases_model.predict({'encoder_input':x_train[0], 'country_onehot':X_train[1], 'target_input': X_train[0][:,-1,0].reshape(list(X_train[0].shape[:-2])+ [1]), 'flag_input':np.ones(( len(x_train[0]), 1)) })
 return y_predict, y_train
def load_models(self, country_list):
self.confirmed_cases_model.load_weights('/kaggle/working/confirmed_cases_gru_attention.h5')
 self.fatalities_model.load_weights('/kaggle/working/fatalities_gru_attention.h5')
 self.country_list = country_list
def predict_encoder_confirmed_cases(self, day_list, country_label, data_df, country_population):
encoded_data_c = []
encoded_data_f = []
country_onehot = np.zeros(len(self.country_list))
country_onehot[self.country_list.index(country_label)] = 1
before_confirmed_case = 0
before_fatalities_case = 0
for day in day_list:
tmp_data_df = data_df[data_df.Date == day]
try:
encoded_data_c.append([(tmp_data_df.ConfirmedCases.values[0] / country_population),tmp_data_df.ConfirmedCases_diff.values[0] / country_population])
 except:
print(country_population, day, country_label, tmp_data_df, day_list)
 return
encoded_data_f.append([\ 
(tmp_data_df.ConfirmedCases.values[0] / country_population), \ 
(tmp_data_df.ConfirmedCases_diff.values[0] / country_population), \ 
(tmp_data_df.Fatalities.values[0] / country_population), \ 
(tmp_data_df.Fatalities_diff.values[0] / country_population), 
(tmp_data_df.Fatalities.values[0] /(tmp_data_df.ConfirmedCases.values[0] + 1e-8)) \
])
 before_confirmed_case = tmp_data_df.ConfirmedCases.values[0]
before_fatalities_case = tmp_data_df.Fatalities.values[0]
return np.array([country_onehot]), np.array([encoded_data_c]), before_confirmed_case, np.array([encoded_data_f]), before_fatalities_case
def predict_encoder_fatalities(self, day_list, country_label, data_df, country_population):
encoded_data = []
before_fatalities = 0
for day in day_list:
tmp_data_df = data_df[data_df.Date == day]
encoded_data.append([\ 
(tmp_data_df.ConfirmedCases.values[0] / country_population), \ 
(tmp_data_df.ConfirmedCases_diff.values[0] / country_population), \ 
(tmp_data_df.Fatalities.values[0] / country_population), \ 
(tmp_data_df.Fatalities_diff.values[0] / country_population), 
(tmp_data_df.Fatalities.values[0] /(tmp_data_df.ConfirmedCases.values[0] + 1e-8)) \
])
 before_fatalities = tmp_data_df.Fatalities.values[0]
return np.array([encoded_data]), before_fatalities
def predict_test(self, test_df):
predict_confirmed_cases = []
predict_fatalities = []
country_list = self.country_list.copy()
p_country = ''
for itest in tqdm(test_df.itertuples() , total=len(test_df), position=0):
date_list = itest.Date.split('-')
 delta = timedelta(days=1)
 date = datetime(int(date_list[0]), int(date_list[1]), int(date_list[2])) - delta
day_list = []
self.country_list = country_list.copy()
for i in range(self.days):
day_list.append(date.strftime('%Y-%m-%d'))
date -= delta
day_list = day_list[::-1]
try:
country_label = itest.Country_Region + '_' + itest.Province_State
except:
country_label = itest.Country_Region
if p_country != country_label:
data_df = self.target_df[self.target_df.country_label == country_label]
country_population = self.population_df[self.population_df.country_label == country_label].Population.values[0]
p_country = country_label
country_onehot, encoded_data_c, bc, encoded_data_f, bf = self.predict_encoder_confirmed_cases(day_list, country_label, data_df, country_population)
 confirmed_cases_increase_rate = self.confirmed_cases_model.predict_on_batch({'encoder_input':encoded_data_c, 'country_onehot':country_onehot, 'target_input':np.array([bc] ).reshape(( 1,1)) , 'flag_input':np.array([1])})
 mortality_rate = self.fatalities_model.predict_on_batch({'encoder_input':encoded_data_f, 'country_onehot':country_onehot, 'target_input': np.array([bf] ).reshape(( 1,1)) , 'flag_input':np.array([1])})
 confirmed_cases_increase_rate = confirmed_cases_increase_rate.numpy().reshape(1)
 mortality_rate = mortality_rate.numpy().reshape(1)
 t_df = self.target_df[(self.target_df.Date == itest.Date)&(self.target_df.country_label == country_label)]
if len(t_df)== 0:
new_data = [-1, itest.Province_State, itest.Country_Region, itest.Date, confirmed_cases_increase_rate[0], mortality_rate[0]]
new_data += [0] *(len(self.target_df.columns)- len(new_data))
self.target_df.loc[len(self.target_df)] = new_data
self.target_df = self.target_df.sort_values(by='Date')
 self.target_df = self.target_df.sort_values(by='Province_State')
 self.target_df = self.target_df.sort_values(by='Country_Region')
 self.cal_increase_rate(self.target_df, self.population_df)
 data_df = self.target_df[self.target_df.country_label == country_label]
predict_confirmed_cases.append(confirmed_cases_increase_rate[0])
 predict_fatalities.append(mortality_rate[0])
 else:
predict_confirmed_cases.append(t_df.ConfirmedCases.values[0])
 predict_fatalities.append(t_df.Fatalities.values[0])
 test_df['ConfirmedCases'] = predict_confirmed_cases
test_df['Fatalities'] = predict_fatalities
test_df[['ForecastId', 'ConfirmedCases', 'Fatalities']].to_csv('submission.csv', index=False)
 return test_df 
<predict_on_test>
myprediction = mymodel.predict(mytest)
 submission = pd.DataFrame({'PassengerId':mytest.index,'Survived':myprediction})
 submission.to_csv('submissionRFC_nosplit.csv',index=False )
test_c19 = corona19_predict(datas[1], population, 71, epochs=200)
 y_predict_f, y_train_f = test_c19.train_data_fatalities() 
<prepare_x_and_y>
from sklearn.model_selection import train_test_split
y_predict_c, y_train_c = test_c19.train_data_confirmed_cases()<predict_on_test>
from sklearn.model_selection import train_test_split
test_c19 = corona19_predict(datas[1], population, 71, epochs=200)
 country_list = test_c19.country_list.copy()
test_c19.load_models(country_list)
 rr = test_c19.predict_test(datas[2] )<filter>
train_x, test_x, train_y, test_y = train_test_split(mytrain, target, test_size = 0.2, random_state=42)
 mymodel = RandomForestClassifier(criterion='entropy', max_depth=7, max_features = 'auto', min_samples_split = 4, n_estimators = 18, random_state=42)
 mymodel.fit(train_x, train_y)
 mymodel.score(test_x, test_y)
datas[1][(datas[1].Country_Region == 'Canada')&(datas[1].Province_State == 'Yukon')]<load_from_csv>
myprediction = mymodel.predict(mytest )
<create_dataframe><EOS> 
submission = pd.DataFrame({'PassengerId':mytest.index,'Survived':myprediction})
 submission.to_csv('submissionRF_optim_param.csv',index=False )