Kaggle competition: Natural Language Processing with Disaster Tweets
train['day_dist'] = train['day_dist'].dt.days<define_variables>
nlp_train = pd.read_csv('/kaggle/input/nlp-getting-started/train.csv')
 nlp_test = pd.read_csv('/kaggle/input/nlp-getting-started/test.csv')
 nlp_train.head(2 )
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<correct_missing_values>
PROJECT_ID = 'kaggle-bqml-course'
bucket_name = 'leosuky_kaggle_competitions'
region = 'us-central1'
storage_client = storage.Client(project=PROJECT_ID)
 automl_client = automl.AutoMlClient()
for cat_col in cat_cols:
train[cat_col].fillna('no_value', inplace = True )<feature_engineering>
def upload_to_gcs(bucket_name, source_file_name, destination_blob_name):
"Uploads a file to the bucket.https://cloud.google.com/storage/docs/"
bucket = storage_client.get_bucket(bucket_name)
 blob = bucket.blob(destination_blob_name)
 blob.upload_from_filename(source_file_name)
train['place'] = train['Province/State']+'_'+train['Country/Region'] 
<import_modules>
destination_path = 'uploads/kaggle_auto_ml/train.csv'
from sklearn import preprocessing<define_variables>
dataset_name = 'disaster_tweets_nlp'
model_name = 'disaster_tweets_nlp'
client = automl_client
region = region
project_id = PROJECT_ID
bucket_name = bucket_name
cat_cols = train.dtypes[train.dtypes=='object'].keys()
cat_cols<categorify>
amw = AutoMLWrapper(client=client,
project_id=PROJECT_ID,
bucket_name=bucket_name,
region='us-central1',
dataset_display_name=dataset_name,
model_display_name=model_name )
for cat_col in ['place']:
le = preprocessing.LabelEncoder()
le.fit(train[cat_col])
 train[cat_col]=le.transform(train[cat_col] )<define_variables>
def create_dataset(project_id, dataset_name):
client = automl.AutoMlClient()
project_location = client.location_path(project_id, region)
 metadata = automl.types.TextSentimentDatasetMetadata(sentiment_max=1)
 dataset = automl.types.Dataset(display_name=dataset_name,
text_sentiment_dataset_metadata=metadata)
 result = client.create_dataset(project_location, dataset)
 print(result )
drop_cols = ['Id','ForecastId', 'ConfirmedCases','Date', 'Fatalities','day_dist', 'Province/State', 'Country/Region']<filter>
def import_dataset(project_id, dataset_id, path):
client = automl.AutoMlClient()
dataset_full_id = client.dataset_path(project_id, 'us-central1', dataset_id)
 input_uris = path.split(",")
 gcs_source = automl.types.GcsSource(input_uris=input_uris)
 input_config = automl.types.InputConfig(gcs_source=gcs_source)
 result = client.import_data(dataset_full_id, input_config)
 print("Processing import...")
 print("Data imported.{}".format(result))
val = train[(train['Date']>='2020-03-12')&(train['Id'].isnull() ==False)] 
<prepare_x_and_y>
file_path = 'gs://leosuky_kaggle_competitions/uploads/kaggle_auto_ml/train.csv'
dataset_id = 'TST2133943162303938560'
y_ft = train["Fatalities"]
y_val_ft = val["Fatalities"]
y_cc = train["ConfirmedCases"]
y_val_cc = val["ConfirmedCases"] 
<compute_test_metric>
def create_model(project_id, dataset_id, dataset_name):
client = automl.AutoMlClient()
project_location = client.location_path(project_id, 'us-central1')
 metadata = automl.types.TextSentimentModelMetadata()
model = automl.types.Model(display_name=dataset_name,
dataset_id=dataset_id,
text_sentiment_model_metadata=metadata)
 result = client.create_model(project_location, model)
 print(u"Training operation name: {}".format(result.operation.name))
print("Training started..." )
def rmsle(y_true, y_pred):
return np.sqrt(np.mean(np.power(np.log1p(y_pred)- np.log1p(y_true), 2)))
<compute_test_metric>
print("model has already been created and trained!" )
def mape(y_true, y_pred):
return np.mean(np.abs(y_pred -y_true)*100/(y_true+1)) 
<import_modules>
def model_evaluations(project_id, model_id):
client = automl.AutoMlClient()
full_model_id = client.model_path(project_id, 'us-central1', model_id)
 print('Model Evaluations:')
 for evaluation in client.list_model_evaluations(full_model_id, ""):
print('Model Evaluation Name: {}'.format(evaluation.name))
print('Model Annotation Spec Id: {}'.format(evaluation.annotation_spec_id))
print("Create Time:")
 print("\tseconds: {}".format(evaluation.create_time.seconds))
print("\tnanos: {}".format(evaluation.create_time.nanos / 1e9))
print('Evaluation Example Count: {}'.format(evaluation.evaluated_example_count))
print('Sentiment Analysis Evaluation Metrics: {}'.format(evaluation.text_sentiment_evaluation_metrics))
print('Translation Model Evaluation Metrics: {}'.format(evaluation.translation_evaluation_metrics))
import numpy as np<init_hyperparams>
model_id = "TST4544356324388896768"
model_evaluations(project_id=project_id, model_id=model_id )
params = {
"objective": "regression",
"boosting": 'gbdt',
"num_leaves": 1280,
"learning_rate": 0.05,
"feature_fraction": 0.9,
"reg_lambda": 2,
"metric": "rmse",
'min_data_in_leaf':20
} 
<filter>
if not amw.get_dataset_by_display_name(dataset_display_name=dataset_name):
print('dataset not found')
 amw.create_dataset()
amw.import_gcs_data(training_gcs_path)
 amw.dataset
dates = dates[dates>'2020-03-24']<filter>
if not amw.get_model_by_display_name() :
amw.train_model()
amw.deploy_model()
amw.model
test[test['Country/Region']=='Italy']<filter>
amw.model_full_path
test[test['Country/Region']=='Italy']<load_from_csv>
def make_predictions(project_id, model_id, content):
prediction_client = automl.PredictionServiceClient()
model_full_id = prediction_client.model_path(project_id, "us-central1", model_id)
 text_snippet = automl.types.TextSnippet(content=content, mime_type="text/plain")
 payload = automl.types.ExamplePayload(text_snippet=text_snippet)
 response = prediction_client.predict(model_full_id, payload)
 for annotation_payload in response.payload:
return annotation_payload.text_sentiment.sentiment
train_sub = pd.read_csv(".. /input/covid19-global-forecasting-week-1/train.csv" )<feature_engineering>
make_predictions(project_id=PROJECT_ID, model_id=model_id, content=corpus[1] )
train_sub.loc[(train_sub['Date']=='2020-03-24')&(train_sub['Country/Region']=='France')&(train_sub['Province/State']=='France'),'ConfirmedCases'] = 22654
train_sub.loc[(train_sub['Date']=='2020-03-24')&(train_sub['Country/Region']=='France')&(train_sub['Province/State']=='France'),'Fatalities'] = 1000<merge>
predictions = []
print('Starting predictions...')
 print('Predicting 1st batch.....')
 for document in corpus[:600]:
predictions.append(
 make_predictions(project_id=PROJECT_ID, model_id=model_id, content=document)
)
 print('Predicting 2nd batch.....')
 for document in corpus[600:1200]:
predictions.append(
 make_predictions(project_id=PROJECT_ID, model_id=model_id, content=document)
)
 print('Predicting 3rd batch.....')
 for document in corpus[1200:1800]:
predictions.append(
 make_predictions(project_id=PROJECT_ID, model_id=model_id, content=document)
)
 print('Predicting 4th batch.....')
 for document in corpus[1800:2400]:
predictions.append(
 make_predictions(project_id=PROJECT_ID, model_id=model_id, content=document)
)
 print('Predicting last batch.....')
 for document in corpus[2400:]:
predictions.append(
 make_predictions(project_id=PROJECT_ID, model_id=model_id, content=document)
)
 print('Done.' )
test = pd.merge(test,train_sub[['Province/State','Country/Region','Lat','Long','Date','ConfirmedCases','Fatalities']], on=['Province/State','Country/Region','Lat','Long','Date'], how='left' )<filter>
sentiment_predictions = pd.DataFrame(predictions, columns=['target'])
 sentiment_predictions.head(4 )
test.loc[test['ConfirmedCases_x'].isnull() ==True]<feature_engineering>
submission_df = pd.concat([nlp_test['id'], sentiment_predictions['target']], axis=1)
 print(submission_df.shape)
 submission_df.head()
<feature_engineering><EOS> 
submission_df.to_csv("submission.csv", index=False, header=True )