Kaggle competition: Homesite Quote Conversion
<SOS> metric: AUC Kaggle data source: homesite-quote-conversion<save_to_csv>
%pylab inline
submission_file.to_csv('submission.csv', index=False )<set_options>
trainPath = r'.. /input/train.csv'
testPath = r'.. /input/test.csv'
%matplotlib inline
pd.set_option('display.max_colwidth', 240)
 if 'transformers' not in sys.modules:
!pip install transformers
if 'nltk' not in sys.modules:
!pip install nltk
nltk.download('stopwords')
 is_colab = 'google.colab' in sys.modules
if is_colab:
is_kaggle = 'kaggle' in os.getcwd()
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
 device
if torch.cuda.is_available() :
print('There are %d GPU(s)available.' % torch.cuda.device_count())
 print('We will use the GPU:', torch.cuda.get_device_name(0))<load_pretrained>
train_df = pd.read_csv(trainPath)
 test_df = pd.read_csv(testPath )
if is_colab:
drive.mount('/content/drive')
 DIR_NAME = '/content/drive/My Drive/Colab Notebooks/Spooky/data/'
elif is_kaggle:
DIR_NAME = "/kaggle/input/spooky-author-identification/"
zips = os.listdir(DIR_NAME)
 for name in zips:
with zipfile.ZipFile(DIR_NAME + name, 'r')as zip_ref:
zip_ref.extractall(".")
 else:
fnames = [r'./train.csv',r'./test.csv', r'./sample_submission.csv']
url = 'https://drive.google.com/drive/folders/1tP8T8_-6Xy5BgQa3q2g7reFHBv4Uer_a?usp=sharing'
if not os.path.exists(fnames[0]):
for fname in fnames:
urllib.request.urlretrieve(url, fname)
 print(fname, 'exists:', os.path.exists(fname))
DIR_NAME = './'<load_from_csv>
train_df['Year'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[:4]))
train_df['Month'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[5:7]))
train_df['Week'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[8:10]))
test_df['Year'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[:4]))
test_df['Month'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[5:7]))
test_df['Week'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[8:10]))
train_df.drop(['Original_Quote_Date'], axis=1,inplace=True)
 test_df.drop(['Original_Quote_Date'], axis=1,inplace=True )
if is_colab:
train = pd.read_csv(DIR_NAME+'train.csv')
 test = pd.read_csv(DIR_NAME+'test.csv')
 sample = pd.read_csv(DIR_NAME+'sample_submission.csv')
 else:
train = pd.read_csv('./train.csv')
 test = pd.read_csv('./test.csv')
 sample = pd.read_csv('./sample_submission.csv')
 train.head(3 )<load_pretrained>
notNumTypeCol = [col for col in train_df.columns if train_df[col].dtype == dtype('O')]
PRE_TRAINED_MODEL_NAME = 'distilbert-base-cased'
tokenizer = DistilBertTokenizer.from_pretrained(PRE_TRAINED_MODEL_NAME)
 bert_model = DistilBertModel.from_pretrained(PRE_TRAINED_MODEL_NAME)
 bert_embeddings = bert_model.get_input_embeddings()<categorify>
def charToInt(data_df,convDict):
for colName in convDict.keys() :
valuesCounter = convDict[colName]
valuesCounterSize = len(valuesCounter)
 mapDict = dict(zip(valuesCounter.keys() ,range(valuesCounterSize)))
 for index,record in enumerate(data_df[colName]):
data_df[colName][index] = mapDict[record]
sample_sentence = "Creative minds are uneven, and the best of fabrics have their dull spots."
tokens = tokenizer.tokenize(sample_sentence)
 token_ids = tokenizer.convert_tokens_to_ids(tokens)
 encoded = tokenizer.encode(sample_sentence)
 print(tokens)
 print(token_ids)
 print(encoded )<feature_engineering>
%pylab inline
trainPath = r'.. /input/train.csv'
testPath = r'.. /input/test.csv'
train_df = pd.read_csv(trainPath)
 test_df = pd.read_csv(testPath)
 train_df['Year'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[:4]))
train_df['Month'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[5:7]))
train_df['Week'] = train_df['Original_Quote_Date'].apply(lambda x: int(str(x)[8:10]))
test_df['Year'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[:4]))
test_df['Month'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[5:7]))
test_df['Week'] = test_df['Original_Quote_Date'].apply(lambda x: int(str(x)[8:10]))
train_df.drop(['Original_Quote_Date'], axis=1,inplace=True)
 test_df.drop(['Original_Quote_Date'], axis=1,inplace=True)
 notNumTypeCol = [col for col in train_df.columns if train_df[col].dtype == dtype('O')]
def charToInt(data_df,convDict):
for colName in convDict.keys() :
valuesCounter = convDict[colName]
valuesCounterSize = len(valuesCounter)
 mapDict = dict(zip(valuesCounter.keys() ,range(valuesCounterSize)))
 for index,record in enumerate(data_df[colName]):
data_df[colName][index] = mapDict[record]
train['word_ids'] = train['text'].apply(lambda x: tokenizer.convert_tokens_to_ids(tokenizer.tokenize(x)))
 train['word_count'] = train['word_ids'].apply(lambda x: len(x))
eng_stopword_toekns = set([tokenizer.convert_tokens_to_ids(str(x)) for x in stopwords.words("english")])
 train['stopword_count'] = train['word_ids'].apply(lambda x: len([token for token in x if token in eng_stopword_toekns]))
train.head(1 )<count_values>
train_x_df = train_df.drop("QuoteConversion_Flag",axis=1)
 train_y_df = train_df["QuoteConversion_Flag"]
def count_words(df, author_key):
df = df[df['author']==author_key]
word_count = df['text'].str.split(expand=True ).stack().value_counts()
return word_count<count_values>
charToInt(train_x_df,notNumTypeCol )
word_count_EAP = count_words(train, 'EAP')
 word_count_HPL = count_words(train, 'HPL')
 word_count_MWS = count_words(train, 'MWS')
 print(f'word_count_EAP size: {len(word_count_EAP)}')
 print(f'word_count_HPL size: {len(word_count_HPL)}')
 print(f'word_count_MWS size: {len(word_count_MWS)}' )<drop_column>
notNumValueCounter = {}
for key in col:
notNumValueCounter[key] = Counter(train_x_df[key] )
english_stopwords = stopwords.words("english")+ list(string.punctuation)
 word_count_EAP = word_count_EAP.drop(labels=english_stopwords, errors='ignore')
 word_count_HPL = word_count_HPL.drop(labels=english_stopwords, errors='ignore')
 word_count_MWS = word_count_MWS.drop(labels=english_stopwords, errors='ignore' )<drop_column>
notNumValueCounter = {}
for key in notNumTypeCol:
notNumValueCounter[key] = Counter(train_x_df[key] )
<create_dataframe><EOS> 
charToInt(train_x_df,notNumValueCounter )