Kaggle competition: House Prices - Advanced Regression Techniques
<SOS> metric: rmsle Kaggle data source: house-prices-advanced-regression-techniques<choose_model_class>
sns.set_style('whitegrid')
 warnings.simplefilter(action='ignore')
model.compile(loss='binary_crossentropy',
optimizer=Adam() ,
metrics=['acc'] )<define_variables>
a = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/train.csv')
 b = pd.read_csv('/kaggle/input/house-prices-advanced-regression-techniques/test.csv' )
BATCH_SIZE = 1024
NUM_EPOCHS = 40<train_model>
pd.set_option('display.max_columns', None)
 pd.set_option('display.max_rows', None )
model.fit(
 train_text,
train_labels,
batch_size=BATCH_SIZE,
epochs=NUM_EPOCHS,
validation_data=(validate_text, validate_labels),
callbacks = [EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)])
<load_from_csv>
print('The shape of our training set: %s houses and %s features'%(a.shape[0],a.shape[1]))
print('The shape of our training set: %s houses and %s features'%(b.shape[0],b.shape[1]))
print('The testing set has 1 feature less than the training set, which is SalePrice, the target to predict ' )
submission = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv', index_col='id')
 submission['prediction'] = model.predict(pad_text(test[TEXT_COLUMN], tokenizer))
submission.reset_index(drop=False, inplace=True)
 submission.head()<save_to_csv>
na = a.shape[0]
nb = b.shape[0]
y_train = a['SalePrice'].to_frame()
c1 = pd.concat(( a, b), sort=False ).reset_index(drop=True)
 c1.drop(['SalePrice'], axis=1, inplace=True)
 c1.drop(['Id'], axis=1, inplace=True)
 print(f"Total size is {c1.shape}" )
submission.to_csv('submission.csv', index=False )<set_options>
msv1(c1, 20, color=('black', 'deeppink'))
%matplotlib inline
pd.set_option('max_colwidth',400)
 print(os.listdir(".. /input"))
def set_seed(seed=0):
random.seed(seed)
 os.environ['PYTHONHASHSEED'] = str(seed)
 np.random.seed(seed)
 set_seed(2411)
 SEED = 42
num_partitions = 10
num_cores = psutil.cpu_count()
print('number of cores:', num_cores)
 def df_parallelize_run(df, func):
df_split = np.array_split(df, num_partitions)
 pool = Pool(num_cores)
 df = pd.concat(pool.map(func, df_split))
pool.close()
pool.join()
return df<categorify>
c=c1.dropna(thresh=len(c1)*0.8, axis=1)
 print(f"We dropped {c1.shape[1]-c.shape[1]} features in the combined set" )
tweet = '
<3 ;D :(:-('
print(f'Original Tweet ----
{tweet}')
 tweet = re.sub(r'
print(f'
Tweet after replacing hashtags ----
{tweet}')
 tweet = re.sub(r'(<3|:\*)', ' EMO_POS ', tweet)
 print(f'
Tweet after replacing Emojis for Love with EMP_POS ----
{tweet}')
 tweet = re.sub(r'(;-?\)|;-?D|\(-?;)', ' EMO_POS ', tweet)
 print(f'
Tweet after replacing Emojis for Wink with EMP_POS ----
{tweet}')
 tweet = re.sub(r'(:\s?\(|:-\(|\)\s?:|\)-:)', ' EMO_NEG ', tweet)
 print(f'
Tweet after replacing Emojis for Sad with EMP_NEG ----
{tweet}' )<categorify>
msv2(c )
tweet = re.sub(r'\s+', ' ', tweet)
 print(f'
Tweet after replacing xtra spaces ----
{tweet}')
 tweet = re.sub(r'[^\w\s]','',tweet)
 print(f'
Tweet after replacing Punctuation + with PUNC ----
{tweet}' )<define_variables>
NA=c[allna.index.to_list() ]
positive_emojis = set([
":‑)",":)",":-]",":]",":-3",":3",":->",":>","8-)","8)",":-}",":}",":o)",":c)",":^)","=]","=)",":‑D",":D","8‑D","8D",
"x‑D","xD","X‑D","XD","=D","=3","B^D",":-)) ",";‑)",";)","*-)","*)",";‑]",";]",";^)",":‑,",";D",":‑P",":P","X‑P","XP",
"x‑p","xp",":‑p",":p",":‑Þ",":Þ",":‑þ",":þ",":‑b",":b","d:","=p",">:P", ":'‑)", ":')", ":-*", ":*", ":×"
])
 negative_emojis = set([
":‑(",":(",":‑c",":c",":‑<",":<",":‑[",":[",":-||",">:[",":{",":@",">:(","D‑':","D:<","D:","D8","D;","D=","DX",":‑/",
":/",":‑.",'>:\', ">:/", ":\", "=/" ,"=\", ":L", "=L",":S",":‑|",":|","|‑O","<:‑|"
])
 del positive_emojis, negative_emojis<define_variables>
c['MasVnrArea']=c.MasVnrArea.fillna(0)
 c['LotFrontage']=c.LotFrontage.fillna(c.LotFrontage.median())
 c['GarageYrBlt']=c["GarageYrBlt"].fillna(1980 )
pattern = r'(19|20)\d\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])'<define_variables>
ffill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
def filling_NA(data, columns, METHOD='ffill'):
fill_cols = columns
for col in data[fill_cols]:
data[col]= data[col].fillna(method=METHOD)
 return data
pattern = r'\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b'<load_from_csv>
msv2(c )
%%time
train = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/train.csv',usecols=['comment_text', 'target'])
 test = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')
 sub = pd.read_csv('.. /input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')
 train.shape, test.shape<string_transform>
d=filling_NA(c, ffill_cols )
spaces = ['\u200b', '\u200e', '\u202a', '\u202c', '\ufeff', '\uf0d8', '\u2061', '\x10', '\x7f', '\x9d', '\xad', '\xa0']
def remove_space(text):

 for space in spaces:
text = text.replace(space, ' ')
 text = text.strip()
text = re.sub('\s+', ' ', text)
 return text
def remove_diacritics(s):
return ''.join(c for c in normalize('NFKD', s.replace('ø', 'o' ).replace('Ø', 'O' ).replace('⁻', '-' ).replace('₋', '-'))
if category(c)!= 'Mn')
 special_punc_mappings = {"—": "-", "–": "-", "_": "-", '”': '"', "″": '"', '“': '"', '•': '.', '−': '-',
"’": "'", "‘": "'", "´": "'", "`": "'", '\u200b': ' ', '\xa0': ' ','،':'','„':'',
'…': '...', '\ufeff': ''}
def clean_special_punctuations(text):
for punc in special_punc_mappings:
if punc in text:
text = text.replace(punc, special_punc_mappings[punc])
 text = remove_diacritics(text)
 return text
def clean_number(text):
if bool(re.search(r'\d', text)) :
text = re.sub(r'(\d+ )([a-zA-Z])', '\g<1> \g<2>', text)
 text = re.sub(r'(\d+ )(th|st|nd|rd)', '\g<1>\g<2> ', text)
 text = re.sub(r'(\d+),(\d+)', '\g<1>\g<2>', text)
 return text
regular_punct = list(string.punctuation)
 extra_punct = [
',', '.', '"', ':', ')', '(', '!', '?', '|', ';', "'", '$', '&',
'/', '[', ']', '>', '%', '=', '
'·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›',
'♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '“', '★', '”',
'–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾',
'═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼',
'▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',
'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',
'，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',
'¹', '≤', '‡', '√', '«', '»', '´', 'º', '¾', '¡', '§', '£', '₤',
':)', ':)', ':-)', '(:', '(:', '(-:', ':')',
':D', ': D', ':-D', 'xD', 'x-D', 'XD', 'X-D',
'<3', ':*',
';-)', ';)', ';-D', ';D', '(;', '(-;',
':-(', ':(', ':(', ''):', ')-:',
'-- :','(', ':'(', ':"('',]
def handle_emojis(text):
text = re.sub(r'(:\s?\)|:-\)|\(\s?:|\(-:|:'\)) ', ' EMO_POS ', text)
 text = re.sub(r'(:\s?D|:-D|x-?D|X-?D)', ' EMO_POS ', text)
 text = re.sub(r'(<3|:\*)', ' EMO_POS ', text)
 text = re.sub(r'(;-?\)|;-?D|\(-?;)', ' EMO_POS ', text)
 text = re.sub(r'(:\s?\(|:-\(|\)\s?:|\)-:)', ' EMO_NEG ', text)
 text = re.sub(r'(:,\(|:'\(|:"\() ', ' EMO_NEG ', text)
 return text
def stop(text):
text = " ".join([w.lower() for w in text.split() ])
 stop_words = stopwords.words('english')
 words = [w for w in text.split() if not w in stop_words]
return " ".join(words)
 all_punct = list(set(regular_punct + extra_punct))
all_punct.remove('-')
 all_punct.remove('.')
 def clean_repeat_words(text):
text = re.sub(r"(I|i )(I|i)+ng", "ing", text)
 text = re.sub(r"(L|l )(L|l )(L|l)+y", "lly", text)
 text = re.sub(r"(A|a )(A|a )(A|a)+", "a", text)
 text = re.sub(r"(C|c )(C|c )(C|c)+", "cc", text)
 text = re.sub(r"(D|d )(D|d )(D|d)+", "dd", text)
 text = re.sub(r"(E|e )(E|e )(E|e)+", "ee", text)
 text = re.sub(r"(F|f )(F|f )(F|f)+", "ff", text)
 text = re.sub(r"(G|g )(G|g )(G|g)+", "gg", text)
 text = re.sub(r"(I|i )(I|i )(I|i)+", "i", text)
 text = re.sub(r"(K|k )(K|k )(K|k)+", "k", text)
 text = re.sub(r"(L|l )(L|l )(L|l)+", "ll", text)
 text = re.sub(r"(M|m )(M|m )(M|m)+", "mm", text)
 text = re.sub(r"(N|n )(N|n )(N|n)+", "nn", text)
 text = re.sub(r"(O|o )(O|o )(O|o)+", "oo", text)
 text = re.sub(r"(P|p )(P|p )(P|p)+", "pp", text)
 text = re.sub(r"(Q|q )(Q|q)+", "q", text)
 text = re.sub(r"(R|r )(R|r )(R|r)+", "rr", text)
 text = re.sub(r"(S|s )(S|s )(S|s)+", "ss", text)
 text = re.sub(r"(T|t )(T|t )(T|t)+", "tt", text)
 text = re.sub(r"(V|v )(V|v)+", "v", text)
 text = re.sub(r"(Y|y )(Y|y )(Y|y)+", "y", text)
 text = re.sub(r"plzz+", "please", text)
 text = re.sub(r"(Z|z )(Z|z )(Z|z)+", "zz", text)
 text = re.sub(r"(-+|\.+)", " ", text)
 return text
def spacing_punctuation(text):

 for punc in all_punct:
if punc in text:
text = text.replace(punc, f' {punc} ')
 return text
def preprocess(text):

 text = remove_space(text)
 text = clean_special_punctuations(text)
 text = handle_emojis(text)
 text = clean_number(text)
 text = spacing_punctuation(text)
 text = clean_repeat_words(text)
 text = remove_space(text)
 return text
mispell_dict = {'😉':'wink','😂':'joy','😀':'stuck out tongue', 'theguardian':'the guardian','deplorables':'deplorable', 'theglobeandmail':'the globe and mail', 'justiciaries': 'justiciary','creditdation': 'Accreditation','doctrne':'doctrine','fentayal': 'fentanyl','designation-': 'designation','CONartist' : 'con-artist','Mutilitated' : 'Mutilated','Obumblers': 'bumblers','negotiatiations': 'negotiations','dood-': 'dood','irakis' : 'iraki','cooerate': 'cooperate','COx':'cox','racistcomments':'racist comments','envirnmetalists': 'environmentalists',}
contraction_mapping = {"ain't": "is not", "aren't": "are not","can't": "cannot", "'cause": "because", "could've": "could have", "couldn't": "could not", "didn't": "did not", "doesn't": "does not", "don't": "do not", "hadn't": "had not", "hasn't": "has not", "haven't": "have not", "he'd": "he would","he'll": "he will", "he's": "he is", "how'd": "how did", "how'd'y": "how do you", "how'll": "how will", "how's": "how is", "I'd": "I would", "I'd've": "I would have", "I'll": "I will", "I'll've": "I will have","I'm": "I am", "I've": "I have", "i'd": "i would", "i'd've": "i would have", "i'll": "i will", "i'll've": "i will have","i'm": "i am", "i've": "i have", "isn't": "is not", "it'd": "it would", "it'd've": "it would have", "it'll": "it will", "it'll've": "it will have","it's": "it is", "let's": "let us", "ma'am": "madam", "mayn't": "may not", "might've": "might have","mightn't": "might not","mightn't've": "might not have", "must've": "must have", "mustn't": "must not", "mustn't've": "must not have", "needn't": "need not", "needn't've": "need not have","o'clock": "of the clock", "oughtn't": "ought not", "oughtn't've": "ought not have", "shan't": "shall not", "sha'n't": "shall not", "shan't've": "shall not have", "she'd": "she would", "she'd've": "she would have", "she'll": "she will", "she'll've": "she will have", "she's": "she is", "should've": "should have", "shouldn't": "should not", "shouldn't've": "should not have", "so've": "so have","so's": "so as", "this's": "this is","that'd": "that would", "that'd've": "that would have", "that's": "that is", "there'd": "there would", "there'd've": "there would have", "there's": "there is", "here's": "here is","they'd": "they would", "they'd've": "they would have", "they'll": "they will", "they'll've": "they will have", "they're": "they are", "they've": "they have", "to've": "to have", "wasn't": "was not", "we'd": "we would", "we'd've": "we would have", "we'll": "we will", "we'll've": "we will have", "we're": "we are", "we've": "we have", "weren't": "were not", "what'll": "what will", "what'll've": "what will have", "what're": "what are", "what's": "what is", "what've": "what have", "when's": "when is", "when've": "when have", "where'd": "where did", "where's": "where is", "where've": "where have", "who'll": "who will", "who'll've": "who will have", "who's": "who is", "who've": "who have", "why's": "why is", "why've": "why have", "will've": "will have", "won't": "will not", "won't've": "will not have", "would've": "would have", "wouldn't": "would not", "wouldn't've": "would not have", "y'all": "you all", "y'all'd": "you all would","y'all'd've": "you all would have","y'all're": "you all are","y'all've": "you all have","you'd": "you would", "you'd've": "you would have", "you'll": "you will", "you'll've": "you will have", "you're": "you are", "you've": "you have" }
def correct_spelling(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x
def correct_contraction(x, dic):
for word in dic.keys() :
x = x.replace(word, dic[word])
 return x<categorify>
msv2(d )
%%time
tqdm.pandas()
def text_clean_wrapper(df):
df["comment_text"] = df["comment_text"].astype('str' ).transform(preprocess)
 df['comment_text'] = df['comment_text'].transform(lambda x: correct_spelling(x, mispell_dict))
df['comment_text'] = df['comment_text'].transform(lambda x: correct_contraction(x, contraction_mapping))
return df
train = df_parallelize_run(train, text_clean_wrapper)
 test = df_parallelize_run(test, text_clean_wrapper)
 gc.enable()
del mispell_dict, all_punct, special_punc_mappings, regular_punct, extra_punct
gc.collect()<choose_model_class>
fill_cols = ['Electrical', 'SaleType', 'KitchenQual', 'Exterior1st',
'Exterior2nd', 'Functional', 'Utilities', 'MSZoning']
for col in c[fill_cols]:
c[col] = c[col].fillna(method='ffill' )
ps = PorterStemmer()
lc = LancasterStemmer()
sb = SnowballStemmer("english")
 class Attention(Layer):
def __init__(self, step_dim,
W_regularizer=None, b_regularizer=None,
W_constraint=None, b_constraint=None,
bias=True, **kwargs):
self.supports_masking = True
self.init = initializers.get('glorot_uniform')
 self.W_regularizer = regularizers.get(W_regularizer)
 self.b_regularizer = regularizers.get(b_regularizer)
 self.W_constraint = constraints.get(W_constraint)
 self.b_constraint = constraints.get(b_constraint)
 self.bias = bias
self.step_dim = step_dim
self.features_dim = 0
super(Attention, self ).__init__(**kwargs)
 def build(self, input_shape):
assert len(input_shape)== 3
self.W = self.add_weight(( input_shape[-1],),
initializer=self.init,
name='{}_W'.format(self.name),
regularizer=self.W_regularizer,
constraint=self.W_constraint)
 self.features_dim = input_shape[-1]
if self.bias:
self.b = self.add_weight(( input_shape[1],),
initializer='zero',
name='{}_b'.format(self.name),
regularizer=self.b_regularizer,
constraint=self.b_constraint)
 else:
self.b = None
self.built = True
def compute_mask(self, input, input_mask=None):
return None
def call(self, x, mask=None):
features_dim = self.features_dim
step_dim = self.step_dim
eij = K.reshape(K.dot(K.reshape(x,(-1, features_dim)) ,
K.reshape(self.W,(features_dim, 1))),(-1, step_dim))
if self.bias:
eij += self.b
eij = K.tanh(eij)
 a = K.exp(eij)
 if mask is not None:
a *= K.cast(mask, K.floatx())
 a /= K.cast(K.sum(a, axis=1, keepdims=True)+ K.epsilon() , K.floatx())
 a = K.expand_dims(a)
 weighted_input = x * a
return K.sum(weighted_input, axis=1)
 def compute_output_shape(self, input_shape):
return input_shape[0], self.features_dim<define_variables>
NAcols=c.columns
for col in NAcols:
if c[col].dtype == "object":
c[col] = c[col].fillna("None" )
start_time = time.time()
print("Loading data...")
 full_text = pd.concat([train['comment_text'].astype(str), test['comment_text'].astype(str)])
 y = train['target']
num_train_data = y.shape[0]
print("--- %s seconds ---" %(time.time() - start_time))
start_time = time.time()
print("Spacy NLP...")
 nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])
 nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)
 word_dict = {}
word_index = 1
lemma_dict = {}
docs = nlp.pipe(full_text, n_threads = 4)
 word_sequences = []
for doc in tqdm(docs):
word_seq = []
for token in doc:
if(token.text not in word_dict)and(token.pos_ is not "PUNCT"):
word_dict[token.text] = word_index
word_index += 1
lemma_dict[token.text] = token.lemma_
if token.pos_ is not "PUNCT":
word_seq.append(word_dict[token.text])
 word_sequences.append(word_seq)
 del docs
gc.collect()
train_word_sequences = word_sequences[:num_train_data]
test_word_sequences = word_sequences[num_train_data:]
print("--- %s seconds ---" %(time.time() - start_time))<categorify>
for col in NAcols:
if c[col].dtype != "object":
c[col]= c[col].fillna(0 )
max_len = 128
X_train = pad_sequences(train_word_sequences, maxlen = max_len)
 X_test = pad_sequences(test_word_sequences, maxlen = max_len)
 del train_word_sequences, test_word_sequences
gc.collect()<init_hyperparams>
c['TotalArea'] = c['TotalBsmtSF'] + c['1stFlrSF'] + c['2ndFlrSF'] + c['GrLivArea'] +c['GarageArea']
c['Bathrooms'] = c['FullBath'] + c['HalfBath']*0.5
c['Year average']=(c['YearRemodAdd']+c['YearBuilt'])/2
embed_size = 300 
<features_selection>
c['MSSubClass'] = c['MSSubClass'].apply(str)
 c['YrSold'] = c['YrSold'].astype(str )
def load_glove(word_dict, lemma_dict):
EMBEDDING_FILE = '.. /input/glove840b300dtxt/glove.840B.300d.txt'
def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')
 embeddings_index = dict(get_coefs(*o.split(" ")) for o in open(EMBEDDING_FILE)if len(o)>100 and o.split(" ")[0] in word_dict.keys())
 embed_size = 300
nb_words = len(word_dict)+1
embedding_matrix = np.zeros(( nb_words, embed_size), dtype=np.float32)
 unknown_vector = np.zeros(( embed_size,), dtype=np.float32)- 1.
 print(unknown_vector[:5])
 for key in tqdm(word_dict):
word = key
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.lower()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.upper()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = key.capitalize()
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = ps.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lc.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = sb.stem(key)
 embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
word = lemma_dict[key]
embedding_vector = embeddings_index.get(word)
 if embedding_vector is not None:
embedding_matrix[word_dict[key]] = embedding_vector
continue
embedding_matrix[word_dict[key]] = unknown_vector
return embedding_matrix, nb_words
print("Loading embedding matrix...")
 embedding_matrix, nb_words = load_glove(word_dict, lemma_dict)
 max_features = nb_words
print("--- %s seconds ---" %(time.time() - start_time)) 
<prepare_x_and_y>
cb=pd.get_dummies(c)
 print(f"the shape of the original dataset {c.shape}")
 print(f"the shape of the encoded dataset {cb.shape}")
 print(f"We have {cb.shape[1]- c.shape[1]} new encoded features" )
y = np.where(train['target'] >= 0.5, True, False)* 1<choose_model_class>
Train = cb[:na]
Test = cb[na:]
K.clear_session()
def build_model(lr=0.0, lr_d=0.0, spatial_dr=0.0, dense_units=128, dr=0.1):
file_path = "best_model.hdf5"
check_point = ModelCheckpoint(file_path, monitor = "val_loss", verbose = 1,
save_best_only = True, mode = "min")
 early_stop = EarlyStopping(monitor = "val_loss", mode = "min", patience = 3)
 inp = Input(shape=(max_len,))
x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False )(inp)
 x = SpatialDropout1D(spatial_dr )(x)
 x1 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)
 x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)
 max_pool1 = GlobalMaxPooling1D()(x1)
 max_pool2 = GlobalMaxPooling1D()(x2)
 conc = Concatenate()([max_pool1, max_pool2])
 predictions = Dense(1, activation='sigmoid' )(conc)
 model = Model(inputs=inp, outputs=predictions)
 model.compile(loss = "binary_crossentropy", optimizer = Adam(lr = lr, decay = lr_d), metrics = ["accuracy"])
 history = model.fit(X_train, y, batch_size = 512, epochs = 15, validation_split=0.1,
verbose = 1, callbacks = [check_point, early_stop])
 return model<choose_model_class>
a['GrLivArea'].sort_values(ascending=False ).head(2 )
model = build_model(lr = 1e-3, lr_d = 0.001, spatial_dr = 0.23, dr=0.2)
 del X_train, embedding_matrix
gc.collect()<predict_on_test>
a['TotalBsmtSF'].sort_values(ascending=False ).head(1 )
pred = model.predict(X_test, batch_size = 512, verbose = 1 )<save_to_csv>
a['MasVnrArea'].sort_values(ascending=False ).head(1 )
sub['prediction'] = pred
sub.to_csv('submission.csv', index=False )<predict_on_test>
a['1stFlrSF'].sort_values(ascending=False ).head(1 )
pred = model.predict(X_test, batch_size = 512, verbose = 1)
 plot_model(model, to_file='model.png' )<import_modules>
a['TotRmsAbvGrd'].sort_values(ascending=False ).head(1 )
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from datetime import date, datetime, timedelta
from sklearn.metrics import accuracy_score
from kaggle.competitions import twosigmanews<set_options>
train=Train[(Train['GrLivArea'] < 4600)&(Train['MasVnrArea'] < 1500)]
print(f'We removed {Train.shape[0]- train.shape[0]} outliers' )
env = twosigmanews.make_env()<define_variables>
target=a[['SalePrice']]
target.loc[1298]
cat_cols = ['assetCodeT']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1', 'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10',
'returnsClosePrevMktres10', 'returnsOpenPrevMktres10', 'close_to_open', 'rolling_average_close_mean', 'rolling_average_vol_mean', 'rolling_average_close_std', 'ewma', 'rolling_average_close_std', 'sourceTimestamp', 'urgency', 'companyCount', 'takeSequence', 'bodySize', 'sentenceCount',
'moving_average_7_day','relevance', 'sentimentClass', 'sentimentNegative', 'sentimentNeutral', 'sentimentPositive',
'noveltyCount24H','noveltyCount7D','volumeCounts24H','volumeCounts7D','assetCodesLen', 'asset_sentiment_count', 'len_audiences', 'tf_score']<split>
target.loc[523]
train_indices, val_indices = train_test_split(market_train.index.values,
test_size=0.25,
random_state=42 )<categorify>
pos = [1298,523, 297]
target.drop(target.index[pos], inplace=True )
def encode(encoder, x):
len_encoder = len(encoder)
 try:
id = encoder[x]
except KeyError:
id = len_encoder
return id
encoders = [{} for cat in cat_cols]
market_train['assetCodeT'] = market_train['assetCode'].astype(str ).apply(lambda x: encode(encoders[0], x))
embed_sizes = [len(encoder)+ 1 for encoder in encoders]<choose_model_class>
print('We make sure that both train and target sets have the same row number after removing the outliers:')
 print('Train: ',train.shape[0], 'rows')
 print('Target:', target.shape[0],'rows' )
categorical_inputs = []
for cat in cat_cols:
categorical_inputs.append(Input(shape=[1], name=cat))
categorical_embeddings = []
for i, cat in enumerate(cat_cols):
categorical_embeddings.append(Embedding(embed_sizes[i], 10 )(categorical_inputs[i]))
categorical_logits = Flatten()(categorical_embeddings[0])
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 categorical_logits = Dropout(0.5 )(categorical_logits)
 categorical_logits = BatchNormalization()(categorical_logits)
 categorical_logits = Dense(32,activation='relu' )(categorical_logits)
 numerical_inputs = Input(shape=(len(num_cols),), name='num')
 numerical_logits = numerical_inputs
numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dropout(0.5 )(numerical_logits)
 numerical_logits = BatchNormalization()(numerical_logits)
 numerical_logits = Dense(128,activation='relu' )(numerical_logits)
 numerical_logits = Dense(64,activation='relu' )(numerical_logits)
 logits = Concatenate()([numerical_logits,categorical_logits])
 logits = Dense(64,activation='relu' )(logits)
 out = Dense(1, activation='sigmoid' )(logits)
 model = Model(inputs = categorical_inputs + [numerical_inputs], outputs=out)
 model.compile(optimizer='adam',loss=binary_crossentropy )<train_model>
from scipy.stats import skew
env = twosigmanews.make_env() 
(market_train, _)= env.get_training_data()<import_modules>
target["SalePrice"] = np.log1p(target["SalePrice"] )
pd.__version__<split>
x=train
y=np.array(target )
cat_cols = ['assetCode']
num_cols = ['volume', 'close', 'open', 'returnsClosePrevRaw1', 'returnsOpenPrevRaw1', 'returnsClosePrevMktres1',
'returnsOpenPrevMktres1', 'returnsClosePrevRaw10', 'returnsOpenPrevRaw10', 'returnsClosePrevMktres10',
'returnsOpenPrevMktres10']
train_indices, val_indices = train_test_split(market_train.index.values,test_size=0 )<categorify>
x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0 )
def encode(encoder, x):
len_encoder = len(encoder)
 try:
ind = encoder[x]
except KeyError:
ind = len_encoder
return ind
encoders = [{} for cat in cat_cols]
for i, cat in enumerate(cat_cols):
print('encoding %s'%cat, end='
')
 encoders[i] = {l:ind for ind, l in enumerate(market_train[cat].unique())}
market_train[cat] = market_train[cat].apply(lambda x:encode(encoders[i],x))
embed_sizes = [len(encoder)+ 1 for encoder in encoders]<normalization>
scaler= RobustScaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test )
market_train[num_cols] = market_train[num_cols].fillna(0)
 scaler = StandardScaler()
market_train[num_cols] = scaler.fit_transform(market_train[num_cols] )<choose_model_class>
cat_inputs = []
for cat in cat_cols:
cat_inputs.append(Input(shape=[1], name=cat))
cat_embs = []
for i,cat in enumerate(cat_cols):
cat_embs.append(Embedding(embed_sizes[i], 10 )(cat_inputs[i]))
cat_logits = Flatten()(cat_embs[0])
 cat_logits = Dense(32 )(cat_logits)
 cat_logits = LeakyReLU(0.1 )(cat_logits)
 cat_logits = Dropout(0.5 )(cat_logits)
 num_input = Input(shape =(len(num_cols),), name='num')
 num_logits = num_input
num_logits = BatchNormalization()(num_logits)
 num_logits = Dense(32 )(num_logits)
 num_logits = LeakyReLU(0.1 )(num_logits)
 num_logits = Dropout(0.5 )(num_logits)
 all_logits = Concatenate()([num_logits, cat_logits])
 logits = Dense(128, activation='relu' )(all_logits)
 logits = Dropout(0.5 )(logits)
 logits = Dense(64, activation='relu' )(logits)
 logits = Dropout(0.5 )(logits)
 out = Dense(1, activation='tanh' )(logits)
 model = Model(inputs = cat_inputs+[num_input],outputs=out)
 model.compile(optimizer='adam', loss=mae )<feature_engineering>
ridge=Ridge()
parameters= {'alpha':[x for x in range(1,101)]}
ridge_reg=ms.GridSearchCV(ridge, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 ridge_reg.fit(x_train,y_train)
 print(f"The best value of Alpha is: {ridge_reg.best_params_}")
 print(f"The best score achieved with Alpha=11 is: {math.sqrt(-ridge_reg.best_score_)}")
 ridge_pred=math.sqrt(-ridge_reg.best_score_ )
def get_input(market_train, indices):
X_num = market_train.loc[indices, num_cols].values
X = {'num':X_num}
for cat in cat_cols:
X[cat] = market_train.loc[indices, cat].values
market_train.loc[indices, 'returnsOpenNextMktres10'] = market_train.loc[indices, 'returnsOpenNextMktres10'].apply(lambda x: 0 if x < -0.3 or x > 0.3else x)
 market_train.loc[indices, 'returnsOpenNextMktres10'] = market_train.loc[indices, 'returnsOpenNextMktres10'].apply(lambda x: -1 if x < 0 else x)
 market_train.loc[indices, 'returnsOpenNextMktres10'] = market_train.loc[indices, 'returnsOpenNextMktres10'].apply(lambda x: 1 if x > 0 else x)
 y = market_train.loc[indices, 'returnsOpenNextMktres10']
y = y.values
r = market_train.loc[indices,'returnsOpenNextMktres10'].values
u = market_train.loc[indices, 'universe']
d = market_train.loc[indices, 'time'].dt.date
return X,y,r,u,d
X_train,y_train,r_train,u_train,d_train = get_input(market_train, train_indices)
 X_valid,y_valid,r_valid,u_valid,d_valid = get_input(market_train, val_indices )<train_model>
ridge_mod=Ridge(alpha=15)
 ridge_mod.fit(x_train,y_train)
 y_pred_train=ridge_mod.predict(x_train)
 y_pred_test=ridge_mod.predict(x_test)
 print(f'Root Mean Square Error train = {str(math.sqrt(sklm.mean_squared_error(y_train, y_pred_train)))}')
 print(f'Root Mean Square Error test = {str(math.sqrt(sklm.mean_squared_error(y_test, y_pred_test)))}' )
check_point = ModelCheckpoint('model.hdf5', verbose=True, save_best_only=True)
 early_stop= EarlyStopping(patience=2, verbose=True)
 model.fit(X_train, y_train,
validation_data =(X_valid, y_valid),
epochs = 4,
verbose = True,
callbacks=[check_point, early_stop] )<normalization>
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
lasso=Lasso()
lasso_reg=ms.GridSearchCV(lasso, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 lasso_reg.fit(x_train,y_train)
 print(f'The best value of Alpha is: {lasso_reg.best_params_}' )
days = env.get_prediction_days()
n_days = 0
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days += 1
if n_days % 10 == 0:
print(n_days,end='
')
 market_obs_df[num_cols] = market_obs_df[num_cols].fillna(0)
 market_obs_df[num_cols] = scaler.transform(market_obs_df[num_cols])
 X_num_test = market_obs_df[num_cols].values
X_test = {'num':X_num_test}
test_cat_cols = []
for i in range(len(cat_cols)) :
market_obs_df[cat_cols[i]+'_encoded'] = market_obs_df[cat_cols[i]].astype(str ).apply(lambda x: encode(encoders[i], x))
X_test[cat_cols[i]] = market_obs_df[cat_cols[i]+'_encoded']
market_prediction = model.predict(X_test)
 preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'], 'confidence':market_prediction.reshape(-1)})
 predictions_template_df = predictions_template_df.merge(preds, how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 env.write_submission_file()
print('Done!' )<set_options>
lasso_mod=Lasso(alpha=0.0009)
 lasso_mod.fit(x_train,y_train)
 y_lasso_train=lasso_mod.predict(x_train)
 y_lasso_test=lasso_mod.predict(x_test)
 print(f'Root Mean Square Error train {str(math.sqrt(sklm.mean_squared_error(y_train, y_lasso_train)))}')
 print(f'Root Mean Square Error test {str(math.sqrt(sklm.mean_squared_error(y_test, y_lasso_test)))}' )
pd.options.mode.chained_assignment = None
%matplotlib inline
init_notebook_mode(connected=True)
 cf.go_offline()<import_modules>
print(f"Lasso kept {sum(coefs != 0)} important features and dropped the other {sum(coefs == 0)} features" )
env = twosigmanews.make_env()
print('Done!' )<train_model>
from sklearn.linear_model import ElasticNetCV
( market_train_df, news_train_df)= env.get_training_data()
print(market_train_df['time'].head)
 print(market_train_df.shape )<feature_engineering>
alphas = [0.000542555]
l1ratio = [0.1, 0.3,0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print(f'Root Mean Square Error test = {str(math.sqrt(sklm.mean_squared_error(y_test, ela_pred)))}')
 print(elastic_cv.alpha_)
 print(elastic_cv.l1_ratio_)
market_train_df['date'] = market_train_df['time'].dt.strftime('%Y-%m-%d')
 print(market_train_df['time'].dtype)
 print(market_train_df['date'].dtype)
 market_train_df['time'] = market_train_df['time'].dt.date
market_train_df = market_train_df.loc[market_train_df['time']>=date(2010, 1, 1)]
print(market_train_df.shape )<define_variables>
def regularization(x,y,modelo=Ridge, scaler=RobustScaler):

 x_train, x_test, y_train, y_test = train_test_split(x, y,test_size =.3, random_state=0)
 scaler= scaler()
x_train = scaler.fit_transform(x_train)
 x_test = scaler.transform(x_test)
 X_test= scaler.transform(Test)
 if modelo != ElasticNetCV:
if modelo == Ridge:
parameters= {'alpha':[x for x in range(1,101)]}
elif modelo == Lasso:
parameters= {'alpha':[0.0001,0.0009,0.001,0.002,0.003,0.01,0.1,1,10,100]}
model=modelo()
model=ms.GridSearchCV(model, param_grid=parameters, scoring='neg_mean_squared_error', cv=15)
 model.fit(x_train,y_train)
 y_pred= model.predict(x_test)
 print("The best RMSE score achieved with %s is: %s " %(model.best_params_,
str(math.sqrt(sklm.mean_squared_error(y_test, y_pred)))))
 elif modelo == ElasticNetCV:
alphas = [0.000542555]
l1ratio = [0.1, 0.3,0.5, 0.9, 0.95, 0.99, 1]
elastic_cv = ElasticNetCV(cv=5, max_iter=1e7, alphas=alphas, l1_ratio=l1ratio)
 elasticmod = elastic_cv.fit(x_train, y_train.ravel())
 ela_pred=elasticmod.predict(x_test)
 print("The best RMSE score achieved with alpha %s and l1_ratio %s is: %s "
%(elastic_cv.alpha_,elastic_cv.l1_ratio_,
str(math.sqrt(sklm.mean_squared_error(y_test, ela_pred)))))
def sampleAssetData(assetCode, date, numDays):
d = datetime.strptime(date,'%Y-%m-%d')
 start = d - timedelta(days=numDays)
 end = d + timedelta(days=numDays)
 return market_train_df[(market_train_df['assetCode'] == assetCode)
 &(market_train_df['time'] >= start.date())
 &(market_train_df['time'] <= end.date())].copy()
def updateRawReturns(assetData, indices):
rowsToUpdate1 = assetData[(assetData.index >= indices[0][0])&(assetData.index <= indices[0][1])]
for index, row in rowsToUpdate1.iterrows() :
market_train_df.loc[[index],['returnsClosePrevRaw1']] = assetData['close'].pct_change()
market_train_df.loc[[index],['returnsOpenPrevRaw1']] = assetData['open'].pct_change()
rowsToUpdate2 = assetData[(assetData.index >= indices[1][0])&(assetData.index <= indices[1][1])]
for index, row in rowsToUpdate2.iterrows() :
market_train_df.loc[[index],['returnsClosePrevRaw10']] = assetData['close'].pct_change(periods=10)
 market_train_df.loc[[index],['returnsOpenPrevRaw10']] = assetData['open'].pct_change(periods=10)
 def estimateMktresReturn(sampleData, mktresCol, index):
sampleData['ones'] = 1
sampleData.dropna(inplace=True)
 rawCol = mktresCol.replace('Mktres','Raw')
 A = sampleData[[rawCol,'ones']]
y = sampleData[mktresCol]
m, c = np.linalg.lstsq(A,y,rcond=-1)[0]
return c + m * market_train_df.loc[index,rawCol]
def updateMktresReturns(assetCode, assetData, indices):
sample1 = assetData[(assetData.index < indices[2][0])|(assetData.index > indices[2][1])]
rowsToUpdate1 = assetData[(assetData.index >= indices[2][0])&(assetData.index <= indices[2][1])]
for index, row in rowsToUpdate1.iterrows() :
market_train_df.loc[[index],['returnsClosePrevMktres1']] = estimateMktresReturn(sample1,'returnsClosePrevMktres1',index)
 market_train_df.loc[[index],['returnsOpenPrevMktres1']] = estimateMktresReturn(sample1,'returnsOpenPrevMktres1',index)
 sample2 = assetData[(assetData.index < indices[3][0])|(assetData.index > indices[3][1])]
rowsToUpdate2 = assetData[(assetData.index >= indices[3][0])&(assetData.index <= indices[3][1])]
l = []
for index, row in rowsToUpdate2.iterrows() :
market_train_df.loc[[index],['returnsClosePrevMktres10']] = estimateMktresReturn(sample2,'returnsClosePrevMktres10',index)
 est = estimateMktresReturn(sample2,'returnsOpenPrevMktres10',index)
 l.append(est)
 market_train_df.loc[[index],['returnsOpenPrevMktres10']] = est
rowsToUpdate3 = assetData[(assetData.index >= indices[4][0])&(assetData.index <= indices[4][1])]
i = 0
for index, row in rowsToUpdate3.iterrows() :
market_train_df.loc[[index],['returnsOpenNextMktres10']] = l[i]
i += 1
def fixBadReturnData(assetCode, badDate, badIndex, badReturnDataRanges, dayWindow):
badDataWindow = sampleAssetData(assetCode,badDate,dayWindow)
 badDataWindow.reset_index(inplace=True)
 newIdx = badDataWindow[badDataWindow['index'] == badIndex].index[0]
indices = [
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[0],'index']],
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[1],'index']],
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[2],'index']],
[badIndex,badDataWindow.loc[newIdx+badReturnDataRanges[3],'index']],
[badDataWindow.loc[newIdx+badReturnDataRanges[4],'index'],badDataWindow.loc[newIdx+badReturnDataRanges[5],'index']]
]
badDataWindow.set_index('index',inplace=True)
 updateRawReturns(badDataWindow,indices)
 updateMktresReturns(assetCode,badDataWindow,indices)
<feature_engineering>
regularization(x,y,Ridge )
assetCode = 'ZNGA.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 19213100
market_train_df.loc[[badIndex],['open']] = 2.64
market_train_df.loc[[badIndex],['close']] = 2.75
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'FLEX.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 5406600
market_train_df.loc[[badIndex],['open']] = 11.580
market_train_df.loc[[badIndex],['close']] = 11.750
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'SHLD.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 279300
market_train_df.loc[[badIndex],['open']] = 12.8900
market_train_df.loc[[badIndex],['close']] = 13.1400
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'MAT.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 3242100
market_train_df.loc[[badIndex],['open']] = 32.13
market_train_df.loc[[badIndex],['close']] = 31.52
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'BBBY.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 4205500
market_train_df.loc[[badIndex],['open']] = 42.23
market_train_df.loc[[badIndex],['close']] = 43.55
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'DISH.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 2303300
market_train_df.loc[[badIndex],['open']] = 50.06
market_train_df.loc[[badIndex],['close']] = 51.33
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'NDAQ.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 733400
market_train_df.loc[[badIndex],['open']] = 64.64
market_train_df.loc[[badIndex],['close']] = 64.74
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'PCAR.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 2394300
market_train_df.loc[[badIndex],['open']] = 50.16
market_train_df.loc[[badIndex],['close']] = 50.79
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow)
 assetCode = 'PZZA.O'
badDate = '2016-07-06'
badIndex = market_train_df[(market_train_df['assetCode'] == assetCode)&(market_train_df['date'] == badDate)].index[0]
market_train_df.loc[[badIndex],['volume']] = 185100
market_train_df.loc[[badIndex],['open']] = 67.86
market_train_df.loc[[badIndex],['close']] = 67.91
badReturnDataRanges = [1,10,20,20,-11,9]
sampleWindow = 45
fixBadReturnData(assetCode,badDate,badIndex,badReturnDataRanges,sampleWindow )<merge>
regularization(x,y, Lasso )
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
n_lag = [3,7,14]
new_df = generate_lag_features(market_train_df,n_lag=n_lag)
 market_train_df = pd.merge(market_train_df,new_df,how='left',on=['time','assetCode'] )<categorify>
regularization(x,y, ElasticNetCV )
def mis_impute(data):
for i in data.columns:
if data[i].dtype == "object":
data[i] = data[i].fillna("other")
 elif(data[i].dtype == "int64" or data[i].dtype == "float64"):
data[i] = data[i].fillna(data[i].mean())
 else:
pass
return data
market_train_df = mis_impute(market_train_df )<categorify>
from xgboost.sklearn import XGBRegressor
def data_prep(market_train):
lbl = {k: v for v, k in enumerate(market_train['assetCode'].unique())}
market_train['assetCodeT'] = market_train['assetCode'].map(lbl)
 market_train = market_train.dropna(axis=0)
 return market_train
market_train_df = data_prep(market_train_df)
 print(market_train_df.shape )<define_variables>
xgb= XGBRegressor(base_score=0.5, booster='gbtree', colsample_bylevel=1,
colsample_bynode=1, colsample_bytree=0.5, gamma=0,
importance_type='gain', learning_rate=0.01, max_delta_step=0,
max_depth=3, min_child_weight=0, missing=None, n_estimators=4000,
n_jobs=1, nthread=None, objective='reg:squarederror', random_state=0,
reg_alpha=0.0001, reg_lambda=0.01, scale_pos_weight=1, seed=None,
silent=None, subsample=1, verbosity=1)
 xgmod=xgb.fit(x_train,y_train)
 xg_pred=xgmod.predict(x_test)
 print(f'Root Mean Square Error test = {str(math.sqrt(sklm.mean_squared_error(y_test, xg_pred)))}' )
up = market_train_df['returnsOpenNextMktres10'] >= 0
universe = market_train_df['universe'].values
d = market_train_df['time']
fcol = [c for c in market_train_df if c not in ['assetCode', 'assetCodes', 'assetCodesLen', 'assetName', 'audiences',
'firstCreated', 'headline', 'headlineTag', 'marketCommentary', 'provider',
'returnsOpenNextMktres10', 'sourceId', 'subjects', 'time', 'time_x', 'date' , 'universe','sourceTimestamp']]
X = market_train_df[fcol].values
up = up.values
r = market_train_df.returnsOpenNextMktres10.values
mins = np.min(X, axis=0)
 maxs = np.max(X, axis=0)
 rng = maxs - mins
X = 1 -(( maxs - X)/ rng)
 assert X.shape[0] == up.shape[0] == r.shape[0]
X_train, X_test, up_train, up_test, r_train, r_test,u_train,u_test,d_train,d_test = model_selection.train_test_split(X, up, r,universe,d, test_size=0.25, random_state=99)
 train_data = lgb.Dataset(X, label=up.astype(int))
test_data = lgb.Dataset(X_test, label=up_test.astype(int))<compute_train_metric>
vote_mod = VotingRegressor([('Ridge', ridge_mod),('Lasso', lasso_mod),('Elastic', elastic_cv), 
('XGBRegressor', xgb)])
 vote= vote_mod.fit(x_train, y_train.ravel())
 vote_pred=vote.predict(x_test)
 print(f'Root Mean Square Error test = {str(math.sqrt(sklm.mean_squared_error(y_test, vote_pred)))}' )
confidence_test =(gbm_1.predict(X_test)+ gbm_2.predict(X_test)) /2
confidence_test =(confidence_test-confidence_test.min())/(confidence_test.max() -confidence_test.min())
 confidence_test = confidence_test*2-1
print(max(confidence_test),min(confidence_test))
r_test = r_test.clip(-1,1)
 x_t_i = confidence_test * r_test * u_test
data = {'day' : d_test, 'x_t_i' : x_t_i}
df = pd.DataFrame(data)
 x_t = df.groupby('day' ).sum().values.flatten()
mean = np.mean(x_t)
 std = np.std(x_t)
 score_test = mean / std
print(score_test)
<set_options>
stregr = StackingRegressor(regressors=[elastic_cv,ridge_mod, lasso_mod, vote_mod],
meta_regressor=xgb, use_features_in_secondary=True 
)
 stack_mod=stregr.fit(x_train, y_train.ravel())
 stacking_pred=stack_mod.predict(x_test)
 print(f'Root Mean Square Error test = {str(math.sqrt(sklm.mean_squared_error(y_test, stacking_pred)))}' )
del X_train,X_test
gc.collect()<feature_engineering>
final_test=(0.3*vote_pred+0.5*stacking_pred+ 0.2*y_lasso_test)
 print(f'Root Mean Square Error test= {str(math.sqrt(sklm.mean_squared_error(y_test, final_test)))}')
days = env.get_prediction_days()
n_days = 0
prep_time = 0
prediction_time = 0
packaging_time = 0
total_market_obs_df = []
for(market_obs_df, news_obs_df, predictions_template_df)in days:
n_days +=1
if(n_days%50==0):
print(n_days,end=' ')
 t = time.time()
market_obs_df['time'] = market_obs_df['time'].dt.date
return_features = ['returnsClosePrevMktres10','returnsClosePrevRaw10','open','close']
total_market_obs_df.append(market_obs_df)
 if len(total_market_obs_df)==1:
history_df = total_market_obs_df[0]
else:
history_df = pd.concat(total_market_obs_df[-(np.max(n_lag)+1):])
 print(history_df)
 new_df = generate_lag_features(history_df,n_lag=[3,7,14])
 market_obs_df = pd.merge(market_obs_df,new_df,how='left',on=['time','assetCode'])
 market_obs_df = mis_impute(market_obs_df)
 market_obs_df = data_prep(market_obs_df)
 X_live = market_obs_df[fcol].values
X_live = 1 -(( maxs - X_live)/ rng)
 prep_time += time.time() - t
t = time.time()
lp =(gbm_1.predict(X_live)+ gbm_2.predict(X_live)) /2
prediction_time += time.time() -t
t = time.time()
confidence = lp
confidence =(confidence-confidence.min())/(confidence.max() -confidence.min())
 confidence = confidence * 2 - 1
preds = pd.DataFrame({'assetCode':market_obs_df['assetCode'],'confidence':confidence})
 predictions_template_df = predictions_template_df.merge(preds,how='left' ).drop('confidenceValue',axis=1 ).fillna(0 ).rename(columns={'confidence':'confidenceValue'})
 env.predict(predictions_template_df)
 packaging_time += time.time() - t
env.write_submission_file()
sub = pd.read_csv("submission.csv" )<import_modules>
vote_test = vote_mod.predict(X_test)
 final1=np.expm1(vote_test)
 stack_test = stregr.predict(X_test)
 final2=np.expm1(stack_test)
 lasso_test = lasso_mod.predict(X_test)
 final3=np.expm1(lasso_test)
<define_variables><EOS> 
final=(0.2*final1+0.6*final2+0.2*final3)
 final_submission = pd.DataFrame({
"Id": b["Id"],
"SalePrice": final
})
 final_submission.to_csv("final_submission.csv", index=False)
 final_submission.head()