Kaggle competition: Bag of Words Meets Bags of Popcorn
imdb_data['sentiment'] = imdb_data['label'].map({"neg":0, "pos":1} )<drop_column>
train = pd.read_csv('.. /input/labeledTrainData.tsv',header = 0, delimiter = '\t')
 test = pd.read_csv('.. /input/testData.tsv',header = 0, delimiter = '\t' )
imdb_data = imdb_data.drop(["label"],axis=1 )<correct_missing_values>
reviews = train['review']
sentiments = train['sentiment']
imdb_data = imdb_data.dropna()<create_dataframe>
reviewsT = test['review']
imdb_data = imdb_data[['sentiment','review']]<prepare_x_and_y>
word_vectorizer = TfidfVectorizer(
 sublinear_tf=True,
strip_accents='ascii',
analyzer='word',
stop_words='english',
ngram_range=(1, 1),
max_features=10000)
 word_vectorizer.fit(reviews + reviewsT )
X = raw_train_data_labeled['review']
y = raw_train_data_labeled['sentiment']<concatenate>
X_train, X_test, y_train, y_test = train_test_split(X, sentiments,test_size=0.25)
 print(X_train.shape, X_test.shape, y_train.shape, y_test.shape )
review_data = X.append(raw_test_data['review'] )<concatenate>
model=LogisticRegression(solver='lbfgs')
 model.fit(X_train,y_train )
review_data = review_data.append(imdb_data['review'] )<drop_column>
preds=model.predict(X_test)
 print(classification_report(preds,y_test))
ntrain = X.shape[0]
df = raw_train_data_labeled.append(raw_test_data, sort=False)
 df = df.drop(['sentiment'], axis=1)
 df_review = df['review']<string_transform>
preds=model.predict(Xt)
 test['sentiment'] = preds
review_lines = list()
lines = review_data.values.tolist()
for line in lines:

 tokens = word_tokenize(line)
 tokens = [w.lower() for w in tokens]
table = str.maketrans('','', string.punctuation)
 stripped = [w.translate(table)for w in tokens]
words = [w for w in stripped if w.isalpha() ]
stop_words = set(stopwords.words('english'))
words = [w for w in words if w not in stop_words]
review_lines.append(words )<set_options>
pd.read_csv('.. /input/sampleSubmission.csv' )
avg_length = sum(length)/len(review_lines)
 print(avg_length )<define_variables>
test = test.drop(['review'],axis = 1 )
<define_variables><EOS> 
test.to_csv('submissions.csv',index=False )