I am working on the file "KaggelChallenges/titanic/explore.ipynb".
The first task is:
This function introduces 4 new features:

FamilySize : the total number of relatives including the passenger (him/her)self.
Sigleton : a boolean variable that describes families of size = 1
SmallFamily : a boolean variable that describes families of 2 <= size <= 4
LargeFamily : a boolean variable that describes families of 5 < size
Can you write Python code for it?

# Family

# Code based on that here: http://ahmedbesbes.com/how-to-score-08134-in-titanic-kaggle-challenge.html
# introducing a new feature : the size of families (including the passenger)
train_df['FamilySize'] = train_df['Parch'] + train_df['SibSp'] + 1
# introducing other features based on the family size
train_df['Singleton'] = train_df['FamilySize'].map(lambda s : 1 if s == 1 else 0)
train_df['SmallFamily'] = train_df['FamilySize'].map(lambda s : 1 if 2<=s<=4 else 0)
train_df['LargeFamily'] = train_df['FamilySize'].map(lambda s : 1 if 5<=s else 0)

# Code based on that here: http://ahmedbesbes.com/how-to-score-08134-in-titanic-kaggle-challenge.html
# introducing a new feature : the size of families (including the passenger)
test_df['FamilySize'] = test_df['Parch'] + test_df['SibSp'] + 1
# introducing other features based on the family size
test_df['Singleton'] = test_df['FamilySize'].map(lambda s : 1 if s == 1 else 0)
test_df['SmallFamily'] = test_df['FamilySize'].map(lambda s : 1 if 2<=s<=4 else 0)
test_df['LargeFamily'] = test_df['FamilySize'].map(lambda s : 1 if 5<=s else 0)

if False:

    # Instead of having two columns Parch & SibSp,
    # we can have only one column represent if the passenger had any family member aboard or not,
    # Meaning, if having any family member(whether parent, brother, ...etc) will increase chances of Survival or not.
    train_df['Family'] =  train_df["Parch"] + train_df["SibSp"]
    train_df['Family'].loc[train_df['Family'] > 0] = 1
    train_df['Family'].loc[train_df['Family'] == 0] = 0

    test_df['Family'] =  test_df["Parch"] + test_df["SibSp"]
    test_df['Family'].loc[test_df['Family'] > 0] = 1
    test_df['Family'].loc[test_df['Family'] == 0] = 0

    # drop Parch & SibSp
    train_df = train_df.drop(['SibSp','Parch'], axis=1)
    test_df    = test_df.drop(['SibSp','Parch'], axis=1)

# plot
if False:
    fig, (axis1,axis2) = plt.subplots(1,2,sharex=True,figsize=(10,5))

    # sns.factorplot('Family',data=train_df,kind='count',ax=axis1)
   ...


I am working on the file "KaggelChallenges/titanic/explore.ipynb".
The first task is:
lets perform the same NaN replacement for the 'Age' with the test data as well
Can you write Python code for it?

## let's pull in the test data
test_df = pd.read_csv("test.csv",dtype={"Age":np.float64},)
#test_df.head()

#### Do the same for the test data
# column 'Age' has some NaN values
# A simple approximation of the distribution of ages is a gaussian, but this is not commonly accurate.
# lets make a vector of random ages centered on the mean, with a width of the std
lower, upper = test_df['Age'].min(), test_df['Age'].max()
mu, sigma = test_df["Age"].mean(), test_df["Age"].std()

# number of rows
n = test_df.shape[0]

print 'max: ',test_df['Age'].max()
print 'min: ',test_df['Age'].min()

# vector of random values using the truncated normal distribution.
X = stats.truncnorm((lower - mu) / sigma, (upper - mu) / sigma, loc=mu, scale=sigma)
rands = X.rvs(n)

# get the indexes of the elements in the original array that are NaN
idx = np.isfinite(test_df['Age'])

# use the indexes to replace the NON-NaNs in the random array with the good values from the original array
rands[idx.values] = test_df[idx]['Age'].values

## At this point rands is now the cleaned column of data we wanted, so push it in to the original df
test_df['Age'] = rands

#test_df['Age'].hist(bins=70)

## Let's make a couple nice plots of survival vs age
# peaks for survived/not survived passengers by their age
if False:
    facet = sns.FacetGrid(train_df, hue="Survived",aspect=4)
    #facet.map(sns.kdeplot,'Age',shade= True) # This keeps crashing the kernal, but I don't know why!!!!!!!!!!
    facet.set(xlim=(0, train_df['Age'].astype(int).max()))
    facet.add_legend()

# average survived passengers by age
if False:
    fig, axis1 = plt.subplots(1,1,figsize=(18,4))
    average_age = train_df[["Age", "Survived"]].groupby(['Age'],as_index=False).mean()
    sns.barplot(x='Age', y='Survived', data=average_age)
    print 'max: ',train_df['Age'].astype(int).max()
    print 'min: ',train_df['Age'].astype(int).min()

# Cabin
if False:
    # It has a lot of NaN values, so it won't cause a remarkable impact on prediction
   ...


I am working on the file "KaggelChallenges/titanic/explore.ipynb".
The first task is:
match up dataframe columns, by removing extras not in the test set.
Can you write Python code for it?

## Remove extra columns in training DF that are not in test DF
train_cs = list(train_df.columns)
train_cs.remove('Survived')
test_cs = list(test_df.columns)
for c in train_cs:
    if c not in test_cs:
        print repr(c)+' not in test columns, so removing it from training df'
        train_df.drop([c], axis=1,inplace=True)
for c in test_cs:
    if c not in train_cs:
        print repr(c)+' not in training columns, so removing it from test df'
        test_df.drop([c], axis=1,inplace=True)

if False:
    print '\nFor train_df:'
    for column in train_df:
        print "# Nans in column '"+column+"' are: "+str(train_df[column].isnull().sum())
        print 'min: ',train_df[column].min()
        print 'max: ',train_df[column].max()

    print '\nFor test_df:'
    for column in test_df:
        print "# Nans in column '"+column+"' are: "+str(test_df[column].isnull().sum())
        print 'min: ',test_df[column].min()
        print 'max: ',test_df[column].max()

# define training and testing sets
X_train = train_df.drop("Survived",axis=1)
Y_train = train_df["Survived"]
X_test  = test_df.copy()

X_train.head()

X_test.head()


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
=> destination latent features seem not to be for any good use?!
Can you write Python code for it?

features = [
    'site_name', 'posa_continent', 'user_location_country',
    'user_location_region', 'user_location_city',
    'is_mobile', 'is_package',
    'channel', 'srch_adults_cnt', 'srch_destination_id',
    'srch_destination_type_id', 'is_booking', 'cnt',
    'hotel_continent', 'hotel_country', 'hotel_market',
    'month', 'year', 'is_family',
]

def fit_features(features, train, test):
    # Data manipulation - split categorical features
    train_data = pd.DataFrame()
    test_data = pd.DataFrame()
    for feature in features:
        if train[feature].dtype == np.dtype('bool'):
            train_data[feature] = train[feature]
            test_data[feature] = test[feature]
        else:
            for elem in train[feature].unique():
                train_data['{}_{}'.format(feature, elem)] = train[feature] == elem
                test_data['{}_{}'.format(feature, elem)] = test[feature] == elem

    # Fitting
    clf = sklearn.linear_model.SGDClassifier(loss='log', n_jobs=4)
    clf.fit(train_data, train['hotel_cluster'])

    # Cross-validate the fit
    result = clf.predict_proba(test_data)
    preds = [heapq.nlargest(5, clf.classes_, row.take) for row in result]
    target = [[l] for l in test['hotel_cluster']]

    return metrics.mapk(target, preds, k=5)

cv_results = {}
for feature in features:
    cv_results[feature] = fit_features([feature], cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results[feature]))

sorted(cv_results.items(), key=operator.itemgetter(1), reverse=True)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
Create new test and training sets, using bookings from 2013 as training data and 2014 as test data.
Can you write Python code for it?

cv_train = sel_train[sel_train.year == 2013]
cv_test = sel_train[sel_train.year == 2014]


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
Data import
Defining a list of available data columns:
Can you write Python code for it?

traincols = ['date_time', 'site_name', 'posa_continent', 'user_location_country',
             'user_location_region', 'user_location_city', 'orig_destination_distance',
             'user_id', 'is_mobile', 'is_package', 'channel', 'srch_ci', 'srch_co',
             'srch_adults_cnt', 'srch_children_cnt', 'srch_rm_cnt', 'srch_destination_id',
             'srch_destination_type_id', 'is_booking', 'cnt', 'hotel_continent',
             'hotel_country', 'hotel_market', 'hotel_cluster']
testcols = ['id', 'date_time', 'site_name', 'posa_continent', 'user_location_country',
            'user_location_region', 'user_location_city', 'orig_destination_distance',
            'user_id', 'is_mobile', 'is_package', 'channel', 'srch_ci', 'srch_co',
            'srch_adults_cnt', 'srch_children_cnt', 'srch_rm_cnt', 'srch_destination_id',
            'srch_destination_type_id', 'hotel_continent', 'hotel_country', 'hotel_market']


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
Getting a list of all user_ids in the sample.
Can you write Python code for it?

train_ids = set(train.user_id.unique())
len(train_ids)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
I would say that not that bad at all (comparing the random forrest classifier in the Dataquest tutorial).
Using destination latent features form destination description data file.
Can you write Python code for it?

dest = pd.read_csv(
    'data/destinations.csv.gz',
    index_col = 'srch_destination_id',
)

pca = sklearn.decomposition.PCA(n_components=10)
dest_small = pca.fit_transform(dest[['d{}'.format(i) for i in range(1,150)]])
dest_small = pd.DataFrame(dest_small, index=dest.index)

cv_train_data = pd.DataFrame({key: cv_train[key] for key in ['srch_destination_id']})
cv_train_data = cv_train_data.join(dest_small, on=['srch_destination_id'], how='left')
cv_train_data = cv_train_data.fillna(dest_small.mean())

cv_test_data = pd.DataFrame({key: cv_test[key] for key in ['srch_destination_id']})
cv_test_data = cv_test_data.join(dest_small, on='srch_destination_id', how='left', rsuffix='dest')
cv_test_data = cv_test_data.fillna(dest_small.mean())

clf = sklearn.linear_model.SGDClassifier(loss='log', n_jobs=4)
clf.fit(cv_train_data, cv_train['hotel_cluster'])

result = clf.predict_proba(cv_test_data)

preds = [heapq.nlargest(5, clf.classes_, row.take) for row in result]

metrics.mapk(cv_target, preds, k=5)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
Make dummy variables from categorical features. Pandas has get_dummies(), but currently this returns only float64-s, that thends to be rather memory hungry and slow. See #8725.
Can you write Python code for it?

cv_train_data = pd.DataFrame()
for elem in cv_train['srch_destination_id'].unique():
    cv_train_data[str(elem)] = cv_train['srch_destination_id'] == elem

cv_test_data = pd.DataFrame()
for elem in cv_train_data.columns:
    cv_test_data[elem] = cv_test['srch_destination_id'] == int(elem)

# cv_train_data['is_booking'] = cv_train['is_booking']
# cv_test_data['is_booking'] = np.ones(len(cv_test_data), dtype=bool)

clf.fit(cv_train_data, cv_train['hotel_cluster'])

result = clf.predict_proba(cv_test_data)

preds = [heapq.nlargest(5, clf.classes_, row.take) for row in result]

metrics.mapk(cv_target, preds, k=5)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
Model 0: Most common clusters
Public solutions to the compedition (Dataquest tutorial by Vik Paruchuri and Leakage solution by ZFTurbo) use most common clusters in following groups:
* srch_destination_id
* user_location_city, orig_destination_distance (data leak)
* srch_destination_id, hotel_country, hotel_market (for year 2014)
* srch_destination_id
* hotel_country
Finding the most common overall clusters
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
Pick a subset of users for testing and validation
Can you write Python code for it?

sel_user_ids = sorted(random.sample(train_ids, 12000))
sel_train = train[train.user_id.isin(sel_user_ids)]


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
The best single predictor of a hotel cluster seems to be hotel_market.
Can you write Python code for it?

features2 = [['hotel_market'] + [f] for f in features if f not in ['hotel_market']]

cv_results2 = {}
for feature in features2:
    cv_results2[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results2[tuple(feature)]))

sorted(cv_results2.items(), key=operator.itemgetter(1), reverse=True)[:3]

features3 = [['hotel_market', 'srch_destination_id'] + [f] for f in features if f not in ['hotel_market', 'srch_destination_id']]

cv_results3 = {}
for feature in features3:
    cv_results3[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results3[tuple(feature)]))

sorted(cv_results3.items(), key=operator.itemgetter(1), reverse=True)[:3]

features4 = [['hotel_market', 'srch_destination_id', 'hotel_country'] + [f] for f in features if f not in ['hotel_market', 'srch_destination_id', 'hotel_country']]

cv_results4 = {}
for feature in features4:
    cv_results4[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results4[tuple(feature)]))

sorted(cv_results4.items(), key=operator.itemgetter(1), reverse=True)[:3]

sel_features = ['hotel_market', 'srch_destination_id', 'hotel_country', 'is_package']
features5 = [sel_features + [f] for f in features if f not in sel_features]

cv_results5 = {}
for feature in features5:
    cv_results5[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results5[tuple(feature)]))

sorted(cv_results5.items(), key=operator.itemgetter(1), reverse=True)[:3]

sel_features = ['hotel_market', 'srch_destination_id', 'hotel_country', 'is_package', 'is_booking']
features6 = [sel_features + [f] for f in features if f not in sel_features]

cv_results6 = {}
for feature in features6:
    cv_results6[tuple(feature)] = fit_features(feature, cv_train, cv_test)
    print('{}: {}'.format(feature, cv_results6[tuple(feature)]))

sorted(cv_results6.items(), key=operator.itemgetter(1), reverse=True)[:3]

sel_features =...


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations - Logistic Regression.ipynb".
The first task is:
This dictionary has a key of srch_destination_id, hotel_country, hotel_market and each value is another dictionary, with hotel clusters as keys and scores as values.
Finding the top 5 for each destination.
Can you write Python code for it?

cluster_dict = {}
for n in top_clusters:
    tc = top_clusters[n]
    top = [l[0] for l in sorted(tc.items(), key=operator.itemgetter(1), reverse=True)[:5]]
    cluster_dict[n] = top


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
According to the specification the data fields are following:
train.csv
| Column name               | Description                                   | Data type | Equiv. type | Notes |
|---------------------------|-----------------------------------------------|-----------|-------------|-------|
| date_time                 | Timestamp                                     | string    |             | [1]   |
| site_name                 | ID of the Expedia point of sale               | int       | np.int32    |       |
| posa_continent            | ID of continent associated with site_name     | int       | np.int32    |       |
| user_location_country     | The ID of the country the customer is located | int       | np.int32    |       |
| user_location_region      | The ID of the region the customer is located  | int       | np.int32    |       |
| user_location_city        | The ID of the city the customer is located    | int       | np.int32    |       |
| orig_destination_distance | Physical distance between a hotel and a customer at the time of search. A null means the distance could not be calculated | double | np.float64 | |
| user_id                   | ID of user                                    | int       | np.int32    |       |
| is_mobile                 | 1 when a user connected from a mobile device, 0 otherwise | tinyint | np.uint8 | [2] |
| is_package                | 1 if the click/booking was generated as a part of a package (i.e. combined with a flight), 0 otherwise | int | np.uint8 | [2] |
| channel                   | ID of a marketing channel                     | int       | np.int32    |       |
| srch_ci                   | Checkin date                                  | string    |             | [1]   |
| srch_co                   | Checkout date                                 | string    |             | [1]   |
| srch_adults_cnt           | The number of adults specified in the hotel room | int    | np.int32    |       |
| srch_children_cnt         | The number of (extra occupancy)...
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Calculating Mean Average Precision with mapk from ml_metrics.
Can you write Python code for it?

target = [[l] for l in t2['hotel_cluster']]
metrics.mapk(target, predictions, k=5)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Classifier accuracy seems rather low here as well.
Random forrest with binary classifier
Random forests should work better if only a single hotel cluster is predicted at times.
Can you write Python code for it?

all_probs = []
unique_clusters = df['hotel_cluster'].unique()
for cluster in unique_clusters:
    df['target'] = 0
    df.loc[df['hotel_cluster'] == cluster, 'target'] = 1
    predictors = [c for c in df.columns if c not in ['hotel_cluster', 'target']]
    probs = []
    cv = sklearn.cross_validation.KFold(len(df), n_folds=5)
    clf = sklearn.ensemble.RandomForestClassifier(
        n_estimators=10,
        min_weight_fraction_leaf=0.1,
    )
    for i, (tr, te) in enumerate(cv):
        clf.fit(df[predictors].iloc[tr], df['target'].iloc[tr])
        preds = clf.predict_proba(df[predictors].iloc[te])
        probs.append(p[1] for p in preds)
    full_probs = itertools.chain.from_iterable(probs)
    all_probs.append(list(full_probs))

prediction_frame = pd.DataFrame(all_probs).T
prediction_frame.columns = unique_clusters
def find_top5(row):
    return list(row.nlargest(5).index)

preds = []
for index, row in prediction_frame.iterrows():
    preds.append(find_top5(row))

metrics.mapk([[l] for l in t2['hotel_cluster']], preds, k=5)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Combining predictions
Combine exact_matches, preds, and most_common_clusters.
Can you write Python code for it?

def f5(seq, idfun=None):
    """Uniquify a list by Peter Bengtsson
    https://www.peterbe.com/plog/uniqifiers-benchmark
    """
    if idfun is None:
        def idfun(x):
            return x

    seen = {}
    result = []
    for item in seq:
        marker = idfun(item)
        if marker in seen:
            continue
        seen[marker] = 1
        result.append(item)
    return result

full_preds = [
    f5(exact_matches[p] + preds[p] + most_common_clusters)[:5]
    for p
    in range(len(preds))
]
metrics.mapk([[l] for l in t2["hotel_cluster"]], full_preds, k=5)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Create new test and training sets
Can you write Python code for it?

t1 = sel_train[((sel_train.year == 2013) | ((sel_train.year == 2014) & (sel_train.month < 8)))]
t2 = sel_train[((sel_train.year == 2014) & (sel_train.month >= 8))]


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Data leak
Utilizing the data leak, that allows matching users in the training set from the testing set using a set of columns inculid user_location_country, and user_location_region.
Can you write Python code for it?

match_cols = [
    'user_location_country',
    'user_location_region',
    'user_location_city',
    'hotel_market',
    'orig_destination_distance',
]

groups = t1.groupby(match_cols)

def generate_exact_matches(row, match_cols):
    index = tuple(row[t] for t in match_cols)
    try:
        group = groups.get_group(index)
    except KeyError:
        return []
    clus = list(set(group.hotel_cluster))
    return clus

exact_matches = []
for i in range(t2.shape[0]):
    exact_matches.append(generate_exact_matches(t2.iloc[i], match_cols))


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Finding columns in testcols but not in traincols and vice versa:
Can you write Python code for it?

[col for col in testcols if col not in traincols]

[col for col in traincols if col not in testcols]


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Finding missing values in test data:
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Generating features

New date features based on date_time, srch_ci, and srch_co.
Remove non-numeric columns like date_time.
Add in features from dest_small.
Replace any missing values with -1. (Initially planned to use unsigned integers for most of the variables, using -1 as fill value would not work then. May test with replacing na's using the most common values.
Can you write Python code for it?

def calc_fast_features(df):
    # Assumes that the data frame date_time, srch_ci and srch_co are already converted to datetime.
    props = {}
    for prop in ['month', 'day', 'hour', 'minute', 'dayofweek', 'quarter']:
        props[prop] = getattr(df['date_time'].dt, prop)

    carryover = [p for p in df.columns if p not in ['date_time', 'srch_ci', 'srch_co']]
    for prop in carryover:
        props[prop] = df[prop]

    date_props = ['month', 'day', 'dayofweek', 'quarter']
    for prop in date_props:
        props['ci_{}'.format(prop)] = getattr(df['srch_ci'].dt, prop)
        props['co_{}'.format(prop)] = getattr(df['srch_co'].dt, prop)
    props['stay_span'] = (df['srch_co'] - df['srch_ci']).astype('timedelta64[h]')

    ret = pd.DataFrame(props)

    ret = ret.join(dest_small, on='srch_destination_id', how='left', rsuffix='dest')
    ret = ret.drop('srch_destination_iddest', axis=1)
    return ret

df = calc_fast_features(t1)


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Generating features from destinations
Can you write Python code for it?

dest = pd.read_csv('data/destinations.csv.gz')
dest.info()

dest.head()

pca = sklearn.decomposition.PCA(n_components=3)
dest_small = pca.fit_transform(dest[['d{}'.format(i) for i in range(1,150)]])
dest_small = pd.DataFrame(dest_small)
dest_small['srch_destination_id'] = dest['srch_destination_id']

dest_small.head()


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
I don't know exactly what data colmuns I will be using eventually but I will define the data types for them here anyway just in case. Looking at the data most of the columns are actually non-negative integers so I can use unsigned integers for the most cases. Usage between uint8, uint32 and others was determined by the min and max values in the test dataset.
Can you write Python code for it?

def read_csv( filename, cols, nrows=None ):
    datecols = ['date_time', 'srch_ci', 'srch_co']
    dateparser = lambda x: pd.to_datetime(x, format='%Y-%m-%d %H:%M:%S', errors='coerce')

    dtypes = {
        'id': np.uint32,
        'site_name': np.uint8,
        'posa_continent': np.uint8,
        'user_location_country': np.uint16,
        'user_location_region': np.uint16,
        'user_location_city': np.uint16,
        'orig_destination_distance': np.float32,
        'user_id': np.uint32,
        'is_mobile': bool,
        'is_package': bool,
        'channel': np.uint8,
        'srch_adults_cnt': np.uint8,
        'srch_children_cnt': np.uint8,
        'srch_rm_cnt': np.uint8,
        'srch_destination_id': np.uint32,
        'srch_destination_type_id': np.uint8,
        'is_booking': bool,
        'cnt': np.uint64,
        'hotel_continent': np.uint8,
        'hotel_country': np.uint16,
        'hotel_market': np.uint16,
        'hotel_cluster': np.uint8,
    }

    df = pd.read_csv(
        filename,
        nrows=nrows,
        usecols=cols,
        dtype=dtypes, # dtype can also specify datatypes for columns that do not excist in the particular datafile
        parse_dates=[col for col in datecols if col in cols], # columns here must be also in usecols
        date_parser=dateparser,
    )
    return df

train = read_csv('data/train.csv.gz', nrows=None, cols=traincols)
train.info()


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Model 0: Most common clusters
Starting looking at the most common clusters and their properties.
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Pick 10000 users for smaller scale testing
Can you write Python code for it?

sel_user_ids = sorted(random.sample(train_ids, 10000))
sel_train = train[train.user_id.isin(sel_user_ids)]


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Predicting most_common_clusters for every single row in selected test data.
Can you write Python code for it?

predictions = [most_common_clusters for i in range(len(t2))]


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Remove click events from t2 as in original test data.
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Submission file
I'll clean up the code here and make a separate script that uses full dataset for generating a submission file.
Here I'll just test the file making part.
Can you write Python code for it?

write_p = [" ".join([str(l) for l in p]) for p in full_preds]
write_frame = ["{},{}".format(t2.index[i], write_p[i]) for i in range(len(full_preds))]
write_frame = ["id,hotel_clusters"] + write_frame
with open('predictions.csv', 'w+') as f:
    f.write('\n'.join(write_frame))


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
That's not too great.
Model 1
Can you write Python code for it?

#train.corr()['hotel_cluster']
# Calculating the correlations takes a while. No linear correlations were anyhow found in tutorial.


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
The variance ratio that is retained using principal components analysis with 3 principal components:
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
There are also some dates where the check in date is later than check out date:
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
This dictionary has a key of srch_destination_id and each value is another dictionary, with hotel clusters as keys and scores as values.
Finding the top 5 for each destination.
Can you write Python code for it?


I am working on the file "Kaggle-Expedia/Expedia Hotel Recommendations.ipynb".
The first task is:
Using just the most popular clusters gives better scores. The approach here isn't particularly promising.
One thing to note is that the input is full of categorical features. Therefore, to properly apply machine learning converting those values to separate binary features may be more appropriate approach.
Model 2
Finding top hotel clusters for each destination.
Can you write Python code for it?

def make_key(items):
    return '_'.join([str(i) for i in items])

match_cols = ['srch_destination_id']
cluster_cols = match_cols + ['hotel_cluster']
groups = t1.groupby(cluster_cols)

top_clusters = {}
for name, group in groups:
    bookings = group['is_booking'].sum()
    clicks = len(group) - bookings

    score = bookings + .15*clicks

    clus_name = make_key(name[:len(match_cols)])
    if clus_name not in top_clusters:
        top_clusters[clus_name] = {}
    top_clusters[clus_name][name[-1]] = score


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
In the above plot, we see that there are some professions involving only male employees - such as pales, production, transportation, construction etc. Similarly, there are occuapations that involve only female employees - such as healthcare professionals, education etc. This is just a sample of 20 occupations. We can analyze more, as seen below.
Can you write Python code for it?

sorted_male_female_count_n_barplot = sb.factorplot(x='Occupations',
           y='value', data=sorted_male_female_count_n.head(n=10), hue='variable',
                                         palette="bright",
                                     size=12, aspect=5,kind='bar')
sorted_male_female_count_n_barplot.set(xlabel = "Occupations", ylabel = "Workers - All, Male, Female",
                             title = "Top male, female employee counts and the occupations")
sb.set(font_scale=4)


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
Now, we will slice the dataset to understand more about the data
Can you write Python code for it?

#selecting just the occupations and worker count into a new dataframe
job_type_by_worker_count = income[['Occupations','All_workers']]

job_type_by_worker_count.sort_values(by='All_workers', ascending =False)


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
The conversion happened for All_workers, Male_Workers (in 1000s), Female_Workers (in 1000s), but not for the other columns. We will see why.
Can you write Python code for it?


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
The first row shows the total number of workers. We will drop the row since it is not useful for our analysis and will skew the overall pattern.
Can you write Python code for it?

import matplotlib
import matplotlib.pyplot as plt
import seaborn as sb
%matplotlib inline

top_worker_count_barplot = sb.factorplot(x='Occupations',
           y='All_workers', data=top10_worker_count.head(n=5), hue='All_workers',
                                     size=7, aspect=2.3,kind='bar')
top_worker_count_barplot.set(xlabel = "Occupations", ylabel = "Workers - All, Male, Female",
                             title = "Top worker counts and the occupations")
sb.set(font_scale=2.3)


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
The following looks more readable
Can you write Python code for it?


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
The sorting did not happen by the All_workers column. Let's see why that is the case.
Can you write Python code for it?


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
This looks better, without the Na string values. Now, we will convert all the columns, except Occupations to int.
Occupations is a string column and we can leave it as an object type.
Can you write Python code for it?

income_clean = income.apply(pd.to_numeric, errors = 'ignore')

income_clean.dtypes


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
To manipulate and perform operations on numerical columns, we will need to handle null/empty/non-numerical values in the numerical columns. In our dataset, this means everything except the Occupations column.
Can you write Python code for it?


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
We assign more meaningful and self explanatory column headers. The names list can be assigned to the names parameter when reading in the csv file.
Can you write Python code for it?

names = ['Occupations','All_workers', 'Weekly_Income_Overall',
                             'Male_Workers (in 1000s)', 'Median_weekly_income (Male)',
                             'Female_Workers (in 1000s)', 'Median_weekly_income (Female)']

#mac
income = pd.read_csv('/Users/Harish/Documents/HK_Work/Python/Python-for-Data-Analysis/Kaggle-US-Incomes/inc_occ_gender.csv'
                   ,names=names, header=None)

#win
#income = pd.read_csv(r'C:\Users\\Documents\Personal\Python-for-Data-Analysis\Kaggle-US-Incomes\inc_occ_gender.csv'
                # ,names=names, header=None)


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
We can observe a similar trend in the above plot as well, where transportation, construction, production, maintenance and groundskeeping involve male employees and education and healthcare involves female employees. We can also see that business, office, sale etc. have both female and male employees.
Now, let us sort the Occupations by weekly median income, irrespective of gender of employees
Can you write Python code for it?

top10_income_count = income_clean.sort_values(by='Weekly_Income_Overall', axis=0,
                                                           ascending=False)[['Occupations',
                                                                                'Weekly_Income_Overall']]

low10_income_count = income_clean.sort_values(by='Weekly_Income_Overall', axis=0,
                                                           ascending=True)[['Occupations',
                                                                                'Weekly_Income_Overall']]

top10_income_count_barplot = sb.factorplot(x='Occupations',
           y='Weekly_Income_Overall', data=top10_income_count.head(n=5), hue='Occupations',
                                         palette="bright",
                                     size=12, aspect=5, kind='bar')
top10_income_count_barplot.set(xlabel = "Occupations", ylabel = "Weekly Income",
                             title = "Top Weekly Incomes and the respective occupations")
sb.set(font_scale=2)
#size=12, aspect=5,

low10_income_count = low10_income_count[(low10_income_count != 0).all(1)]

# lowest10_income_count_barplot = sb.factorplot(x='Weekly_Income_Overall',
#             y='Occupations', data=low10_income_count.head(n=10), hue='Occupations',palette="bright",
#                                              size=12, aspect=5, kind='bar')
# lowest10_income_count_barplot.set(xlabel='Occupations', ylabel = 'Weekly Income',
#                                  title = 'Lowest Weekly Incomes and the respective occupations')


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
We have a plot showing top employee count for 5 occupations. We will append gender wise numbers on the same plot. For this purpose, we use pandas' melt function. https://pandas.pydata.org/pandas-docs/stable/generated/pandas.melt.html
Can you write Python code for it?

top10_worker_count.head()

top10_worker_count = pd.melt(top10_worker_count, id_vars=["Occupations"],
                 value_vars=['All_workers','Male_Workers (in 1000s)','Female_Workers (in 1000s)'])

top10_mf_count = income_clean.sort_values(by='All_workers', axis=0,
                                                           ascending=False)[['Occupations',
                                                                                'Male_Workers (in 1000s)',
                                                                                'Female_Workers (in 1000s)']]

top10_mf_count.drop(1, axis=0, inplace=True)

top10_male_female = pd.melt(top10_mf_count, id_vars=["Occupations"],
                 value_vars=['Male_Workers (in 1000s)','Female_Workers (in 1000s)'])

top10_worker_count.replace('NaN',0,inplace=True)
top10_male_female.replace('NaN',0,inplace=True)

sorted_worker_count_n = top10_worker_count.sort_values(by='value', axis=0,
                                                           ascending=False)[['Occupations','variable','value']]

sorted_male_female_count_n = top10_male_female.sort_values(by='value', axis=0,
                                                           ascending=False)[['Occupations','variable','value']]

sorted_worker_count_n.head(n=5)

sorted_male_female_count_n.head(n=5)

sorted_worker_count_n_barplot = sb.factorplot(x='Occupations',
           y='value', data=sorted_worker_count_n.head(n=10), hue='variable',
                                         palette="bright",
                                     size=10, aspect=5,kind='bar')
sorted_worker_count_n_barplot.set(xlabel = "Occupations", ylabel = "Workers - All, Male, Female",
                             title = "Top worker counts and the occupations")
sb.set(font_scale=4.0)


I am working on the file "Kaggle-US-Incomes/US Income Analysis Notebook.ipynb".
The first task is:
We will now slice the data and analyze the count of workers in each occupation.
Can you write Python code for it?

top10_worker_count = income_clean.sort_values(by='All_workers', axis=0,
                                                           ascending=False)[['Occupations','All_workers',
                                                                                'Male_Workers (in 1000s)',
                                                                                'Female_Workers (in 1000s)']]

top10_worker_count.drop(1, axis=0, inplace=True)


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Convert "PubDate" into two columns: Weekday and Hour:
Can you write Python code for it?

joined['PubDate'] = pd.to_datetime(joined['PubDate'])
joined['Weekday'] = joined['PubDate'].dt.weekday
joined['Hour'] = joined['PubDate'].dt.hour

print("At the moment, we have %d entries with NewsDesk=Nan." % len(joined.loc[joined['NewsDesk'].isnull()]))


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Create additional features:
* "QorE": question or exclamation mark in the headline
* "Q&A": "Q. and A." phrase in the headline (I don't think it was valuable, but stayed here from my previous attemps)
Can you write Python code for it?

joined['QorE'] = joined['Headline'].str.contains(r'\!|\?').astype(int)
joined['Q&A'] = joined['Headline'].str.contains(r'Q\. and A\.').astype(int)


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Define a function for cross-validating and plotting ensemble of my two models:
Can you write Python code for it?

from sklearn import cross_validation
from sklearn.metrics import roc_auc_score

def calculate_ensemble_score(model1, model2, Xcols, ycol, dataset, cv=10):
    '''Calculates the score for various weights of two models in an ensemble'''

    num_points = 21
    score_arr = np.zeros((cv, num_points))

    kf = cross_validation.KFold(len(dataset), cv, shuffle=True)

    i = 0
    for xtrain, xtest in kf:
        train, test = dataset.ix[xtrain], dataset.ix[xtest]

        model1.fit(train[Xcols], train[ycol])
        probs1 = model1.predict_proba(test[Xcols])[:,1]

        model2.fit(train[Xcols], train[ycol])
        probs2 = model2.predict_proba(test[Xcols])[:,1]

        for wg in range(num_points):
            probs = wg/(num_points-1)*probs1 + (1-wg/(num_points-1))*probs2
            score_arr[i][wg] = roc_auc_score(test[ycol], probs)

        i+=1

    return np.mean(score_arr, axis=0)

def plot_ensemble_score(scores):
    import seaborn as sbs

    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.axhline(y=scores[0], linestyle='--', color='red')
    ax.axhline(y=scores[-1], linestyle='--', color='green')
    ax.text(0.03, scores[0]+0.00001, "Pure GBM", verticalalignment='bottom', horizontalalignment='left', color='red', size='larger')
    ax.text(0.98, scores[-1]+0.00001, "Pure RF", verticalalignment='bottom', horizontalalignment='right', color='green', size='larger')

    ax.plot(np.linspace(0,1,len(scores)), scores)
    ax.set_xlabel("RF model weight")
    ax.set_ylabel("AUC")
    ax.set_title("Choosing the weights for two models in an ensemble (10-fold cross-validation)")

    return fig

means = calculate_ensemble_score(forest, est, Xcols, 'Popular', train)

myplot = plot_ensemble_score(means)
myplot.savefig("AUC.png", dpi=300)

test['Popular'] = (0.6*probsGBC+0.4*probsRF)
test['UniqueID'] = test['UniqueID'].astype(int)
test.to_csv('preds.csv', columns=['UniqueID', 'Popular'], header=['UniqueID', 'Probability1'], index=False)


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Filling even more gaps with some clustering. I created a TFI-DF matrix and did Ward clustering on words. Four of six clusters I thought were meaningful and fitted well into existing NewsDesk/S(ubs)ectionName.
Can you write Python code for it?

from sklearn.cluster import AgglomerativeClustering
from sklearn.feature_extraction.text import TfidfVectorizer

nans = joined.loc[joined['NewsDesk'].isnull()]
words = list(nans.apply(lambda x:'%s' % (x['Abstract']),axis=1))
tfv = TfidfVectorizer(min_df=0.005,  max_features=None,
        strip_accents='unicode', analyzer='word',token_pattern=r'\w{1,}',
        ngram_range=(1, 2), use_idf=1,smooth_idf=1,sublinear_tf=1,
        stop_words = 'english')
X_tr = tfv.fit_transform(words)

ward = AgglomerativeClustering(n_clusters=6,
        linkage='ward').fit(X_tr.toarray())

joined.loc[joined['NewsDesk'].isnull(), 'cluster'] = ward.labels_
cluster_to = {}
cluster_to['NewsDesk'] = {4: 'Metro', 3: 'National', 2: 'Foreign', 1: 'National'}
cluster_to['SectionName'] = {4: 'N.Y. / Region', 3: 'U.S.', 2: 'Not_Asia', 1: 'U.S.'}
cluster_to['SubsectionName'] = {4: 'NYT', 3: 'Politics', 2: 'Not_Asia', 1: 'Politics'}

for key in cluster_to:
    for key2 in cluster_to[key]:
        joined.loc[(joined['cluster'] == key2) & (nans['NewsDesk'].isnull()), key] = cluster_to[key][key2]


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Filling the gaps in NewsDesk, SectionName and SubsectionName.
Can you write Python code for it?

section_to_newsdesk = {'Business Day': 'Business', 'Crosswords/Games': 'Business', 'Technology': 'Business',
     'Arts': 'Culture',
     'World': 'Foreign',
     'Magazine': 'Magazine',
     'N.Y. / Region': 'Metro',
     'Opinion': 'OpEd',
     'Travel': 'Travel',
     'Multimedia': 'Multimedia',
     'Open': 'Open'}

section_to_subsection = {'Crosswords/Games': 'Crosswords/Games',
                        'Technology': 'Technology'}

newsdesk_to_section = {'TStyle': 'TStyle',
                      'Culture': 'Arts',
                      'OpEd': 'Opinion',
                      'History': 'History'}

newsdesk_to_subsection = {'TStyle': 'TStyle',
                         'Culture': 'Arts',
                         'Daily Rubric': 'Rubric',
                         'Magazine': 'Magazine',
                         'Metro': 'Metro',
                         'Multimedia': 'Multimedia',
                         'OpEd': 'OpEd',
                         'Science': 'Science',
                         'Sports': 'Sports',
                         'Styles': 'Styles',
                         'Travel': 'Travel',
                         'History': 'History'}

for sec in set(joined['SectionName']):
    try: section_to_newsdesk[sec]
    except KeyError:
        pass
    else:
        joined['NewsDesk'].fillna(joined.loc[(joined['SectionName'] == sec)]['NewsDesk'].fillna(section_to_newsdesk[sec]), inplace=True)

    try: section_to_subsection[sec]
    except KeyError:
        pass
    else:
        joined['SubsectionName'].fillna(joined.loc[(joined['SectionName'] == sec)]['SubsectionName'].fillna(section_to_subsection[sec]), inplace=True)

for nd in set(joined['NewsDesk']):
    try: newsdesk_to_section[nd]
    except KeyError:
        pass
    else:
        joined['SectionName'].fillna(joined.loc[(joined['NewsDesk'] == nd)]['SectionName'].fillna(newsdesk_to_section[nd]), inplace=True)

    try: newsdesk_to_subsection[nd]
    except KeyError:
        pass
    else:
       ...


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Finally, use a few (6) obvious keywords to categorise the data even more. After this, we are left with 950 entries where NewsDesk, SectionName and SubsectionName are NaN, but I didn't have an idea how to deal with them.
Can you write Python code for it?

joined.drop('cluster', axis=1, inplace=True)

keywords = {}
keywords['clinton|white house|obama'] = {'NewsDesk': 'National', 'SectionName': 'U.S.', 'SubsectionName': 'Politics'}
keywords['isis|iraq'] = {'NewsDesk': 'Foreign', 'SectionName': 'Not_Asia', 'SubsectionName': 'Not_Asia'}
keywords['york'] = {'NewsDesk': 'Metro', 'SectionName': 'N.Y. / Region', 'SubsectionName': 'N.Y. / Region'}

for key in keywords:
    indices = (joined['NewsDesk'].isnull()) & (joined['Abstract'].str.contains(key, case=False))
    for sec in keywords[key]:
        joined.loc[indices, sec] = keywords[key][sec]

print("Now we have %d entries with NewsDesk=Nan." % len(joined.loc[joined['NewsDesk'].isnull()]))


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
Join the data for preprocessing
Can you write Python code for it?

print('Max train ID: %d. Max test ID: %d' % (np.max(NYT_train_raw['UniqueID']), np.max(NYT_test_raw['UniqueID'])))
joined = NYT_train_raw.merge(NYT_test_raw, how = 'outer')


I am working on the file "Kaggle/Kaggle - final model, RF+GBM.ipynb".
The first task is:
You can see what these clusters look like by typing:
Can you write Python code for it?


I am working on the file "Kaggle/Mercedes-Benz Greener Manufacturing/neural_network.ipynb".
The first task is:
Adjust for multicollinearity

Idea: treat the features as sequences but using the raw data 0/1 can cause NaN easily / probably due to multicollinearity
Therefore either take VIF or PCA to adjust for it

vif = pd.DataFrame()
vif["VIF Factor"] = [variance_inflation_factor(x_train.values, i) for i in range(x_train.shape[1])]
vif["features"] = x_train.columns
Can you write Python code for it?

n_comp = 128
pca = PCA(n_components=n_comp, random_state=42)
pca2_results_train = pca.fit_transform(x_train)
pca2_results_test = pca.transform(x_test)

train_reshaped = np.array([i.reshape((-1, 1)) for i in pca2_results_train])
train_reshaped = train_reshaped.astype(np.float32)
train_reshaped.shape


I am working on the file "Kaggle/Mercedes-Benz Greener Manufacturing/neural_network.ipynb".
The first task is:
Combine submission
Can you write Python code for it?

sub_1 = pd.read_csv("submission_baseLine.csv")

sub_2 = pd.read_csv("submissions_2017-05-31 15:48:40.546392.csv")

sub_3 = output.copy()

mean_pred = (sub_1.y.values + sub_2.y.values + sub_3.y.values) / 3

output_mean = pd.DataFrame({"ID": test.index, "y": mean_pred})

output_mean.to_csv("submissions_mean_{}.csv".format(datetime.datetime.today()), index=False)

sub_1 = pd.read_csv("submission_baseLine.csv")

sub_2 = pd.read_csv("submissions_2017-05-31 15:48:40.546392.csv")

mean_pred = (sub_1.y.values + sub_2.y.values ) / 2

output_mean = pd.DataFrame({"ID": test.index, "y": mean_pred})

output_mean.to_csv("submissions_mean_2_{}.csv".format(datetime.datetime.today()), index=False)


I am working on the file "Kaggle/Mercedes-Benz Greener Manufacturing/neural_network.ipynb".
The first task is:
Prediction
Can you write Python code for it?

test_reshaped = np.array([i.reshape((-1, 1)) for i in pca2_results_test])
test_reshaped = test_reshaped.astype(np.float32)
test_reshaped.shape

y_pred_test = model.predict(test_reshaped)

output = pd.DataFrame({"ID": test.index, "y": y_pred_test.reshape(-1)})

output.head()

output.to_csv("submissions_{}.csv".format(datetime.datetime.today()), index=False)


I am working on the file "Kaggle/Mercedes-Benz Greener Manufacturing/neural_network.ipynb".
The first task is:
Train the model
Can you write Python code for it?

# Idea: Simple model
model = Sequential()
model.add(Bidirectional(SimpleRNN(128, return_sequences=True, activation="relu"), input_shape=(None, 1)))
model.add(Bidirectional(SimpleRNN(64, return_sequences=True, activation="relu")))
model.add(Bidirectional(SimpleRNN(32, return_sequences=False, activation="relu")))
model.add(Dropout(0.5))
model.add(Dense(1, activation="linear"))
model.compile(optimizer="rmsprop", loss="mse")
model.summary()

# Idea: Funnel -> reduce information after each layer / deep model
model = Sequential()
model.add(Bidirectional(SimpleRNN(64, return_sequences=True, activation="relu"), input_shape=(None, 1)))
model.add(Bidirectional(SimpleRNN(64, return_sequences=True, activation="relu")))
model.add(TimeDistributed(Dense(32, activation="relu")))
model.add(Bidirectional(SimpleRNN(32, return_sequences=True, activation="relu")))
model.add(Bidirectional(SimpleRNN(32, return_sequences=True, activation="relu")))
model.add(TimeDistributed(Dense(16, activation="relu")))
model.add(Bidirectional(SimpleRNN(16, return_sequences=False, activation="relu")))
model.add(Dropout(0.5))
model.add(Dense(1, activation="linear"))
model.compile(optimizer="rmsprop", loss="mse")
model.summary()

early_stop = EarlyStopping(monitor="loss", patience=10)
file_path = "weights.{epoch:02d}-{val_loss:.2f}.hdf5"
checkpoint = ModelCheckpoint(file_path)
model_run = model.fit(train_reshaped, y_train, epochs=100 ,validation_split=0.02, callbacks=[early_stop, checkpoint])

y_pred_train = model.predict(train_reshaped)

print("the R2 score is : {}".format(r2_score(y_train, y_pred_train)))


I am working on the file "Kaggle/Titanic/Life_Predictor.ipynb".
The first task is:

What is a Random Forest?

As with all the important questions in life, this is best deferred to the Wikipedia page. A random forest is an ensemble of decision trees which will output a prediction value, in this case survival. Each decision tree is constructed by using a random subset of the training data. After you have trained your forest, you can then pass each test row through it, in order to output a prediction. Simple! Well not quite! This particular python function requires floats for the input variables, so all strings need to be converted, and any missing data needs to be filled.
Replace Male=2 female=1 & child=0
Can you write Python code for it?

import random
def PersonFunc(x):
    if x=="male":
        return 2
    elif x=="female":
        return 1
    elif x=="child":
        return 0
    else:
        return x

def EmbarkedFunc(x):
    if x=="S":
        return 1
    elif x=="C":
        return 2
    elif x=="Q":
        return 3
    elif np.isnan(x):
        return random.choice([1,2,3])
    else:
        return x

mData["Person"]  = mData["Person"].apply(PersonFunc)
test["Person"]  = test["Person"].apply(PersonFunc)

mData["Embarked"]  = mData["Embarked"].apply(EmbarkedFunc)
test["Embarked"]  = test["Embarked"].apply(EmbarkedFunc)

mData[:8]

print "Nan Entries in Cabin :",mData["Cabin"].isnull().sum()

def CabinFun(x):
    if type(x)==str:
        return ord(x[0])-ord('A')
    elif np.isnan(x):
        return 0
    else:
        return x

mData["Cabin"]  = mData["Cabin"].apply(CabinFun)
test["Cabin"]  = test["Cabin"].apply(CabinFun)

mData.head()

avgAge=mData["Age"].mean()
stander_dev=avgAge=mData["Age"].std()

avgAge_test = test["Age"].mean()
satnder_dev_test = test["Age"].std()

#select Randint between avg-std,avg+std
train_replacement = np.random.randint(avgAge-stander_dev,avgAge+stander_dev, size=mData["Age"].isnull().sum())
test_replacement = np.random.randint(avgAge_test-satnder_dev_test,avgAge_test+satnder_dev_test,size=test["Age"].isnull().sum())

train_replacement

mData["Age"][np.isnan(mData["Age"])] = train_replacement
test["Age"][np.isnan(test["Age"])] = test_replacement

mData.head()


I am working on the file "Kaggle/Titanic/Life_Predictor.ipynb".
The first task is:
Decision Tree Classification
Can you write Python code for it?

train_data = mData.values[0::,1::] #remove label i.e Survived col [start with zero row select all row
                                    # & start from 1dt coloumn select all coloumn except 0th]
train_label = mData.values[0::,0]   #select all row & 0th coloumn
test_data=test.values #We will remove PassangerID when required i.e During prediction

train_data=normalize(train_data, norm='l2', axis=1, copy=True)
n_test_data=normalize(test_data, norm='l2', axis=1, copy=True)

clsf = RandomForestClassifier(n_estimators=100)

clsf=clsf.fit(train_data,train_label)

pred_train=clsf.predict(train_data)
print "Accuracy Score for trainig Data :",accuracy_score(pred_train,train_label)*100,"%"

pred_test=clsf.predict(n_test_data[0::,1::])

#Convert to dataframe
result=pd.DataFrame({"PassengerId" :test_data[0::,0],"Survived" : pred_test})

result["Survived"] = result["Survived"].astype(int)
result["PassengerId"] = result["PassengerId"].astype(int)

result


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
1.
In the cell below, create a DataFrame fruits that looks like this:
Can you write Python code for it?

# Your code goes here. Create a dataframe matching the above diagram and assign it to the variable fruits.
fruits = pd.DataFrame({'Apples': ['30'],
              'Bananas': ['21']})

#q1.check()
fruits

fruits = pd.DataFrame([[30, 21]], columns=['Apples', 'Bananas'])
fruits


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
1.
Who are the most common wine reviewers in the dataset? Create a Series whose index is the taster_twitter_handle category from the dataset, and whose values count how many reviews each person wrote.
Can you write Python code for it?

reviews_written = reviews.groupby('taster_twitter_handle').size()

#or
reviews_written

reviews_written = reviews.groupby('taster_twitter_handle').taster_twitter_handle.count()
reviews_written


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
1.
region_1 and region_2 are pretty uninformative names for locale columns in the dataset. Create a copy of reviews with these columns renamed to region and locale, respectively.
Can you write Python code for it?

renamed = reviews.rename(columns=dict(region_1='region', region_2='locale'))
# q1.check()
renamed


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
2.
Create a Series from entries in the points column, but convert the entries to strings. Hint: strings are str in native Python.
Can you write Python code for it?

point_strings = reviews.points.astype(str)

point_strings


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
2.
Set the index name in the dataset to wines.
Can you write Python code for it?

reindexed = reviews.rename_axis('wines', axis='rows')
reindexed


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
2.
What is the best wine I can buy for a given amount of money? Create a Series whose index is wine prices and whose values is the maximum number of points a wine costing that much was given in a review. Sort the values by price, ascending (so that 4.0 dollars is at the top and 3300.0 dollars is at the bottom).
Can you write Python code for it?

best_rating_per_price = reviews.groupby('price')['points'].max().sort_index()
best_rating_per_price


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
3.
Create a variable ingredients with a Series that looks like:
Flour     4 cups
Milk       1 cup
Eggs     2 large
Spam       1 can
Name: Dinner, dtype: object
Can you write Python code for it?

quantities = ['4 cups', '1 cup', '2 large', '1 can']
items = ['Flour', 'Milk', 'Eggs', 'Spam']
recipe = pd.Series(quantities, index=items, name='Dinner')
recipe


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
3.
Select the first row of data (the first record) from reviews, assigning it to the variable first_row.
Can you write Python code for it?

first_row = reviews.iloc[0]

# q3.check()
first_row

# Solution:

first_row = reviews.iloc[0]


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
3.
What are the minimum and maximum prices for each variety of wine? Create a DataFrame whose index is the variety category from the dataset and whose values are the min and max values thereof.
Can you write Python code for it?

price_extremes = reviews.groupby('variety').price.agg([min, max])
price_extremes


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
4.
Read the following csv dataset of wine reviews into a DataFrame called reviews:

The filepath to the csv file is ../input/wine-reviews/winemag-data_first150k.csv. The first few lines look like:
,country,description,designation,points,price,province,region_1,region_2,variety,winery
0,US,"This tremendous 100% varietal wine[...]",Martha's Vineyard,96,235.0,California,Napa Valley,Napa,Cabernet Sauvignon,Heitz
1,Spain,"Ripe aromas of fig, blackberry and[...]",Carodorum Selección Especial Reserva,96,110.0,Northern Spain,Toro,,Tinta de Toro,Bodega Carmen Rodríguez
Can you write Python code for it?

#reviews = pd.read_csv("../input/wine-reviews/winemag-data_first150k.csv", index_col=0)
reviews = pd.read_csv("winemag-data_first150k.csv", index_col=0)

reviews


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
4.
The Powerlifting Database dataset on Kaggle includes one CSV table for powerlifting meets and a separate one for powerlifting competitors. Run the cell below to load these datasets into dataframes:
Can you write Python code for it?

# powerlifting_meets = pd.read_csv("../input/powerlifting-database/meets.csv")
# powerlifting_competitors = pd.read_csv("../input/powerlifting-database/openpowerlifting.csv")
powerlifting_meets = pd.read_csv("meets.csv")
powerlifting_meets.head()

powerlifting_competitors = pd.read_csv("openpowerlifting.csv")
powerlifting_competitors.head()


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
5.
Create a Series whose index is reviewers and whose values is the average review score given out by that reviewer. Hint: you will need the taster_name and points columns.
Can you write Python code for it?

reviewer_mean_ratings = reviews.groupby('taster_name').points.mean()
reviewer_mean_ratings

reviewer_mean_ratings.describe()


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
5.
Run the cell below to create and display a DataFrame called animals:
Can you write Python code for it?

animals = pd.DataFrame({'Cows': [12, 20], 'Goats': [22, 19]},
                       index=['Year 1', 'Year 2'])
animals


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
6.
Create a variable df containing the country, province, region_1, and region_2 columns of the records with the index labels 0, 1, 10, and 100. In other words, generate the following DataFrame:
Can you write Python code for it?

df = reviews.loc[[0,1,10,100],['country', 'province', 'region_1', 'region_2']]

# q6.check()
df

# Solution:

cols = ['country', 'province', 'region_1', 'region_2']
indices = [0, 1, 10, 100]
df = reviews.loc[indices, cols]
df


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
6.
What combination of countries and varieties are most common? Create a Series whose index is a MultiIndexof {country, variety} pairs. For example, a pinot noir produced in the US should map to {"US", "Pinot Noir"}. Sort the values in the Series in descending order based on wine count.
Can you write Python code for it?

country_variety_counts = reviews.groupby(['country', 'variety']).size().sort_values(ascending=False)
country_variety_counts


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
8.
Create a DataFrame italian_wines containing reviews of wines made in Italy. Hint: reviews.country equals what?
Can you write Python code for it?

italian_wines = reviews.loc[reviews.country == 'Italy']

# q8.check()
italian_wines

# Solution:

italian_wines = reviews[reviews.country == 'Italy']
italian_wines


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
A DataFrame or Series index has its own dtype, too:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
A Series is, in essence, a single column of a DataFrame. So you can assign column values to the Series the same way as before, using an index parameter. However, a Series does not have a column name, it only has one overall name:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Alternatively, the dtypes property returns the dtype of every column in the DataFrame:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Both loc and iloc are row-first, column-second. This is the opposite of what we do in native Python, which is column-first, row-second.
This means that it's marginally easier to retrieve rows, and marginally harder to get retrieve columns. To get a column with iloc, we can do the following:
Can you write Python code for it?

reviews.iloc[:, 0]

On its own, the : operator, which also comes from native Python, means "everything". When combined with other selectors, however, it can be used to indicate a range of values. For example, to select the country column from just the first, second, and third row, we would do:

reviews.iloc[:3, 0]


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Choosing between loc and iloc
When choosing or transitioning between loc and iloc, there is one "gotcha" worth keeping in mind, which is that the two methods use slightly different indexing schemes.
iloc uses the Python stdlib indexing scheme, where the first element of the range is included and the last one excluded. So 0:10 will select entries 0,...,9. loc, meanwhile, indexes inclusively. So 0:10 will select entries 0,...,10.
Why the change? Remember that loc can index any stdlib type: strings, for example. If we have a DataFrame with index values Apples, ..., Potatoes, ..., and we want to select "all the alphabetical fruit choices between Apples and Potatoes", then it's a lot more convenient to index df.loc['Apples':'Potatoes'] than it is to index something like df.loc['Apples', 'Potatoet] (t coming after s in the alphabet).
This is particularly confusing when the DataFrame index is a simple numerical list, e.g. 0,...,1000. In this case df.iloc[0:1000] will return 1000 entries, while df.loc[0:1000] return 1001 of them! To get 1000 elements using loc, you will need to go one lower and ask for df.iloc[0:999].
Otherwise, the semantics of using loc are the same as those for iloc.
Manipulating the index
Label-based selection derives its power from the labels in the index. Critically, the index we use is not immutable. We can manipulate the index in any way we see fit.
The set_index() method can be used to do the job. Here is what happens when we set_index to the title field:
Can you write Python code for it?

# reviews.set_index("title")  # da error, no existe esa columna

reviews.set_index("variety")


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Creating data¶
There are two core objects in pandas: the DataFrame and the Series.
DataFrame
A DataFrame is a table. It contains an array of individual entries, each of which has a certain value. Each entry corresponds to a row (or record) and a column.
For example, consider the following simple DataFrame:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Exercises
1.
What is the data type of the points column in the dataset?
Can you write Python code for it?

# Your code here
dtype = reviews.points.dtype


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Exercises
View the first several lines of your data by running the cell below:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Finally, it's worth knowing that negative numbers can be used in selection. This will start counting forwards from the end of the values. So for example here are the last five elements of the dataset.
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
If we have a Python dictionary, we can access its values using the indexing ([]) operator. We can do the same with columns in a DataFrame:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
If you want to get some particular simple summary statistic about a column in a DataFrame or a Series, there is usually a helpful pandas function that makes it happen.
For example, to see the mean of the points allotted (e.g. how well an averagely rated wine does), we can use the mean() function:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
In Python, we can access the property of an object by accessing it as an attribute. A book object, for example, might have a title property, which we can access by calling book.title. Columns in a pandas DataFrame work in much the same way.
Hence to access the country property of reviews we can use:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
In this code we are performing an operation between a lot of values on the left-hand side (everything in the Series) and a single value on the right-hand side (the mean value). Pandas looks at this expression and figures out that we must mean to subtract that mean value from every value in the dataset.
Pandas will also understand what to do if we perform these operations between Series of equal length. For example, an easy way of combining country and region information in the dataset would be to do the following:
Can you write Python code for it?

reviews.country + " - " + reviews.region_1


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Indexing in pandas
The indexing operator and attribute selection are nice because they work just like they do in the rest of the Python ecosystem. As a novice, this makes them easy to pick up and use. However, pandas has its own accessor operators, loc and iloc. For more advanced operations, these are the ones you're supposed to be using.
Index-based selection
Pandas indexing works in one of two paradigms. The first is index-based selection: selecting data based on its numerical position in the data. iloc follows this paradigm.
To select the first row of data in a DataFrame, we may use the following:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Look at an overview of your data by running the following line.
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Maps
A map is a term, borrowed from mathematics, for a function that takes one set of values and "maps" them to another set of values. In data science we often have a need for creating new representations from existing data, or for transforming data from the format it is in now to the format that we want it to be in later. Maps are what handle this work, making them extremely important for getting your work done!
There are two mapping methods that you will use often.
map() is the first, and slightly simpler one. For example, suppose that we wanted to remean the scores the wines received to 0. We can do this as follows:
Can you write Python code for it?

review_points_mean = reviews.points.mean()
reviews.points.map(lambda p: p - review_points_mean)


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Or we could fill each missing value with the first non-null value that appears sometime after the given record in the database. This is known as the backfill strategy.
Alternatively, we may have a non-null value that we would like to replace. For example, suppose that since this dataset was published, reviewer Kerin O'Keefe has changed her Twitter handle from @kerinokeefe to @kerino. One way to reflect this in the dataset is using the replace() method:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Or with an iterable of values:
Can you write Python code for it?

reviews['index_backwards'] = range(len(reviews), 0, -1)
reviews['index_backwards']


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Or, to select just the second and third entries, we would do:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Pandas comes with a few built-in conditional selectors, two of which we will highlight here.
The first is isin. isin is lets you select data whose value "is in" a list of values. For example, here's how we can use it to select wines only from Italy or France:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Pandas provides many common mapping operations as built-ins. For example, here's a faster way of remeaning our points column:
Can you write Python code for it?

review_points_mean = reviews.points.mean()
reviews.points - review_points_mean


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Para bajar los datos:
https://www.kaggle.com/luisrivera/exercise-creating-reading-and-writing/edit
Para subir los datos a Google Collaborative:
D:\kaggle\Cursos\Panda
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Renaming-and-combining columns
https://www.kaggle.com/residentmario/renaming-and-combining
Introduction
Oftentimes data will come to us with column names, index names, or other naming conventions that we are not satisfied with. In that case, you'll learn how to use pandas functions to change the names of the offending entries to something better.
You'll also explore how to combine data from multiple DataFrames and/or Series.
Renaming
The first function we'll introduce here is rename(), which lets you change index names and/or column names. For example, to change the points column in our dataset to score, we would do:
Can you write Python code for it?

import pandas as pd

# reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv", index_col=0)
reviews = pd.read_csv("winemag-data-130k-v2.csv", index_col=0)
# from learntools.core import binder; binder.bind(globals())
# from learntools.pandas.renaming_and_combining import *
# print("Setup complete.")

reviews.head()

reviews.rename(columns={'points': 'score'})


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Replacing missing values is a common operation. Pandas provides a really handy method for this problem: fillna(). fillna() provides a few different strategies for mitigating such data. For example, we can simply replace each NaN with an "Unknown":
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Series
A Series, by contrast, is a sequence of data values. If a DataFrame is a table, a Series is a list. And in fact you can create one with nothing more than a list
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
So our new DataFrame has 130,000 records split across 14 different columns. That's almost 2 million entries!
We can examine the contents of the resultant DataFrame using the head() command, which grabs the first five rows:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Sorting
Looking again at countries_reviewed we can see that grouping returns data in index order, not in value order. That is to say, when outputting the result of a groupby, the order of the rows is dependent on the values in the index, not in the data.
To get data in the order want it in we can sort it ourselves. The sort_values() method is handy for this.
Can you write Python code for it?

countries_reviewed = countries_reviewed.reset_index()
countries_reviewed.sort_values(by='len')

countries_reviewed.sort_values(by='len', ascending=False)

countries_reviewed.sort_index()

countries_reviewed.sort_values(by=['country', 'len'])


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Summary functions
Pandas provides many simple "summary functions" (not an official name) which restructure the data in some useful way. For example, consider the describe() method:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
Suppose we'll buy any wine that's made in Italy or which is rated above average. For this we use a pipe (|):
Can you write Python code for it?

reviews.loc[(reviews.country == 'Italy') | (reviews.points >= 90)]


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The Series and the DataFrame are intimately related. It's helpful to think of a DataFrame as actually being just a bunch of Series "glued together". We'll see more of this in the next section of this tutorial.
Reading data files
Being able to create a DataFrame or Series by hand is handy. But, most of the time, we won't actually be creating our own data by hand. Instead, we'll be working with data that already exists.
Data can be stored in any of a number of different forms and formats. By far the most basic of these is the humble CSV file. When you open a CSV file you get something that looks like this:
Product A,Product B,Product C,
30,21,9,
35,34,1,
41,11,11
So a CSV file is a table of values separated by commas. Hence the name: "Comma-Separated Values", or CSV.
Let's now set aside our toy datasets and see what a real dataset looks like when we read it into a DataFrame. We'll use the pd.read_csv() function to read the data into a DataFrame. This goes thusly:
Can you write Python code for it?

# wine_reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv")


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The dictionary-list constructor assigns values to the column labels, but just uses an ascending count from 0 (0, 1, 2, 3, ...) for the row labels. Sometimes this is OK, but oftentimes we will want to assign these labels ourselves.
The list of row labels used in a DataFrame is known as an Index. We can assign values to it by using an index parameter in our constructor:
Can you write Python code for it?

pd.DataFrame({'Bob': ['I liked it.', 'It was awful.'],
              'Sue': ['Pretty good.', 'Bland.']},
             index=['Product A', 'Product B'])


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The function you pass to map() should expect a single value from the Series (a point value, in the above example), and return a transformed version of that value. map() returns a new Series where all the values have been transformed by your function.
apply() is the equivalent method if we want to transform a whole DataFrame by calling a custom method on each row.
La función que pase a map () debería esperar un único valor de la Serie (un valor de punto, en el ejemplo anterior) y devolver una versión transformada de ese valor. map () devuelve una nueva serie donde todos los valores han sido transformados por su función.
apply () es el método equivalente si queremos transformar un DataFrame completo llamando a un método personalizado en cada fila.
Can you write Python code for it?

def remean_points(row):
    row.points = row.points - review_points_mean
    return row

reviews.apply(remean_points, axis='columns')


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The lsuffix and rsuffix parameters are necessary here because the data has the same column names in both British and Canadian datasets. If this wasn't true (because, say, we'd renamed them beforehand) we wouldn't need them.
Exercise: Renaming and Combining
https://www.kaggle.com/luisrivera/exercise-renaming-and-combining/edit
Can you write Python code for it?

# import pandas as pd

# reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv", index_col=0)

# from learntools.core import binder; binder.bind(globals())
# from learntools.pandas.renaming_and_combining import *
# print("Setup complete.")


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The middlemost combiner in terms of complexity is join(). join() lets you combine different DataFrame objects which have an index in common. For example, to pull down videos that happened to be trending on the same day in both Canada and the UK, we could do the following:
Can you write Python code for it?

left = canadian_youtube.set_index(['title', 'trending_date'])
right = british_youtube.set_index(['title', 'trending_date'])

left.join(right, lsuffix='_CAN', rsuffix='_UK')


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The pd.read_csv() function is well-endowed, with over 30 optional parameters you can specify. For example, you can see in this dataset that the CSV file has a built-in index, which pandas did not pick up on automatically. To make pandas use that column for the index (instead of creating a new one from scratch), we can specify an index_col.
Can you write Python code for it?

# wine_reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv", index_col=0)
# wine_reviews.head()

wine_reviews = pd.read_csv("winemag-data-130k-v2.csv", index_col=0)
wine_reviews.head()


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The replace() method is worth mentioning here because it's handy for replacing missing data which is given some kind of sentinel value in the dataset: things like "Unknown", "Undisclosed", "Invalid", and so on.
Exercise: Data Types and Missing Values
https://www.kaggle.com/luisrivera/exercise-data-types-and-missing-values/edit
Can you write Python code for it?

# import pandas as pd

# reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv", index_col=0)

# from learntools.core import binder; binder.bind(globals())
# from learntools.pandas.data_types_and_missing_data import *
# print("Setup complete.")


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
The second is isnull (and its companion notnull). These methods let you highlight values which are (or are not) empty (NaN). For example, to filter out wines lacking a price tag in the dataset, here's what we would do:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
These are the two ways of selecting a specific Series out of a DataFrame. Neither of them is more or less syntactically valid than the other, but the indexing operator [] does have the advantage that it can handle column names with reserved characters in them (e.g. if we had a country providence column, reviews.country providence wouldn't work).
Doesn't a pandas Series look kind of like a fancy dictionary? It pretty much is, so it's no surprise that, to drill down to a single specific value, we need only use the indexing operator [] once more:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
This is useful if you can come up with an index for the dataset which is better than the current one.
Conditional selection
So far we've been indexing various strides of data, using structural properties of the DataFrame itself. To do interesting things with the data, however, we often need to ask questions based on conditions.
For example, suppose that we're interested specifically in better-than-average wines produced in Italy.
We can start by checking if each wine is Italian or not:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
This method generates a high-level summary of the attributes of the given column. It is type-aware, meaning that its output changes based on the data type of the input. The output above only makes sense for numerical data; for string data here's what we get:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
This operation produced a Series of True/False booleans based on the country of each record. This result can then be used inside of loc to select the relevant data:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
To see a list of unique values we can use the unique() function:
Can you write Python code for it?

# reviews.taster_name.unique()    # se demora mucho??


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
We can use the shape attribute to check how large the resulting DataFrame is:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
https://www.kaggle.com/luisrivera/exercise-grouping-and-sorting/edit
Exercise: Grouping and Sorting
Can you write Python code for it?

import pandas as pd

# reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv", index_col=0)
#pd.set_option("display.max_rows", 5)
reviews = pd.read_csv("./winemag-data-130k-v2.csv", index_col=0)

# from learntools.core import binder; binder.bind(globals())
# from learntools.pandas.grouping_and_sorting import *
# print("Setup complete.")


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
https://www.kaggle.com/luisrivera/exercise-indexing-selecting-assigning/edit
Ejercicios
Run the following cell to load your data and some utility functions (including code to check your answers).
Can you write Python code for it?

import pandas as pd

# reviews = pd.read_csv("../input/wine-reviews/winemag-data-130k-v2.csv", index_col=0)
reviews = pd.read_csv("winemag-data-130k-v2.csv", index_col=0)
pd.set_option("display.max_rows", 5)

# from learntools.core import binder; binder.bind(globals())
# from learntools.pandas.indexing_selecting_and_assigning import *
# print("Setup complete.")


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
https://www.kaggle.com/residentmario/data-types-and-missing-values
data-types-and-missing-values
Dtypes
The data type for a column in a DataFrame or a Series is known as the dtype.
You can use the dtype property to grab the type of a specific column. For instance, we can get the dtype of the price column in the reviews DataFrame:
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
https://www.kaggle.com/residentmario/summary-functions-and-maps
Funciones y Mapas
Can you write Python code for it?


I am working on the file "Kaggle_Panda_Curso.ipynb".
The first task is:
iloc is conceptually simpler than loc because it ignores the dataset's indices. When we use iloc we treat the dataset like a big matrix (a list of lists), one that we have to index into by position. loc, by contrast, uses the information in the indices to do its work. Since your dataset usually has meaningful indices, it's usually easier to do things using loc instead. For example, here's one operation that's much easier using loc:
Can you write Python code for it?

reviews.loc[:, ['taster_name', 'taster_twitter_handle', 'points']]


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Did you have your voila moment? :D
Lets also look at the error for the estimate, lets plot it.
Can you write Python code for it?

plt.figure()
plt.plot(range(iter_max),error_estimate_vector)
plt.xlabel('Time Instances')
plt.ylabel('Voltage Error')


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Kalman Filter
You might be wondering where is the control in all this. So now that's easily introduced in the actual formulation of Kalman filter which we will look at now. In the actual Kalman Filter we deal with multi-variable setting unlike that of the voltmeter example. So let me describe the inputs and the outputs and the parameters
Inputs
* $\textbf{Z}_m : $ The measurements vector at each instant m
* $\textbf{U}_m : $ This is new!, it denotes the controls provided to the system at instance m (this is just like force being the control for velocity of the car)
Outputs
* $\textbf{X}_m : $ Newest estimate of the current state (state can be thought of as a parameter vector)
* $P_m :$ The newest estimate for the average error of the state
Parameters
* A : State transition matrix, basically the constant matrix multiplied to the previous estimate in the process equation
* B : Control matrix, one to be multiplied to the control vector in the process equation
* H : Observation matrix, the proportionality factor for state to be equal to the measurements
* Q : Covariance matrix for the process error (this is again assumed to be known)
* R : Covariance matrix for the measurements error (again known)
Now we are ready to write the basic equations for the KF, don't worry much of the above will be cleared as you look at the equation.
Lets start with the two basic equations, first is the process equation
$$\textbf{X}m = A\textbf{X}{m-1} + B\textbf{U}_{m-1} + \pmb{\omega}_m \qquad (8)$$
and then we have the measurements equation
$$\textbf{Z}_m = H\textbf{X}_m + \pmb{\nu}_m \qquad (9)$$
Here, we model the two error terms $\pmb{\omega}_m$ and $\pmb{\nu}_m$ as multi-variate Gaussian distributions with zero mean and covariance matrices Q and P respectively, i.e. $\pmb{\omega}_m \textrm{~} \mathcal{N}(0,Q)$
and $\pmb{\nu}_m \textrm{~} \mathcal{N}(0,P)$
So now following the exact same philosophy of that we followed for the derivation in voltmeter example we get the update equations for the general Kalman Filter. I will just...
Can you write Python code for it?

class kalmanFilter:
    def __init__(self, X0, P0, A, B, H, Q, R):
        self.A = A                       # State Transition Matrix
        self.B = B                       # Control Matrix
        self.H = H                       # Observation Matrix
        self.Q = Q                       # Covariance for the process error
        self.R = R                       # Covariance for the measurements error
        self.current_estimate = X0       # this is the initial guess of the state
        self.current_error_estimate = P0 # initial guess for the state estimate error

    def getEstimate(self):
        # returns the current state estimate
        return self.current_estimate

    def getErrorEstimate(self):
        # returns the current state error estimate
        return self.current_error_estimate

    def iteration(self, U, Z):
        # here is where the updates happen
        # U = control vector
        # Z = measurements vector

        # prior prediction step
        prior_estimate = self.A * self.current_estimate + self.B * U
        prior_error_estimate = (self.A * self.current_error_estimate) * np.transpose(self.A) + self.Q

        # intermediate observation
        y = Z - self.H * prior_estimate
        y_covariance = self.H * prior_error_estimate * np.transpose(self.H) + self.R

        # Correction Step
        K = prior_error_estimate * np.transpose(self.H) * np.linalg.inv(y_covariance)
        self.current_estimate = prior_estimate + K * y
        # We need the size of the matrix so we can make an identity matrix.
        size = self.current_error_estimate.shape[0]
        # eye(n) = nxn identity matrix.
        self.current_error_estimate = (np.eye(size) - K * self.H) * prior_error_estimate


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Let's plot how the measurements look as compared to the true voltage
Can you write Python code for it?

plt.plot(range(iter_max),true_voltage_mat,'b',range(iter_max), measurements,'r')
plt.xlabel('Time Instances')
plt.ylabel('Voltage')
plt.legend(('true voltage', 'measured'), loc=4)


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Lets plot the position data and measurements
Can you write Python code for it?

plt.figure()
plt.plot(true_value[0,:],true_value[2,:],'b',measurements[0,:],measurements[2,:],'r')
plt.xlabel('X Position')
plt.ylabel('Y Position')
plt.legend(('true trajectory', 'measured trajectory'), loc=2)


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Now for solving any computational problem we need to model the system (in layman terms, set of equations which is used to describe the overall situation and how it varies with physical inputs and parameters).
The simplest way to model the noisy voltmeter at every measurements instant is through the following equation
\begin{equation}
V_m = V_{m-1} + \omega_m \qquad (1)
\end{equation}
Here, $V_m$ : Voltage at current time,
      $V_{m-1}$ : Voltage at previous time instant,
      $\omega_m$ : Random Noise (process noise)
We also have measurements taken at each instance m which is given by $Z_m$ and is corrupted by some sensor (measurements) noise $\nu_m$
$$Z_m = V_m + \nu_m \qquad (2)$$
Usually in such cases as $\nu_m$ is introduced due to faults in voltmeter (sensor) and hence we usually know it's characteristics (least count, precision ... ring any bells?). So here we will model $\nu_m$ as a Gaussian Random Variable (as it is usually done) with zero mean and standard deviation $\sigma_{\nu} = 0.3$ (for simulation sake). $\omega_m$ is more difficult to predict and is introduced by error due to non-ideality (we know it's constant DC voltage, but is it really!) of our process equations but we still assume that we know it. Again, $\omega_m$ is to be modeled by a Gaussian Random Variable with zero mean and standard deviation $\sigma_{\omega} = 0.01$.
So before solving this let's model this voltmeter, What are the parameters to give. The true constant voltage lets say variable true_voltage = 1, and we need a noise level as well (this is voltmeter's error so will feature in measurements error), say variable noise_level_sig = 0.2 (KF works when noise estimate are off by a mark...). Let's take measurements for 50 instances store it in iter_max and we will generate the measurements for each instant which will be just $~ \mathcal{N}(true_voltage,noise_level_sig)$ (Think about this :))
Can you write Python code for it?

true_voltage = 1
noise_level_sig = 0.2
iter_max = 50
measurements = []
true_voltage_mat = []

for i in range(iter_max):
    measured = random.gauss(true_voltage , noise_level_sig)
    measurements.append(measured)
    true_voltage_mat.append(true_voltage)


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Now lets plot again to see weather we are good to go or not.
Can you write Python code for it?

plt.figure()
plt.plot(range(iter_max),true_voltage_mat,'b',range(iter_max), measurements,'r', range(iter_max), estimate_vector_new,'g')
plt.xlabel('Time Instances')
plt.ylabel('Voltage')
plt.legend(('true voltage', 'measured', 'filtered'), loc=1)


I am working on the file "Kalman Filter/Kalman-first.ipynb".
The first task is:
Now starts the actual thing, we want to filter this :D, by using Kalman Filtering. But first we need to derive, so do get ready for a chunk load of theory, but but please be patient as once we are done with this, the standard Kalman Filter will be a piece of cake
:)
Filtering!!
Math Math Math.
That being said, to solve this problem iteratively we need two major step

Predict the voltage at next instance using the previous estimate (1)
Correct the estimate based on the measurements at that instant

So we define two variables, $\hat{V}^-_m :$ prior estimate of the voltage given only the knowledge of the process (the equation (1)) and $\hat{V}_m :$ posterior estimate of the voltage at step m given the knowledge of the measurements $Z_m$
So lets' start the firing of equations, just one comment before this, we will also estimate the error in the estimate at every iteration along with the estimate itself.
$$e_m = V_m - \hat{V}_m \quad \textrm{and} \quad \sigma^2_m = \mathbb{E}[e_m^2]$$
and
$$e^-_m = V_m - \hat{V}^-_m \quad \textrm{and} \quad \sigma^{2-}_m = \mathbb{E}[e_m^{2-}]$$
We have to minimize this $\sigma^2_m$. So now as any sane person (ok statistician) would do we would model the posterior estimate $\hat{V}_m$ as the linear combination of the prior estimate $\hat{V}^-_m$ and the deviation of the estimate from the measurements (also called as innovation term) given as
$$y_m = Z_m - \hat{V}^-_m \qquad (3)$$
Putting the above ramble in equation we have
$$\hat{V}_m = \hat{V}^-_m + k_my_m \qquad (4)$$
subtracting $V_m$ from both sides we get
$$\hat{V}_m - V_m = \hat{V}^-_m - V_m + k_m(Z_m - \hat{V}^-_m)$$
To compute $k_m$ we take the square and it's expectation and then differentiate the quadratic in $k_m$ to get something like (try this your self)
$$k_m = \frac{\mathbb{E}[(V_m - \hat{V}^-_m)(y_m)]}{\mathbb{E}[y_m^2]}$$
The numerator and denominator when expanded and taking into the account for independence of the R.V $Z_m$, $V_m$, and $\hat{V}^-_m$ (think about this too) we get
$$k_m =...
Can you write Python code for it?

# Initialize the parameters

initial_guess = 3
initial_guess_error = 1
sig_nu = 0.3
sig_omega = 0.01
estimate_vector = []
estimate_vector.append(initial_guess)
error_estimate_vector = []
error_estimate_vector.append(initial_guess_error)

# Run the Filter

for i in range(iter_max-1):
    # first the prior estimation step

    volt_prior_est = estimate_vector[i]
    error_prior_est = error_estimate_vector[i] + sig_omega * sig_omega

    # estimate correction

    k = error_prior_est/(error_prior_est + sig_nu * sig_nu)
    volt_corrected_est = volt_prior_est + k * (measurements[i+1] - volt_prior_est)
    error_corrected_est = (1 - k) * error_prior_est
    estimate_vector.append(volt_corrected_est)
    error_estimate_vector.append(error_corrected_est)


I am working on the file "Kalman1.ipynb".
The first task is:
A kibővített Kálmán-szűrés egy lépésében kiszámítjuk a rendszerállapot predikcióját majd mindig az újabb $\mathbf{z}_k$ méréssel frissítjük az állapot becslését.
predikció:
$$ \hat{\mathbf{x}}_k^-{} = f(\mathbf{x}_k)$$
$$ \mathbf{P}_k^-{} = \mathbf{A} \mathbf{P}_k \mathbf{A}^T + \mathbf{Q} $$
frissítés:
$$ \mathbf{K}_k = \mathbf{P}_k^-{} \mathbf{H}_k^T (\mathbf{H} \mathbf{P}_k^-{} \mathbf{H}^T + \mathbf{R})^{-1} $$
$$ \hat{\mathbf{x}}_k = \hat{\mathbf{x}}_k^-{} + \mathbf{K}_k (\mathbf{z}_k - h(\hat{\mathbf{x}}_k^-{}) $$
$$ \mathbf{P}_k = \mathbf{P}_k^-{} - \mathbf{K}_k \mathbf{H} \mathbf{P}_k^-{} $$
A lépést megvalósító Python függvény:
Can you write Python code for it?

def EKFstep(xk,Pk,A,H,Q,R,zk,dt,rs,rw):
    """a kibővített Kálmán-szűrő egy lépése
    """
    # állapot frissítése
    xkm = f(xk,dt)
    Pkm = np.dot(np.dot(A,Pk),A.T) + Q
    # újabb mérés, frissítés
    Kk = np.dot(Pkm,np.dot(H.T,np.linalg.pinv(np.dot(np.dot(H,Pkm),H.T) + R)))
    xk1 = xkm + np.dot(Kk,(zk - h(xkm,rs,rw)))
    Pk1 = Pkm - np.dot(np.dot(Kk,H),Pkm)

    return xk1,Pk1


I am working on the file "Kalman1.ipynb".
The first task is:
A számítást az EKFWheel(t,a1,a2,q,r) függvényben programoztuk. A függvény paraméterei a következők:
*  t: a mérési időpontok vektora
* a1, a2: a gyorsulásmérő szenzor adatai a t-vel megadott időpontokban
*  q: folyamat zaj
*  r: mérési zaj
A függvény az xe mátrixban adja vissza a becsült rendszer állapotot és a $-\pi \le \theta \le \pi$ közé eső szögértékeket.
A függvény Python kódja:
Can you write Python code for it?

def EKFwheel(t,a1,a2,q=0.07,r=5):
    """
    Kibővített Kálmán-szűrő kéttengelyű gyorsulásmérő szenzoros odometriához
     t - mérési időpontok vektora
     a1, a2 - mért szenzor centripetális és centrifugális gyorsulások (m/s**2)
     q - folyamat zaj variancia (m/s**2)
     r - mérési zaj variancia (m/s**2)
    """
    # szenzor tengelytávolsága és a kerék sugara (Samsung Galaxy S2 biciklire szerelve)
    rs = 0.095  # méter
    rw = 0.35   # méter
    nt = t.shape[0]

    # állapot vektor: t, p,vel,acc,theta
    xe = np.zeros((nt,5))
    xe[:,0]=t
    # indítás
    Q = q**2*np.eye(3)
    R = r**2*np.eye(2)
    xk = np.zeros((3,1))
    Pk = q**2*np.diag([0, 0, 1])
    Ak = np.zeros((3,3))

    for i in range(1,nt):
        dt = t[i]-t[i-1]
        Hk = H(xk,rs,rw)
        A = np.eye(3)+ np.array([[0,dt,0.5*dt**2], [0,0,dt], [0,0,0]])
        zk = np.array([a1[i],a2[i]])
        # EKF lépés
        xk1,Pk1 = EKFstep(xk,Pk,A,Hk,Q,R,zk,dt,rs,rw)
        xe[i,1:4] = xk1
        xk = xk1  # állapot frissítése
        Pk = Pk1  # állapot kovariancia frissítése
        om = xk1[0]/rw  # kerék elfordulási szöge
        xe[i,4]=np.arctan2(np.sin(om),np.cos(om));  # szög: (-pi, pi)

    return xe


I am working on the file "Kalman1.ipynb".
The first task is:
Az állapot átmenet $A$ mátrixa már linearizálva van:
$$ A = \begin{bmatrix}
1 & \Delta t & \frac{1}{2}\Delta t^2 \
0 & 1 & \Delta t\
0 & 0 & 1
\end{bmatrix}$$
Az állapot terjedés Python függvényét tehát egyszerűen megírhatjuk:
Can you write Python code for it?

def f(x,dt):
    """ állapot terjedés függvénye
     x  = állapot vektor (p,pdot,pdotdot)
     dt = időkülönbség
    """
    f1 = x[0] + x[1]*dt + 0.5*x[2]*dt**2
    f2 = x[1] + x[2]*dt
    f3 = x[2]
    return np.array([f1,f2,f3]).flatten()