I am working on the file "MultivariableRegression_ChallengeWithCrossValidation.ipynb".
The first task is:
Validating regression models for prediction
Now let's use cross-validation to obtain a more accurate description of our accuracy:
Can you write Python code for it?

from sklearn.cross_validation import cross_val_score

regr = linear_model.LinearRegression()

y = data['Property\ncrime'].values.reshape(-1, 1)
X = data[['Population', 'Violent\ncrime',
       'Murder', 'Rape\n(legacy\ndefinition)2',
       'Robbery', 'Aggravated\nassault']]

scores = cross_val_score(regr, X, y, cv = 10)

print("Percent accuracy within each fold:\n")
print(scores)
print("\nMean accuracy:\n")
print(scores.mean())


I am working on the file "Multivariate_Example.ipynb".
The first task is:
Let's try to detect the changes with independent features:
Can you write Python code for it?

from bayesian_changepoint_detection.priors import const_prior
from bayesian_changepoint_detection.offline_likelihoods import IndepentFeaturesLikelihood
from bayesian_changepoint_detection.bayesian_models import offline_changepoint_detection
from functools import partial

Q_ifm, P_ifm, Pcp_ifm = offline_changepoint_detection(
    data, partial(const_prior, p=1/(len(data) + 1)), IndepentFeaturesLikelihood(), truncate=-20
)

fig, ax = plt.subplots(2, figsize=[18, 8])
for p in changes:
    ax[0].plot([p,p],[np.min(data),np.max(data)],'r')
for d in range(2):
    ax[0].plot(data[:,d])
plt.legend(['Raw data with Original Changepoints'])
ax[1].plot(np.exp(Pcp_ifm).sum(0))
plt.legend(['Independent Factor Model'])
plt.show()


I am working on the file "Multivariate_Example.ipynb".
The first task is:
Unfortunately, not very good... Now let's try the full covariance model (warning, it'll take a while):
Can you write Python code for it?

from bayesian_changepoint_detection.offline_likelihoods import FullCovarianceLikelihood
Q_full, P_full, Pcp_full = offline_changepoint_detection(
    data, partial(const_prior, p=1/(len(data)+1)), FullCovarianceLikelihood(), truncate=-20
)

fig, ax = plt.subplots(2, figsize=[18, 8])
for p in changes:
    ax[0].plot([p,p],[np.min(data),np.max(data)],'r')
for d in range(2):
    ax[0].plot(data[:,d])
plt.legend(['Raw data with Original Changepoints'])
ax[1].plot(np.exp(Pcp_full).sum(0))
plt.legend(['Full Covariance Model'])
plt.show()


I am working on the file "MutEx and CoOc test in python.ipynb".
The first task is:
Big matrix
Can you write Python code for it?

row, col = 10000, 800
np.random.seed(77)
df = pd.DataFrame(sparse.random(row, col).A).apply(np.ceil)

df.columns = ['s' + str(x) for x in df.columns]
df.index = ['gene' + str(x) for x in df.index]

pd.set_option('display.max_columns', 1000)
df.head()

m = MutEx(df, permutations=10000)
group = random.sample( df.index.tolist(), 20)
%time r = m.calculate(group)
r


I am working on the file "MutEx and CoOc test in python.ipynb".
The first task is:
Example run , multi-threaded
Can you write Python code for it?

%time r = m.calculate(['gene0', 'gene1', 'gene2'])
print(r)


I am working on the file "MutEx and CoOc test in python.ipynb".
The first task is:
Example with many groups
Can you write Python code for it?

#some random groups and put it in a generator (alternatively list)
random.seed(18)
group_generator = (random.sample( df.index.tolist(), random.sample([2,3,4], 1)[0]) for x in range(10) )

result_list = [m.calculate(g) for g in group_generator]
result_df = pd.DataFrame.from_records([r.__dict__ for r in result_list])
result_df[['items','coverage', 'signal', 'mutex_pvalue', 'co_occurence_pvalue', 'signal_coverage_ratio', 'mean_sim_coverage']].sort_values('mutex_pvalue')


I am working on the file "MutEx and CoOc test in python.ipynb".
The first task is:
The 500 simulations of the mutation matrix - parallel execution
Can you write Python code for it?

from functools import partial
import multiprocessing as mp
pool = mp.Pool(processes=mp.cpu_count())
def dosim(m,x):
    np.random.seed(x)
    return m._simulate_observations(df.apply(sum, axis=1)).apply(sum)
partial_simul = partial(dosim)
n = 500
%time simulated_results = pool.starmap(partial_simul.func, zip(itertools.repeat(m,n), range(n)) )
pool.close()  # we are not adding any more processes
pool.join()  # tell it to wait until all threads are done before going on


I am working on the file "MutEx and CoOc test in python.ipynb".
The first task is:
calculate mean and standard deviation of simulated mutational burden for each sample
Can you write Python code for it?

sim_sample_muts = pd.concat(simulated_results, axis=1).transpose()
sim_sample_muts.columns = df.columns
sim_std = sim_sample_muts.apply(np.std)
sim_mean = sim_sample_muts.apply(np.mean)
sim_sample_muts.head()


I am working on the file "My+own+KNN+Classifier.ipynb".
The first task is:
4. The Real Deal : Building (Coding up) the classifier
The details are explained in comment form line-by-line in the following section.
Can you write Python code for it?

# First we implement a class. Classes let you structure code in a specific way.(source --> https://learnpythonthehardway.org/book/ex40.html)

class OneNeighborClassifier():                # This 'class' has 2 Methods : Fit and Predict

    #Each step is followed by a comment which explains how the classifier is working

    def fit(self, X_train, y_train):          # Takes features and labels as input
        self.X_train = X_train                # Storing the X_train in self.X_train
        self.y_train = y_train                # Storing the y_train in self.y_train
                                              # This is like the ML classifier will memorize the values

    def predict (self, X_test):               # Receives features from the testing data and returns predictions
        predictions = []                      # List of predictions, since X_test is a 2D array or a list of lists.
        for row in X_test:                    # Each row contains the features for one testing example
            label = self.closest(row)         # We are calling the function that we are creating in the next block
                                              # to find the closest training point from the test point
            predictions.append(label)         # Add the labels to the predictions list to fill it.
        return predictions                    # Return predictions as the output

    def closest(self, row):                   # Create the function closest such that -->
        best_dist = euc(row, self.X_train[0]) # Measure the shortest distance a test point and the first train point
        best_index = 0                        # Keep track of the index of the train point that is closest
        for i in range (1, len(self.X_train)):# Iterate over the different training points
            dist = euc(row, self.X_train[i])
            if dist < best_dist:              # The moment we find a closer one, we update our variables.
                best_dist = dist              # If dist is...


I am working on the file "My+own+KNN+Classifier.ipynb".
The first task is:
5. Final Steps
The classifier is built to utilize the standard pipeline that we use in scikit learn i.e. :

Call the classifier
Fit the model to train it
Predict the model on the test set
Check for accuracy between the real values and the predicted values
Can you write Python code for it?

my_classifier = OneNeighborClassifier()
my_classifier.fit(X_train, y_train)

pred = my_classifier.predict(X_test)

from sklearn.metrics import accuracy_score
print ('Accuracy of the classifier is', accuracy_score(y_test, pred)*100, '%')


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Inception Model
The inception model used for this implementation is inception5h because this model works with any image size and the output is more beautiful as compared to Inception v3.
Can you write Python code for it?


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Layers in Inception Model used for this implementation : 12
Can you write Python code for it?


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Load the Inception Model
Can you write Python code for it?


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Normalize an image so its values are between 0.0 and 1.0. This is useful for plotting the gradient.
Can you write Python code for it?

def normalize_image(x):
    # Get the min and max values for all pixels in the input.
    x_min = x.min()
    x_max = x.max()

    # Normalize so all values are between 0.0 and 1.0
    x_norm = (x - x_min) / (x_max - x_min)

    return x_norm


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Now using the 3rd Layer (layer index = 2) of the Inception Model on the input image
The layer_tensor will hold the inception model 3rd layer and shows that it has 192 channels
Can you write Python code for it?

layer_tensor = model.layer_tensors[2]
layer_tensor


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Optimize Image
This is an Optimization that runs in a loop which forms a main part of DeepDream algorithm. It calculates the gradient of the given layer of Inception Model with respect to the input image which is then added to the input image. This increases the mean value of the layer-tensor and this process is repeated a number of times which helps in amplifying the patterns which the Inception Model sees in the input image.
Can you write Python code for it?

def optimize_image(layer_tensor, image, gradient,
                   num_iterations=10, step_size=3.0, tile_size=400,
                   show_gradient=True, filename='test'):
    """
    Use gradient ascent to optimize an image so it maximizes the
    mean value of the given layer_tensor.

    Parameters:
    layer_tensor: Reference to a tensor that will be maximized.
    image: Input image used as the starting point.
    num_iterations: Number of optimization iterations to perform.
    step_size: Scale for each step of the gradient ascent.
    tile_size: Size of the tiles when calculating the gradient.
    show_gradient: Plot the gradient in each iteration.
    """

    # Copy the image so we don't overwrite the original image.
    img = image.copy()

    print("Image before:")
    plot_image(img)

    # save the file showing the before image
    filename1 = 'images/deepdream_BeforeO_'+filename+'.jpg'

    # kruti sharme - uncomment the below line to save intermediate results
    #save_image(img,filename=filename1)

    print("Processing image: ", end="")

    #kruti sharma - the below function is called outside optimize function now. This is called only once for each tensor layer.
    # Use TensorFlow to get the mathematical function for the
    # gradient of the given layer-tensor with regard to the
    # input image. This may cause TensorFlow to add the same
    # math-expressions to the graph each time this function is called.

    #gradient = model.get_gradient(layer_tensor)

    for i in range(num_iterations):
        # Calculate the value of the gradient.
        # This tells us how to change the image so as to
        # maximize the mean of the given layer-tensor.
        grad = tiled_gradient(gradient=gradient, image=img)

        # Blur the gradient with different amounts and add
        # them together. The blur amount is also increased
        # during the optimization. This was found to give
        # nice, smooth images. You can try and change the...


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Plot the gradient after normalizing the image
Can you write Python code for it?

def plot_gradient(gradient):
    # Normalize the gradient so it is between 0.0 and 1.0
    gradient_normalized = normalize_image(gradient)

    # Plot the normalized gradient.
    plt.imshow(gradient_normalized, interpolation='bilinear')
    plt.show()


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Plot the image using the PIL since matplotlib gives low resolution images.
Can you write Python code for it?

def plot_image(image):
    # Assume the pixel-values are scaled between 0 and 255.

    if False:
        # Convert the pixel-values to the range between 0.0 and 1.0
        image = np.clip(image/255.0, 0.0, 1.0)

        # Plot using matplotlib.
        plt.imshow(image, interpolation='lanczos')
        plt.show()
    else:
        # Ensure the pixel-values are between 0 and 255.
        image = np.clip(image, 0.0, 255.0)

        # Convert pixels to bytes.
        image = image.astype(np.uint8)

        # Convert to a PIL-image and display it.
        display(PIL.Image.fromarray(image))


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Resize the image : this function resizes the image to the desired pixels or to the rescaling factor.
Can you write Python code for it?

def resize_image(image, size=None, factor=None):
    # If a rescaling-factor is provided then use it.
    if factor is not None:
        # Scale the numpy array's shape for height and width.
        size = np.array(image.shape[0:2]) * factor

        # The size is floating-point because it was scaled.
        # PIL requires the size to be integers.
        size = size.astype(int)
    else:
        # Ensure the size has length 2.
        size = size[0:2]

    # The height and width is reversed in numpy vs. PIL.
    size = tuple(reversed(size))

    # Ensure the pixel-values are between 0 and 255.
    img = np.clip(image, 0.0, 255.0)

    # Convert the pixels to 8-bit bytes.
    img = img.astype(np.uint8)

    # Create PIL-object from numpy array.
    img = PIL.Image.fromarray(img)

    # Resize the image.
    img_resized = img.resize(size, PIL.Image.LANCZOS)

    # Convert 8-bit pixel values back to floating-point.
    img_resized = np.float32(img_resized)

    # print(img_resized)

    return img_resized


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Running the DeepDream Optimization algorithm with iterations as 10, step size as 6.0.
Can you write Python code for it?

gradient = call_get_gradient(layer_tensor)
img_result = optimize_image(layer_tensor, image, gradient,
                            num_iterations=20, step_size=3.0, tile_size=400,
                            show_gradient=False, filename=filename)

def process_inputs():

    print('Tensor Layer to be Used: '+layer_tensor_ip)
    new_layer_tensor_ip = model.layer_tensors[int(layer_tensor_ip)]

    print('*************************************************')
    print('layer tensor actual value after input from user: ')
    print(new_layer_tensor_ip)

    print('*************************************************')

    if image_ip == "":
        image_value = 'willy_wonka_new.jpg'

    filename_ip = 'images/'+image_ip
    new_image_ip = load_image(filename_ip)
    print('New Input image from user')
    print('*************************************************')
    plot_image(new_image_ip)
    print('*************************************************')

    print('Step Size: '+step_size_ip)
    print('*************************************************')

    print('Rescale factor: '+rescale_factor_ip)
    print('*************************************************')

    print('Number of Iterations: '+num_iterations_ip)
    print('*************************************************')

    print('Number of Repeats: '+num_repeats_ip)
    print('*************************************************')

    print('*************** PROCESSING with Optimize Image **********************')

    parts = image_ip.split('.')
    inputImage = parts[0]
    print('New input image: ',inputImage)

    # calling the gradient function outside the optimize_image() function - to reduce the memory consumption
    gradient = call_get_gradient(new_layer_tensor_ip)

    img_result = optimize_image(new_layer_tensor_ip, new_image_ip, gradient,
                                num_iterations=int(num_iterations_ip), step_size=float(step_size_ip), tile_size=400,
                            ...


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
TensorFlow session to see all the outputs for the image.
Can you write Python code for it?


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
Test the algorithm for Willu Wonka Old image.
Can you write Python code for it?

image = load_image('images/willy_wonka_old.jpg')
filename = 'willy_wonka_old'
plot_image(image)


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
The Inception Model can accept image of any size, but this may require more RAM for processing. In order to get the results from the DeepDream algorithm, if we downscale the image directly to 200*200 pixels (on which the model is actually trained) this will result in an image in which the patterns may not be clearly visible. Thus this algorithm splits the image into smaller tiles and then use TensorFlow to calculate gradient for each of the tiles.
Below function is used to determine the appropritate tile size. The desired tile-size default value = 400*400 pixels and the actual tile-size depends on the image-dimensions.
Can you write Python code for it?

def get_tile_size(num_pixels, tile_size=400):
    """
    num_pixels is the number of pixels in a dimension of the image.
    tile_size is the desired tile-size.
    """

    # How many times can we repeat a tile of the desired size.
    num_tiles = int(round(num_pixels / tile_size))

    # Ensure that there is at least 1 tile.
    num_tiles = max(1, num_tiles)

    # The actual tile-size.
    actual_tile_size = math.ceil(num_pixels / num_tiles)

    return actual_tile_size


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
This function calculates the gradient for an input image. The input image is split into tiles and the gradient is calculated for each of the tile. The tiles are chosen randomly - this is to avoid visible lines in the final output image from DeepDream.
Can you write Python code for it?

def tiled_gradient(gradient, image, tile_size=400):
    # Allocate an array for the gradient of the entire image.
    grad = np.zeros_like(image)

    # Number of pixels for the x- and y-axes.
    x_max, y_max, _ = image.shape

    # Tile-size for the x-axis.
    x_tile_size = get_tile_size(num_pixels=x_max, tile_size=tile_size)
    # 1/4 of the tile-size.
    x_tile_size4 = x_tile_size // 4

    # Tile-size for the y-axis.
    y_tile_size = get_tile_size(num_pixels=y_max, tile_size=tile_size)
    # 1/4 of the tile-size
    y_tile_size4 = y_tile_size // 4

    # Random start-position for the tiles on the x-axis.
    # The random value is between -3/4 and -1/4 of the tile-size.
    # This is so the border-tiles are at least 1/4 of the tile-size,
    # otherwise the tiles may be too small which creates noisy gradients.
    x_start = random.randint(-3*x_tile_size4, -x_tile_size4)

    while x_start < x_max:
        # End-position for the current tile.
        x_end = x_start + x_tile_size

        # Ensure the tile's start- and end-positions are valid.
        x_start_lim = max(x_start, 0)
        x_end_lim = min(x_end, x_max)

        # Random start-position for the tiles on the y-axis.
        # The random value is between -3/4 and -1/4 of the tile-size.
        y_start = random.randint(-3*y_tile_size4, -y_tile_size4)

        while y_start < y_max:
            # End-position for the current tile.
            y_end = y_start + y_tile_size

            # Ensure the tile's start- and end-positions are valid.
            y_start_lim = max(y_start, 0)
            y_end_lim = min(y_end, y_max)

            # Get the image-tile.
            img_tile = image[x_start_lim:x_end_lim,
                             y_start_lim:y_end_lim, :]

            # Create a feed-dict with the image-tile.
            feed_dict = model.create_feed_dict(image=img_tile)

            # Use TensorFlow to calculate the gradient-value.
            g = session.run(gradient, feed_dict=feed_dict)

            # Normalize the gradient...


I am working on the file "MyDeepDreamCode.ipynb".
The first task is:
This function loads an image and return its numpy array of floating-points
Can you write Python code for it?

def load_image(imageFileName):
    image = PIL.Image.open(imageFileName)

    return np.float32(image)

img = load_image('images/elon_musk_100x100.jpg')
# print(img)


I am working on the file "My_own_tiny_similarity_and_plagiarism_detector_in_Python_scikit-learn.ipynb".
The first task is:
And then filter also:
Can you write Python code for it?

def filter_plagiarism(sorted_data, threshold=0.70):
    return [ scoren1n2 for scoren1n2 in sorted_data if scoren1n2[-1] >= threshold ]

for n1, n2, score in filter_plagiarism(sort_plagiarism(s_vectors)):
    name1 = n1.replace(f'.{extension}', '')[:5]
    name2 = n2.replace(f'.{extension}', '')[:5]
    print(f"Files {name1} and {name2} have similarity = {score:.2%}")


I am working on the file "My_own_tiny_similarity_and_plagiarism_detector_in_Python_scikit-learn.ipynb".
The first task is:
Now let's sort by increasing similarity index:
Can you write Python code for it?

def sort_plagiarism(s_vectors):
    data_plagiarism = check_plagiarism(s_vectors)
    sorted_data = sorted(data_plagiarism, key=lambda n1n2score: n1n2score[::-1])
    return sorted_data


I am working on the file "NA_Resampling.ipynb".
The first task is:
Functions to transform the scale of model parameters
Can you write Python code for it?

def log_df(df):
    pnames = p_ranges[p_ranges['log']].index
    pnames_log = ["log({})".format(p) for p in pnames]

    df_log = df.copy()
    df_log.loc[:, pnames] = np.log(df.loc[:, pnames])
    df_log.rename(columns=dict(zip(pnames, pnames_log)),
                  inplace=True)

    return df_log

def delog_df(df_log):
    pnames = p_ranges[p_ranges['log']].index
    pnames_log = ["log({})".format(p) for p in pnames]

    df = df_log.copy()
    df.loc[:, pnames_log] = np.exp(df_log.loc[:, pnames_log])
    df.rename(columns=dict(zip(pnames_log, pnames)),
              inplace=True)

    return df


I am working on the file "NA_Resampling.ipynb".
The first task is:
Initialize the nearest neighbour interpolator from the original inversion ensemble. It forms the NA approximation of the misfit.
Can you write Python code for it?

orig_ensemble_log = log_df(orig_ensemble)

na_approx = NearestNDInterpolator(
    orig_ensemble_log.ix[:, 0:len(p_ranges)].values,
    #orig_ensemble_log['misfit'].values
    orig_ensemble_log['RMSD'].values * orig_ensemble_log['D_mean'].values
)


I am working on the file "NA_Resampling.ipynb".
The first task is:
Plot the trace of the MCMC iterations.
Can you write Python code for it?

axes = mcmc_samples[:].plot(
    kind='line', subplots=True,
    figsize=(10, 12), color=clr_plt[0]
)

mcmc_kept_samples = mcmc_samples[:]

def jointplot_density(xcol, ycol):
    p = sns.jointplot(
        xcol, ycol,
        data=mcmc_kept_samples,
        xlim=(mcmc_kept_samples[xcol].min(),
              mcmc_kept_samples[xcol].max()),
        ylim=(mcmc_kept_samples[ycol].min(),
              mcmc_kept_samples[ycol].max()),
        joint_kws={'alpha': 0.02}
    )

jointplot_density('log(P_0)', 'log(K_g)')

def jointplot_density_i(xcol, ycol):
    p = sns.jointplot(
        xcol, ycol,
        data=init_walkers,
        xlim=(init_walkers[xcol].min(),
              init_walkers[xcol].max()),
        ylim=(init_walkers[ycol].min(),
              init_walkers[ycol].max()),
    )

jointplot_density_i('log(P_0)', 'log(K_g)')


I am working on the file "NA_Resampling.ipynb".
The first task is:
Set the log-likelihood: the negative of the misfit. Values are calculated from the NA approximation initialized above.
Can you write Python code for it?


I am working on the file "NA_Resampling.ipynb".
The first task is:
Set the log-posterior.
Can you write Python code for it?

def lnprob(m):
    lp = lnprior(m)
    if not np.isfinite(lp):
        return -np.inf
    return lp + lnlike(m)


I am working on the file "NA_Resampling.ipynb".
The first task is:
Set the log-prior: a uniform probability density within the bounded parameter search space
Can you write Python code for it?

p_ranges_log = p_ranges.drop('log', axis=1)
islog = p_ranges['log'] == True
p_ranges_log['min'][islog] = np.log(p_ranges_log['min'][islog])
p_ranges_log['max'][islog] = np.log(p_ranges_log['max'][islog])

p_ranges_log['prior'] = [
    stats.uniform(loc=pmin, scale=pmax - pmin)
    for pmin, pmax in p_ranges_log.values
]

def lnprior(m):
    lps = [p.logpdf(v)
           for (p, v) in zip(p_ranges_log['prior'], m)]
    if not np.all(np.isfinite(lps)):
        return -np.inf
    return np.sum(lps)


I am working on the file "NA_Resampling.ipynb".
The first task is:
Setup of the Affine-Invariant Ensemble Sampler: set the number and initial positions of the independent walkers. The initial positions are given by the models of lowest misfit after the 1st NA iteration in the original ensemble.
Can you write Python code for it?

n_walkers = 200

orig_ensemble_iter0 = orig_ensemble.loc[orig_ensemble['NA_iter'] == 0]
sort_misift_idx = orig_ensemble_iter0.sort('misfit').index
init_idx = sort_misift_idx[:n_walkers]

init_walkers = log_df(orig_ensemble_iter0).ix[init_idx, 0:len(p_ranges)]

# uniform random initial positions
uniform_pos = np.column_stack([
    p.rvs(size=n_walkers) for p in p_ranges_log['prior']
])
init_walkers = pd.DataFrame(data=uniform_pos, columns=init_walkers.columns)


I am working on the file "NB1_IdentifyProblem+DataClean.ipynb".
The first task is:
From the above results, from the 32, variables,column id number 1 is an integer, diagnosis 569 non-null object. and rest are float. More on python variables
Can you write Python code for it?

#check for missing variables
#data.isnull().any()

data.diagnosis.unique()


I am working on the file "NB1_IdentifyProblem+DataClean.ipynb".
The first task is:
In the result displayed, you can see the data has 569 records, each with 32 columns.
The “info()” method provides a concise summary of the data; from the output, it provides the type of data in each column, the number of non-null values in each column, and how much memory the data frame is using.
The method get_dtype_counts() will return the number of columns of each type in a DataFrame:
Can you write Python code for it?

# Review data types with "info()".
data.info()

# Review number of columns of each data type in a DataFrame:
data.get_dtype_counts()


I am working on the file "NB1_IdentifyProblem+DataClean.ipynb".
The first task is:
You can check the number of cases, as well as the number of fields, using the shape method, as shown below.
Can you write Python code for it?

# Id column is redundant and not useful, we want to drop it
data.drop('id', axis =1, inplace=True)
#data.drop('Unnamed: 0', axis=1, inplace=True)
data.head(2)

data.shape


I am working on the file "NB2_ExploratoryDataAnalysis.ipynb".
The first task is:
Box plot "_se" suffix designition
Can you write Python code for it?

# box and whisker plots
#plt=data_se.plot(kind= 'box' , subplots=True, layout=(4,4), sharex=False, sharey=False,fontsize=12)


I am working on the file "NB2_ExploratoryDataAnalysis.ipynb".
The first task is:
Density plot "_worst" suffix designition
Can you write Python code for it?

#Density Plots
#plt = data_worst.plot(kind= 'kde', subplots=True, layout=(4,3), sharex=False, sharey=False,fontsize=5,
#                     figsize=(15,10))


I am working on the file "NB2_ExploratoryDataAnalysis.ipynb".
The first task is:
Histogram for  the "_se" suffix designition
Can you write Python code for it?

#Plot histograms of _se variables
#hist_se=data_se.hist(bins=10, figsize=(15, 10),grid=False,)


I am working on the file "NB2_ExploratoryDataAnalysis.ipynb".
The first task is:
Observation:
We can see strong positive relationship exists with mean values paramaters between 1-0.75;.
* The mean area of the tissue nucleus has a strong positive correlation with mean values of radius and parameter;
* Some paramters are moderately positive corrlated (r between 0.5-0.75)are concavity and area, concavity and perimeter etc
* Likewise, we see some strong negative correlation between fractal_dimension with radius, texture, parameter mean values.
Can you write Python code for it?

plt.style.use('fivethirtyeight')
sns.set_style("white")

data = pd.read_csv('data/clean-data.csv', index_col=False)
g = sns.PairGrid(data[[data.columns[1],data.columns[2],data.columns[3],
                     data.columns[4], data.columns[5],data.columns[6]]],hue='diagnosis' )
g = g.map_diag(plt.hist)
g = g.map_offdiag(plt.scatter, s = 3)


I am working on the file "NB2_ExploratoryDataAnalysis.ipynb".
The first task is:
The skew result show a positive (right) or negative (left) skew. Values closer to zero show less skew.
 From the graphs, we can see that radius_mean, perimeter_mean, area_mean, concavity_mean and concave_points_mean are useful in predicting cancer type due to the distinct grouping between malignant and benign cancer types in these features. We can also see that area_worst and perimeter_worst are also quite useful.
Can you write Python code for it?

data.diagnosis.unique()

# Group by diagnosis and review the output.
diag_gr = data.groupby('diagnosis', axis=0)
pd.DataFrame(diag_gr.size(), columns=['# of observations'])


I am working on the file "NB3_DataPreprocesing.ipynb".
The first task is:
After encoding the class labels(diagnosis) in an array y, the malignant tumors are now represented as class 1(i.e prescence of cancer cells) and the benign tumors are represented as class 0 (i.e no cancer cells detection), respectively, illustrated by calling the transform method of LabelEncorder on two dummy variables.**

Assesing Model Accuracy: Split data into training and test sets
The simplest method to evaluate the performance of a machine learning algorithm is to use different training and testing datasets. Here I will
* Split the available data into a training set and a testing set. (70% training, 30% test)
* Train the algorithm on the first part,
* make predictions on the second part and
* evaluate the predictions against the expected results.
The size of the split can depend on the size and specifics of your dataset, although it is common to use 67% of the data for training and the remaining 33% for testing.
Can you write Python code for it?

from sklearn.model_selection import train_test_split

##Split data set in train 70% and test 30%
X_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.25, random_state=7)
X_train.shape, y_train.shape, X_test.shape, y_test.shape


I am working on the file "NB3_DataPreprocesing.ipynb".
The first task is:
Feature Standardization

Standardization is a useful technique to transform attributes with a Gaussian distribution and differing means and standard deviations to a standard Gaussian distribution with a mean of 0 and a standard deviation of 1.

As seen in NB2_Exploratory data analysis the raw data has differing distributions which may have an impact on the most ML algorithms. Most machine learning and optimization algorithms behave much better if features are on the same scale.

Let’s evaluate the same algorithms with a standardized copy of the dataset. Here, I use sklearn to scale and transform the data such that each attribute has a mean value of zero and a standard deviation of one
Can you write Python code for it?

from sklearn.preprocessing import StandardScaler

# Normalize the  data (center around 0 and scale to remove the variance).
scaler =StandardScaler()
Xs = scaler.fit_transform(X)


I am working on the file "NB3_DataPreprocesing.ipynb".
The first task is:
Feature decomposition using Principal Component Analysis(  PCA)
From the pair plot in NB2, lot of feature pairs divide nicely the data to a similar extent, therefore, it makes sense to use one of the dimensionality reduction methods to try to use as many features as possible and maintian as much information as possible when working with only 2 dimensions. I will use PCA
Can you write Python code for it?

from sklearn.decomposition import PCA
# feature extraction
pca = PCA(n_components=10)
fit = pca.fit(Xs)

# summarize components
#print("Explained Variance: %s") % fit.explained_variance_ratio_
#print(fit.components_)

X_pca = pca.transform(Xs)

PCA_df = pd.DataFrame()

PCA_df['PCA_1'] = X_pca[:,0]
PCA_df['PCA_2'] = X_pca[:,1]

plt.plot(PCA_df['PCA_1'][data.diagnosis == 'M'],PCA_df['PCA_2'][data.diagnosis == 'M'],'o', alpha = 0.7, color = 'r')
plt.plot(PCA_df['PCA_1'][data.diagnosis == 'B'],PCA_df['PCA_2'][data.diagnosis == 'B'],'o', alpha = 0.7, color = 'b')

plt.xlabel('PCA_1')
plt.ylabel('PCA_2')
plt.legend(['Malignant','Benign'])
plt.show()


I am working on the file "NB3_DataPreprocesing.ipynb".
The first task is:
Label encoding
Here, I assign the 30 features to a NumPy array X, and transform the class labels from their original string representation (M and B) into integers
Can you write Python code for it?

#Assign predictors to a variable of ndarray (matrix) type
array = data.values
X = array[:,1:31]
y = array[:,0]

#transform the class labels from their original string representation (M and B) into integers
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y = le.fit_transform(y)

#Call the transform method of LabelEncorder on two dummy variables
#le.transform (['M', 'B'])


I am working on the file "NB3_DataPreprocesing.ipynb".
The first task is:
Now, what we got after applying the linear PCA transformation is a lower dimensional subspace (from 3D to 2D in this case), where the samples are “most spread” along the new feature axes.
Can you write Python code for it?

#The amount of variance that each PC explains
var= pca.explained_variance_ratio_
#Cumulative Variance explains
#var1=np.cumsum(np.round(pca.explained_variance_ratio_, decimals=4)*100)
#print(var1)


I am working on the file "NB4_PredictiveModelUsingSVM.ipynb".
The first task is:
Classification with cross-validation
As discussed in notebook NB3 splitting the data into test and training sets is crucial to avoid overfitting. This allows generalization of real, previously-unseen data. Cross-validation extends this idea further. Instead of having a single train/test split, we specify so-called folds so that the data is divided into similarly-sized folds.

Training occurs by taking all folds except one – referred to as the holdout sample.

On the completion of the training, you test the performance of your fitted model using the holdout sample.

The holdout sample is then thrown back with the rest of the other folds, and a different fold is pulled out as the new holdout sample.

Training is repeated again with the remaining folds and we measure performance using the holdout sample. This process is repeated until each fold has had a chance to be a test or holdout sample.

The expected performance of the classifier, called cross-validation error, is then simply an average of error rates computed on each holdout sample.

This process is demonstrated by first performing a standard train/test split, and then computing cross-validation error.
Can you write Python code for it?

# 5. Divide records in training and testing sets.
X_train, X_test, y_train, y_test = train_test_split(Xs, y, test_size=0.3, random_state=2, stratify=y)

# 6. Create an SVM classifier and train it on 70% of the data set.
clf = SVC(probability=True)
clf.fit(X_train, y_train)

 #7. Analyze accuracy of predictions on 30% of the holdout test sample.
classifier_score = clf.score(X_test, y_test)
print '\nThe classifier accuracy score is {:03.2f}\n'.format(classifier_score)


I am working on the file "NB4_PredictiveModelUsingSVM.ipynb".
The first task is:
Observation
There are two possible predicted classes: "1" and "0". Malignant = 1 (indicates prescence of cancer cells) and Benign
= 0 (indicates abscence).

The classifier made a total of 174 predictions (i.e 174 patients were being tested for the presence breast cancer).
Out of those 174 cases, the classifier predicted "yes" 58 times, and "no" 113 times.
In reality, 64 patients in the sample have the disease, and 107 patients do not.

Rates as computed from the confusion matrix

Accuracy: Overall, how often is the classifier correct?

(TP+TN)/total = (57+106)/171 = 0.95

Misclassification Rate: Overall, how often is it wrong?

(FP+FN)/total = (1+7)/171 = 0.05 equivalent to 1 minus Accuracy also known as "Error Rate"

True Positive Rate: When it's actually yes, how often does it predict 1?

TP/actual yes = 57/64 = 0.89 also known as "Sensitivity" or "Recall"

False Positive Rate: When it's actually 0, how often does it predict 1?

FP/actual no = 1/107 = 0.01

Specificity: When it's actually 0, how often does it predict 0? also know as true positive rate

TN/actual no = 106/107 = 0.99 equivalent to 1 minus False Positive Rate

Precision: When it predicts 1, how often is it correct?

TP/predicted yes = 57/58 = 0.98

Prevalence: How often does the yes condition actually occur in our sample?

actual yes/total = 64/171 = 0.34
Can you write Python code for it?

from sklearn.metrics import roc_curve, auc
# Plot the receiver operating characteristic curve (ROC).
plt.figure(figsize=(10,8))
probas_ = clf.predict_proba(X_test)
fpr, tpr, thresholds = roc_curve(y_test, probas_[:, 1])
roc_auc = auc(fpr, tpr)
plt.plot(fpr, tpr, lw=1, label='ROC fold (area = %0.2f)' % (roc_auc))
plt.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), label='Random')
plt.xlim([-0.05, 1.05])
plt.ylim([-0.05, 1.05])
plt.xlabel('False Positive Rate')
plt.ylabel('True Positive Rate')
plt.title('Receiver operating characteristic example')
plt.axes().set_aspect(1)


I am working on the file "NB4_PredictiveModelUsingSVM.ipynb".
The first task is:
The above evaluations were based on using the entire set of features. You will now employ the correlation-based feature selection strategy to assess the effect of using 3 features which have the best correlation with the class labels.
Can you write Python code for it?

from sklearn.feature_selection import SelectKBest, f_regression
clf2 = make_pipeline(SelectKBest(f_regression, k=3),SVC(probability=True))

scores = cross_val_score(clf2, Xs, y, cv=3)

# Get average of 3-fold cross-validation score using an SVC estimator.
n_folds = 3
cv_error = np.average(cross_val_score(SVC(), Xs, y, cv=n_folds))
print '\nThe {}-fold cross-validation accuracy score for this classifier is {:.2f}\n'.format(n_folds, cv_error)

print scores
avg = (100*np.mean(scores), 100*np.std(scores)/np.sqrt(scores.shape[0]))
print "Average score and uncertainty: (%.2f +- %.3f)%%"%avg


I am working on the file "NB5 OptimizingSVMClassifier.ipynb".
The first task is:
Build a predictive model and evaluate with 5-cross validation using support vector classifies (ref NB4) for details
Can you write Python code for it?

X

data = pd.read_csv('data/clean-data.csv', index_col=False)
data.drop('Unnamed: 0',axis=1, inplace=True)

#Assign predictors to a variable of ndarray (matrix) type
array = data.values
X = array[:,1:31]
y = array[:,0]

#transform the class labels from their original string representation (M and B) into integers
le = LabelEncoder()
y = le.fit_transform(y)

# Normalize the  data (center around 0 and scale to remove the variance).
scaler =StandardScaler()
Xs = scaler.fit_transform(X)

from sklearn.decomposition import PCA
# feature extraction
pca = PCA(n_components=10)
fit = pca.fit(Xs)
X_pca = pca.transform(Xs)

# 5. Divide records in training and testing sets.
X_train, X_test, y_train, y_test = train_test_split(X_pca, y, test_size=0.3, random_state=2, stratify=y)

# 6. Create an SVM classifier and train it on 70% of the data set.
clf = SVC(probability=True)
clf.fit(X_train, y_train)

 #7. Analyze accuracy of predictions on 30% of the holdout test sample.
classifier_score = clf.score(X_test, y_test)
print ('\nThe classifier accuracy score is {:03.2f}\n'.format(classifier_score))

clf2 = make_pipeline(SelectKBest(f_regression, k=3),SVC(probability=True))
scores = cross_val_score(clf2, X_pca, y, cv=3)

# Get average of 5-fold cross-validation score using an SVC estimator.
n_folds = 5
cv_error = np.average(cross_val_score(SVC(), X_pca, y, cv=n_folds))
#print ('\nThe {}-fold cross-validation accuracy score for this classifier is {:.2f}\n'.format(n_folds, cv_error))

y_pred = clf.fit(X_train, y_train).predict(X_test)
cm = metrics.confusion_matrix(y_test, y_pred)

print(classification_report(y_test, y_pred ))

fig, ax = plt.subplots(figsize=(5, 5))
ax.matshow(cm, cmap=plt.cm.Reds, alpha=0.3)
for i in range(cm.shape[0]):
     for j in range(cm.shape[1]):
         ax.text(x=j, y=i,
                s=cm[i, j],
                va='center', ha='center')
plt.xlabel('Predicted Values', )
plt.ylabel('Actual Values')
plt.show()


I am working on the file "NB5 OptimizingSVMClassifier.ipynb".
The first task is:
Decision boundaries of different classifiers
Let's see the decision boundaries produced by the linear, Gaussian and polynomial classifiers.
Can you write Python code for it?

import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn import svm, datasets

def decision_plot(X_train, y_train, n_neighbors, weights):
       h = .02  # step size in the mesh

Xtrain = X_train[:, :2] # we only take the first two features.

#================================================================
# Create color maps
#================================================================
cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])

#================================================================
# we create an instance of SVM and fit out data.
# We do not scale ourdata since we want to plot the support vectors
#================================================================

C = 1.0  # SVM regularization parameter

svm = SVC(kernel='linear', random_state=0, gamma=0.1, C=C).fit(Xtrain, y_train)
rbf_svc = SVC(kernel='rbf', gamma=0.7, C=C).fit(Xtrain, y_train)
poly_svc = SVC(kernel='poly', degree=3, C=C).fit(Xtrain, y_train)

%matplotlib inline
plt.rcParams['figure.figsize'] = (15, 9)
plt.rcParams['axes.titlesize'] = 'large'

    # create a mesh to plot in
x_min, x_max = Xtrain[:, 0].min() - 1, Xtrain[:, 0].max() + 1
y_min, y_max = Xtrain[:, 1].min() - 1, Xtrain[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, 0.1),
                         np.arange(y_min, y_max, 0.1))

# title for the plots
titles = ['SVC with linear kernel',
          'SVC with RBF kernel',
          'SVC with polynomial (degree 3) kernel']

for i, clf in enumerate((svm, rbf_svc, poly_svc)):
    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    plt.subplot(2, 2, i + 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)

    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    plt.contourf(xx, yy, Z, cmap=plt.cm.coolwarm, alpha=0.8)

    # Plot also the training...


I am working on the file "NB6 Comparison between different classifiers.ipynb".
The first task is:
Evaluate Some Algorithms
Now it is time to create some models of the data and estimate their accuracy on unseen data. Here is what we are going to cover in this step:
1. Separate out a validation dataset.
2. Setup the test harness to use 10-fold cross validation.
3. Build 5 different models
4. Select the best model
1.0 Validation Dataset
Can you write Python code for it?

#load data
data = pd.read_csv('data/clean-data.csv', index_col=False)
data.drop('Unnamed: 0',axis=1, inplace=True)

# Split-out validation dataset
array = data.values
X = array[:,1:31]
y = array[:,0]

# Divide records in training and testing sets.
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=7)

#transform the class labels from their original string representation (M and B) into integers
le = LabelEncoder()
y = le.fit_transform(y)


I am working on the file "NB6 Comparison between different classifiers.ipynb".
The first task is:
Finalize Model
Can you write Python code for it?

#Use best parameters
clf_svc = gs.best_estimator_

#Get Final Scores
clf_svc.fit(X_train, y_train)
scores = cross_val_score(estimator=clf_svc,
                         X=X_train,
                         y=y_train,
                         cv=10,
                         n_jobs=1)
print('--> Final Model Training Accuracy: %.3f +/- %.3f' %(np.mean(scores), np.std(scores)))

print('--> Final Accuracy on Test set: %.5f' % clf_svc.score(X_test,y_test))

clf_svc.fit(X_train, y_train)
y_pred = clf_svc.predict(X_test)

print(accuracy_score(y_test, y_pred))
print(confusion_matrix(y_test, y_pred))
print(classification_report(y_test, y_pred))


I am working on the file "NB6 Comparison between different classifiers.ipynb".
The first task is:
Observation

The results show a similar tight distribution for all classifiers except SVM which is encouraging, suggesting low variance. The poor results for SVM are surprising.
It is possible the varied distribution of the attributes may have an effect on the accuracy of algorithms such as SVM. In the next section we will repeat this spot-check with a standardized copy of the training dataset.

2.1 Evaluate Algorithms: Standardize Data
Can you write Python code for it?

# Standardize the dataset
pipelines = []
pipelines.append(( 'ScaledLR' , Pipeline([( 'Scaler' , StandardScaler()),( 'LR' ,
    LogisticRegression())])))
pipelines.append(( 'ScaledLDA' , Pipeline([( 'Scaler' , StandardScaler()),( 'LDA' ,
    LinearDiscriminantAnalysis())])))
pipelines.append(( 'ScaledKNN' , Pipeline([( 'Scaler' , StandardScaler()),( 'KNN' ,
    KNeighborsClassifier())])))
pipelines.append(( 'ScaledCART' , Pipeline([( 'Scaler' , StandardScaler()),( 'CART' ,
    DecisionTreeClassifier())])))
pipelines.append(( 'ScaledNB' , Pipeline([( 'Scaler' , StandardScaler()),( 'NB' ,
    GaussianNB())])))
pipelines.append(( 'ScaledSVM' , Pipeline([( 'Scaler' , StandardScaler()),( 'SVM' , SVC())])))

results = []
names = []
for name, model in pipelines:
  kfold = KFold(n=num_instances, n_folds=num_folds, random_state=seed)
  cv_results = cross_val_score(model, X_train, y_train, cv=kfold,
      scoring=scoring)
  results.append(cv_results)
  names.append(name)
  msg = "%s: %f (%f)" % (name, cv_results.mean(), cv_results.std())
  print(msg)

# Compare Algorithms
fig = plt.figure()
fig.suptitle( 'Scaled Algorithm Comparison' )
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()


I am working on the file "NB6 Comparison between different classifiers.ipynb".
The first task is:
Observations

The results show that standardization of the data has lifted the skill of SVM to be the most accurate algorithm tested so far.

The results suggest digging deeper into the SVM and LDA and LR algorithms. It is very likely that configuration beyond the default may yield even more accurate models.
3.0 Algorithm Tuning
In this section we investigate tuning the parameters for three algorithms that show promise from the spot-checking in the previous section: LR, LDA and SVM.
Tuning hyper-parameters - SVC estimator
Can you write Python code for it?

#Make Support Vector Classifier Pipeline
pipe_svc = Pipeline([('scl', StandardScaler()),
                     ('pca', PCA(n_components=2)),
                     ('clf', SVC(probability=True, verbose=False))])

#Fit Pipeline to training Data
pipe_svc.fit(X_train, y_train)

#print('--> Fitted Pipeline to training Data')

scores = cross_val_score(estimator=pipe_svc, X=X_train, y=y_train, cv=10, n_jobs=1, verbose=0)
print('--> Model Training Accuracy: %.3f +/- %.3f' %(np.mean(scores), np.std(scores)))

#Tune Hyperparameters
param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
param_grid = [{'clf__C': param_range,'clf__kernel': ['linear']},
              {'clf__C': param_range,'clf__gamma': param_range,
               'clf__kernel': ['rbf']}]
gs_svc = GridSearchCV(estimator=pipe_svc,
                  param_grid=param_grid,
                  scoring='accuracy',
                  cv=10,
                  n_jobs=1)
gs_svc = gs_svc.fit(X_train, y_train)
print('--> Tuned Parameters Best Score: ',gs.best_score_)
print('--> Best Parameters: \n',gs.best_params_)


I am working on the file "NB6 Comparison between different classifiers.ipynb".
The first task is:
Tuning the hyper-parameters - k-NN hyperparameters
For your standard k-NN implementation, there are two primary hyperparameters that you’ll want to tune:

The number of neighbors k.
The distance metric/similarity function.

Both of these values can dramatically affect the accuracy of your k-NN classifier. Grid object is ready to do 10-fold cross validation on a KNN model using classification accuracy as the evaluation metric
In addition, there is a parameter grid to repeat the 10-fold cross validation process 30 times
Each time, the n_neighbors parameter should be given a different value from the list
We can't give GridSearchCV just a list
We've to specify n_neighbors should take on 1 through 30
You can set n_jobs = -1 to run computations in parallel (if supported by your computer and OS)
Can you write Python code for it?

from sklearn.neighbors import KNeighborsClassifier as KNN

pipe_knn = Pipeline([('scl', StandardScaler()),
                     ('pca', PCA(n_components=2)),
                     ('clf', KNeighborsClassifier())])

#Fit Pipeline to training Data
pipe_knn.fit(X_train, y_train)

scores = cross_val_score(estimator=pipe_knn,
                         X=X_train,
                         y=y_train,
                         cv=10,
                         n_jobs=1)
print('--> Model Training Accuracy: %.3f +/- %.3f' %(np.mean(scores), np.std(scores)))

#Tune Hyperparameters
param_range = range(1, 31)
param_grid = [{'clf__n_neighbors': param_range}]
# instantiate the grid
grid = GridSearchCV(estimator=pipe_knn,
                    param_grid=param_grid,
                    cv=10,
                    scoring='accuracy')
gs_knn = grid.fit(X_train, y_train)
print('--> Tuned Parameters Best Score: ',gs.best_score_)
print('--> Best Parameters: \n',gs.best_params_)


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Are there different ways to print the first n rows of something? Sure! Try using break, which will stop a for loop from running. E.g. the code:
python
for idx in xrange(100):
    if idx > 10:
        break
...will stop the for loop after 10 iterations.
Next, write a for loop that can access and print every row.
Can you write Python code for it?

## Write the Python to print *every* row of the CSV here.


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Fetching the Data
You have a couple of options of fetching the data set to begin your analysis:

Click on the link above and Download the file.
Write a Python function that automatically downloads the data as a comma-separated value file (CSV) and writes it to disk.

In either case, you'll have to be cognizant of where the CSV file lands. Here is a quick implementation of a function to download a URL at a file and write it to disk. Note the many approaches to do this as outlined here: How do I download a file over HTTP using Python?.
Can you write Python code for it?

def download(url, path):
    """
    Downloads a URL and writes it to the specified path. The "path"
    is like the mailing address for the file - it tells the function
    where on your computer to send it!

    Also note the use of "with" to automatically close files - this
    is a good standard practice to follow.
    """
    response = urllib2.urlopen(url)
    with open(path, 'w') as f:
        f.write(response.read())

    response.close()


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Keep playing with the above function to get it to work more efficiently or to reduce bad data in the computation - e.g. what are all those zero salaries?
Visualization
Congratulations if you've made it this far! It's time for the bonus round!
You've now had some summary statistics about the salaries of NBA players, but what we're really interested in is the relationship between SALARY and the rest of the fields in the data set. The PER - Player Efficiency Rating, is an aggregate score of all performance statistics; therefore if we determine the relationship of PER to SALARY, we might learn a lot about how to model NBA salaries.
In order to explore this, let's create a scatter plot of SALARY to PER, where each point is an NBA player.
Visualization is going to require a third party library. You probably already have matplotlib, so that might be the simplest if you're having trouble with installation. If you don't, pip install it now! Follow the documentation to create the scatter plot inline in the notebook in the following cells.
Can you write Python code for it?

# Insert your Python to create the visualization here
import os
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
%matplotlib inline  # Makes the plot appear inline in your iPython Notebook.

def read_data(path):
    # Pandas is an efficient way to wrangle the data quickly
    return pd.DataFrame(pd.read_csv(path))

def graph_data(path, xkey='PER', ykey='SALARY'):
    data = read_data(path)
    ## Fill this in yourself!
    plt.show()

graph_data('fixtures/nba_players.csv') # Or whatever your path is


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Next step: The read_csv function "returns" a generator. How can we access just the first row? Remember how to access the next row of a generator?
Can you write Python code for it?

## Now write the Python to print the first row of the CSV here.


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Nice work! Now... calculating the mode is a bit different. Remember about the Decorate-Sort-Undecorate pattern that we learned about in ThinkPython? That will work here!
Can you write Python code for it?

## Write the Python to get the mode of the salaries.


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
The "DSU" approach is a little inefficient. Instead of using a dictionary as our data type to solve the mode problem, we could use counter() from the Collections module. Read more about counter() and try it out here:
Can you write Python code for it?

## Experiment with using counter() here.


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Your turn: use the above function to download the data!
Can you write Python code for it?

## Write the Python to execute the function and download the file here:


I am working on the file "NBA Player Statistics Workshop.ipynb".
The first task is:
Your turn: use the above function to open the file and print out the first row of the CSV!
To do this, you'll need to do three things:
First, remember where you told the download function to store your file? Pass that same path into read_csv:
Can you write Python code for it?

## Write the Python to execute our read_csv function.


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
A second file, players.csv, contains static information for each player, as height, weight, etc.
Can you write Python code for it?

players = pd.read_csv(r'data/players.csv', index_col=0)
players.head(10)


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
Curry is labeled as a point guards (his real position) from 1973 until today, and as a shooting guard before that. Perhaps because of his heigh (191cm), or perhaps because he is too much of a scorer. Bob Pettit is labeled as a center until 1967, and as a power forward after that (he played both roles, but nowadays he would have difficulties to play as a center, and would be for sure a forwards, perhaps even a small forward).
Changing positions
Many players go towards more interior roles with age, as they lose velocity. We will follow two cases, Magic Johnson, and Michael Jordan. Both of the retire, and return years later with more interior roles.
Can you write Python code for it?

magic = data[(data['Player'] == 'Magic Johnson')]
jordan = data[(data['Player'] == 'Michael Jordan')]

# Magic
X = magic.drop(['Player', 'Pos', 'G', 'GS', 'MP'], axis=1).as_matrix()
y = magic['Pos'].as_matrix()

y_cat = encoder.transform(y)
Xnorm = scaler.transform(X)

magic_pred = pd.DataFrame(index=magic.loc[:, 'Age'].values,
                                data={'Real': magic.loc[:, 'Pos'].values,
    'Predicted':encoder.inverse_transform(model.predict(Xnorm))})

# Jordan
X = jordan.drop(['Player', 'Pos', 'G', 'GS', 'MP'], axis=1).as_matrix()
y = jordan['Pos'].as_matrix()

y_cat = encoder.transform(y)
Xnorm = scaler.transform(X)

jordan_pred = pd.DataFrame(index=jordan.loc[:, 'Age'].values,
                                data={'Real': jordan.loc[:, 'Pos'].values,
    'Predicted':encoder.inverse_transform(model.predict(Xnorm))})

pd.concat([magic_pred,jordan_pred],axis=1,keys=['Magic Johnson','Michael Jordan'])


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
Data preparation
We will use the Kaggle dataset "NBA Players stats since 1950", with stats for all players since 1950. We will take special interest in how the pass of time affects to the position of each player, and the definition of the positions themselves (a Small Forward, for example, was absolutely different in the 60's than now)
Can you write Python code for it?

stats = pd.read_csv(r'data/Seasons_Stats.csv', index_col=0)


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
Neural network training
We build using Keras (with Tensorflow as beckend) a neural network with two hidden layers. We will use relu activations, except for the last one, where we use a softmax to properly obtain the label probability. We will use a 20% of the data as a validation set, to make sure we are not overfitting.
Can you write Python code for it?

model = Sequential()
model.add(Dense(40, activation='relu', input_dim=46))
model.add(Dropout(0.5))
model.add(Dense(30, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(nlabels, activation='softmax'))

model.compile(loss='categorical_crossentropy', optimizer='sgd', metrics=['accuracy'])

# x_train and y_train are Numpy arrays --just like in the Scikit-Learn API.
model.fit(X_train, y_train, epochs=200, batch_size=128, validation_split=0.2, verbose=1)

model.test_on_batch(X_test, y_test, sample_weight=None)


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
The file Seasons_Stats.csv contains the statics of all players since 1950. First, we drop a couple of blank columns, and the "Tm" column, that contains the team.
Can you write Python code for it?

stats = pd.read_csv(r'data/Seasons_Stats.csv', index_col=0)
stats_clean = stats.drop(['blanl', 'blank2', 'Tm'], axis=1)

stats_clean.head()


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
The model gets right four of the five. It's even more interesting that the one that gets wrong, Anthony Davis, can play in both PF and C positions, and that in the last season, he played more as a Power Forward than as a Center, as the model predicts:
New Orleans Pelicans Depth Chart - 2016-17.
Predicting the positions of the NBA MVP
We will use now the model to predict the positions of all the NBA MVP since the creation of the award, in 1956.
Can you write Python code for it?

mvp = [(1956, 'Bob Pettit'), (1957, 'Bob Cousy'), (1958, 'Bill Russell'), (1959, 'Bob Pettit'),
(1960, 'Wilt Chamberlain'), (1961, 'Bill Russell'), (1962, 'Bill Russell'), (1963, 'Bill Russell'),
(1964, 'Oscar Robertson'), (1965, 'Bill Russell'), (1966, 'Wilt Chamberlain'), (1967, 'Wilt Chamberlain'),
(1968, 'Wilt Chamberlain'), (1969, 'Wes Unseld'), (1970, 'Willis Reed'), (1971, 'Lew Alcindor'),
(1972, 'Kareem Abdul-Jabbar'), (1973, 'Dave Cowens'), (19704, 'Kareem Abdul-Jabbar'), (1975, 'Bob McAdoo'),
(1976, 'Kareem Abdul-Jabbar'), (1977, 'Kareem Abdul-Jabbar'), (1978, 'Bill Walton'), (1979, 'Moses Malone'),
(1980, 'Kareem Abdul-Jabbar'), (1981, 'Julius Erving'), (1982, 'Moses Malone'), (1983, 'Moses Malone'),
(1984, 'Larry Bird'), (1985, 'Larry Bird'), (1986, 'Larry Bird'), (1987, 'Magic Johnson'),
(1988, 'Michael Jordan'), (1989, 'Magic Johnson'), (1990, 'Magic Johnson'), (1991, 'Michael Jordan'),
(1992, 'Michael Jordan'), (1993, 'Charles Barkley'), (1994, 'Hakeem Olajuwon'), (1995, 'David Robinson'),
(1996, 'Michael Jordan'), (1997, 'Karl Malone'), (1998, 'Michael Jordan'), (1999, 'Karl Malone'),
(2000, 'Shaquille O\'Neal'), (2001, 'Allen Iverson'), (2002, 'Tim Duncan'), (2003, 'Tim Duncan'),
(2004, 'Kevin Garnett'), (2005, 'Steve Nash'), (2006, 'Steve Nash'), (2007, 'Dirk Nowitzki'),
(2008, 'Kobe Bryant'), (2009, 'LeBron James'), (2010, 'LeBron James'), (2011, 'Derrick Rose'),
(2012, 'LeBron James'), (2013, 'LeBron James'), (2014, 'Kevin Durant'), (2015, 'Stephen Curry'),
(2016, 'Stephen Curry')]

mvp_stats = pd.concat([data[(data['Player'] == x[1]) & (data['Year']==x[0])] for x in mvp], axis=0)

mvp_stats

mvp_pred = pd.DataFrame(index=mvp_stats.loc[:, 'Player'].values, data={'Real': mvp_stats.loc[:, 'Pos'].values,
    'Predicted':encoder.inverse_transform(model.predict(Xnorm[mvp_stats.index, :]))})

mvp_pred


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
The model gets right most of the players, and the errors are always for a contiguous position (it is interesting that the model does it without having been provided with any information about the distances between the labels.)
Does year metter?
The definition of what a forward, or a center are always changing (in the very recent years, there is, for example, a trend towards having scoring point guards (as Stephen Curry) and forwards that direct the game instead of the guard (as Lebron James).
Also, the physical requirements are increasing, and a height that in the 50's could characterize you as a center you will be a forward today.
We will follow the first and last MVP's, Stephen Curry and Bob Pettit, and see where our model will put them in different years in the NBA history.
Can you write Python code for it?

curry2017 = data[(data['Player'] == 'Stephen Curry') & (data['Year']==2017)]
pettit1956 = data[(data['Player'] == 'Bob Pettit') & (data['Year']==1956)]

time_travel_curry = pd.concat([curry2017 for year in range(1956, 2018)], axis=0)
time_travel_curry['Year'] = range(1956, 2018)

X = time_travel_curry.drop(['Player', 'Pos', 'G', 'GS', 'MP'], axis=1).as_matrix()
y = time_travel_curry['Pos'].as_matrix()

y_cat = encoder.transform(y)
Xnorm = scaler.transform(X)

time_travel_curry_pred = pd.DataFrame(index=time_travel_curry.loc[:, 'Year'].values,
                                data={'Real': time_travel_curry.loc[:, 'Pos'].values,
    'Predicted':encoder.inverse_transform(model.predict(Xnorm))})

time_travel_pettit = pd.concat([pettit1956 for year in range(1956, 2018)], axis=0)
time_travel_pettit['Year'] = range(1956, 2018)

X = time_travel_pettit.drop(['Player', 'Pos', 'G', 'GS', 'MP'], axis=1).as_matrix()
y = time_travel_pettit['Pos'].as_matrix()

y_cat = encoder.transform(y)
Xnorm = scaler.transform(X)

time_travel_pettit_pred = pd.DataFrame(index=time_travel_pettit.loc[:, 'Year'].values,
                                data={'Real': time_travel_pettit.loc[:, 'Pos'].values,
    'Predicted':encoder.inverse_transform(model.predict(Xnorm))})

pd.concat([time_travel_curry_pred,time_travel_pettit_pred],axis=1,keys=['Stephen Curry','Bob Pettit'])


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
The model performs well both for the validation and the test sets (65% might not seem a lot, but it is satisfying enough for our problem, where all the labels are very subjective (Was Larry Bird a "SM-PF" or a "PF-SF"? Nobody can tell).
Now we train again the model, using all the training data (we will still reserve the 2017 season out of the training).
Can you write Python code for it?

# Production model, using all data
model.fit(X_train, y_train, epochs=200, batch_size=128, validation_split=0, verbose=1)


I am working on the file "NBA_Keras/.ipynb_checkpoints/Predicting NBA players positions using Keras-checkpoint.ipynb".
The first task is:
We will train a neural network with this data, to try to predict the position of each player.
A way we didn't follow was to transform the positions into numbers from 1 to 5 (1 for a PG, 2 for a SG, 1.5 for a PG-SG, and so on, until 5 for a C), and use the network for regression instead of classification. But we wanted to see if the network was able to predict labels as "SG-PF", so we decided to work with the categorical labels. Another reason is that this makes this study more easily portable to other areas.
We convert our DataFrame into a matrix X with the inputs, and a vector y with the labels. We scale the inputs and encode the outputs into dummy variables using the corresponding sklearn utilities.
Instead of a stochastic partition, we decided to use the 2017 season as our test data, and all the previous as the train set.
Can you write Python code for it?

X = data.drop(['Player', 'Pos', 'G', 'GS', 'MP'], axis=1).as_matrix()
y = data['Pos'].as_matrix()

encoder = LabelBinarizer()
y_cat = encoder.fit_transform(y)
nlabels = len(encoder.classes_)

scaler =StandardScaler()
Xnorm = scaler.fit_transform(X)

stats2017 = (data['Year'] == 2017)
X_train = Xnorm[~stats2017]
y_train = y_cat[~stats2017]
X_test = Xnorm[stats2017]
y_test = y_cat[stats2017]


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
A second file, players.csv, contains static information for each player, as height, weight, etc.
Can you write Python code for it?


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
Curry is labeled as a point guard (his real position) from 1973 until today, and as a shooting guard before that. Perhaps because of his heigh (191cm), or perhaps because he is too much of a scorer. Bob Pettit is labeled as a center until 1967, and as a power forward after that (he played both roles, but nowadays he would have difficulties to play as a center, and would be for sure a forward, perhaps even a small forward).
Changing positions
Many players go towards more interior roles with age, as they lose velocity. We will follow two cases, Magic Johnson, and Michael Jordan. Both of them retired, and returned years later with more interior roles.
Can you write Python code for it?


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
Data preparation
We will use the Kaggle dataset "NBA Players stats since 1950", with stats for all players since 1950. We will take special interest in how the pass of time affects to the position of each player, and the definition of the positions themselves (a Small Forward, for example, was absolutely different in the 60's to what it is now)
Can you write Python code for it?


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
The file Seasons_Stats.csv contains the statics of all players since 1950. First, we drop a couple of blank columns, and the "Tm" column, that contains the team.
Can you write Python code for it?


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
The model gets right four of the five. It's even more interesting that the one that gets wrong, Anthony Davis, can play in both PF and C positions, and that in the last season, he played more as a Power Forward than as a Center, as the model predicts:
New Orleans Pelicans Depth Chart - 2016-17.
Predicting the positions of the NBA MVP
We will use now the model to predict the positions of all the NBA MVP since the creation of the award, in 1956.
Can you write Python code for it?


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
The model is able to detect the conversion of Jordan into a forward at the end of his career, but not the return of Magic as a power forward. Also, in his rookie season, he is classified as a small forward instead of as a shooting guard (Magic was clearly and outlier in the data, a 205cm point guard who could easily play in the five positions. It is even surprising that is properly labelled as a point guard during most of his career)
How important are height and weight?
A concern we have before training the model was that it would use the height and weight as the main classifiers, and that it would label incorrectly players as Magic Johnson (a 205 cm point guard), or Charles Barkley (a 196cm power forward). Almost surprisingly, it works properly on this two players.
We will use again the 2017 First NBA Team and play with the heights and weights of the players. Keeping constant all other statistics, we will change the height and weight and observe how the predicted positions change.
Can you write Python code for it?

first_team_stats

multiplier = np.arange(0.8,1.2,0.02)
growing_predicted = []

for p in first_team_stats.iterrows():
    growing = pd.concat([p[1].to_frame().T for x in multiplier], axis=0)
    growing['height'] = growing['height'] * multiplier
    growing['weight'] = growing['weight'] * (multiplier ** 3)

    X = growing.drop(['Player', 'Pos', 'G', 'GS', 'MP'], axis=1).as_matrix()
    y = growing['Pos'].as_matrix()

    y_cat = encoder.transform(y)
    Xnorm = scaler.transform(X)

    growing_predicted.append(pd.DataFrame(index=multiplier, data={'height': growing.loc[:, 'height'].values,
            'Real': growing.loc[:, 'Pos'].values, 'Predicted':encoder.inverse_transform(model.predict(Xnorm))}))

pd.concat(growing_predicted,axis=1,keys=first_team_stats['Player'])


I am working on the file "NBA_Keras/Predicting NBA players positions using Keras.ipynb".
The first task is:
We merge both tables, and do some data cleaning:

Keep only players with more than 400 minutes for each season (with a 82 games regular season, thats around 5 minutes per game. Players with less than that will be only anecdotical, and will distort the analysis).
Replace the * sign in some of the names
For the stats that represent total values (others, as TS%, represent percentages), we will take the values per 36 minutes. The reason is to judge every player according to his characteristics, not the time he was on the floor.
Can you write Python code for it?

data = pd.merge(stats_clean, players[['Player', 'height', 'weight']], left_on='Player', right_on='Player', right_index=False,
      how='left', sort=False).fillna(value=0)
data = data[~(data['Pos']==0) & (data['MP'] > 200)]
data.reset_index(inplace=True, drop=True)
data['Player'] = data['Player'].str.replace('*','')

totals = ['PER', 'OWS', 'DWS', 'WS', 'OBPM', 'DBPM', 'BPM', 'VORP', 'FG', 'FGA', '3P', '3PA', '2P', '2PA', 'FT', 'FTA',
         'ORB', 'DRB', 'TRB', 'AST', 'STL', 'BLK', 'TOV', 'PF', 'PTS']

for col in totals:
    data[col] = 36 * data[col] / data['MP']

data.tail()


I am working on the file "NBA_stats_visualization/.ipynb_checkpoints/Shot charts Notebook-checkpoint.ipynb".
The first task is:
Creating some Shot Charts
Now plot our properly adjusted shot chart data along with the court. We can adjust
the x-values in two ways. We can either pass in the the negative inverse of LOC_X to
plt.scatter or we can pass in descending values to plt.xlim . We'll do the latter to plot
our shot chart.
Can you write Python code for it?

plt.figure(figsize=(12,11))
plt.scatter(shot_df.LOC_X, shot_df.LOC_Y)
draw_court(outer_lines=True)
# Descending values along the axis from left to right
plt.xlim(300,-300)
plt.show()


I am working on the file "NBA_stats_visualization/.ipynb_checkpoints/Shot charts Notebook-checkpoint.ipynb".
The first task is:
Getting the data
Getting the data from stats.nba.com is pretty straightforward. While there isn't a a public API provided by the NBA,
we can actually access the API that the NBA uses for stats.nba.com using the requests library.
This blog post
by Greg Reda does a great job on explaining how to access this API (or finding an API to any web app for that matter).
Can you write Python code for it?

playerID='2200'

shot_chart_url ='http://stats.nba.com/stats/shotchartdetail?CFID=33&CFPARAMS=2015-16&' \
'ContextFilter=&ContextMeasure=FGA&DateFrom=&DateTo=&GameID=&GameSegment=&LastNGames=0&' \
'LeagueID=00&Location=&MeasureType=Base&Month=0&OpponentTeamID=0&Outcome=&PaceAdjust=N&' \
'PerMode=PerGame&Period=0&PlayerID='+playerID+'&PlusMinus=N&Position=&Rank=N&RookieYear=&' \
'Season=2015-16&SeasonSegment=&SeasonType=Regular+Season&TeamID=0&VsConference=&' \
'VsDivision=&mode=Advanced&showDetails=0&showShots=1&showZones=0'

print(shot_chart_url)


I am working on the file "NBA_stats_visualization/.ipynb_checkpoints/Shot charts Notebook-checkpoint.ipynb".
The first task is:
Lets create a few shot charts using jointplot from seaborn .
Can you write Python code for it?

 # create our jointplot
joint_shot_chart = sns.jointplot(shot_df.LOC_X, shot_df.LOC_Y,
                    stat_func=None,kind='scatter', space=0, alpha=0.5)
joint_shot_chart.fig.set_size_inches(12,11)
# A joint plot has 3 Axes, the first one called ax_joint
# is the one we want to draw our court onto and adjust some other settings
ax = joint_shot_chart.ax_joint
draw_court(ax, outer_lines=True)
# Adjust the axis limits and orientation of the plot in order
# to plot half court, with the hoop by the top of the plot
ax.set_xlim(-250,250)
ax.set_ylim(422.5, -47.5)
# Get rid of axis labels and tick marks
ax.set_xlabel('')
ax.set_ylabel('')
ax.tick_params(labelbottom='off', labelleft='off')
# Add a title
ax.set_title('James Harden FGA \n2015-16 Reg. Season',
             y=1.2, fontsize=18)
# Add Data Scource and Author
authors="""Data Source: stats.nba.com
Author: Juan Ignacio Gil
Original code by Savvas Tjortjoglou (savvastjortjoglou.com)"""

ax.text(-250,460,authors,fontsize=12)
plt.show()


I am working on the file "NBA_stats_visualization/.ipynb_checkpoints/Shot charts Notebook-checkpoint.ipynb".
The first task is:
Please note that the above plot misrepresents the data. The x-axis values are the inverse
of what they actually should be. Lets plot the shots taken from only the right side to see
this issue.
Can you write Python code for it?

right = shot_df[shot_df.SHOT_ZONE_AREA == "Right Side(R)"]
plt.figure(figsize=(12,11))
plt.scatter(right.LOC_X, right.LOC_Y)
plt.xlim(-300,300)
plt.ylim(-100,500)
plt.show()


I am working on the file "NBA_stats_visualization/.ipynb_checkpoints/Shot charts Notebook-checkpoint.ipynb".
The first task is:
The above url sends us to the JSON file contatining the data we want.
Also note that the url contains the various API parameters used to access the data.
The PlayerID parameter in the url is set to 201935, which is James Harden's PlayerID.
Now lets use requests to get the data we want
Can you write Python code for it?

# Get the webpage containing the data
response = requests.get(shot_chart_url)

# Grab the headers to be used as column headers for our DataFrame
headers = response.json()['resultSets'][0]['headers']
# Grab the shot chart data
shots = response.json()['resultSets'][0]['rowSet']


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
And another jointplot but with hexbins.
Can you write Python code for it?

# create our jointplot

cmap=plt.cm.gist_heat_r
joint_shot_chart = sns.jointplot(shot_df.LOC_X, shot_df.LOC_Y, stat_func=None,
                                 kind='hex', space=0, color=cmap(.2), cmap=cmap)

joint_shot_chart.fig.set_size_inches(12,11)

# A joint plot has 3 Axes, the first one called ax_joint
# is the one we want to draw our court onto
ax = joint_shot_chart.ax_joint
draw_court(ax)

# Adjust the axis limits and orientation of the plot in order
# to plot half court, with the hoop by the top of the plot
ax.set_xlim(-250,250)
ax.set_ylim(422.5, -47.5)

# Get rid of axis labels and tick marks
ax.set_xlabel('')
ax.set_ylabel('')
ax.tick_params(labelbottom='off', labelleft='off')

# Add a title
ax.set_title('FGA 2015-16 Reg. Season', y=1.2, fontsize=14)

# Add Data Source and Author
ax.text(-250,450,authors, fontsize=12)

# Add James Harden's image to the top right
img = OffsetImage(harden_pic, zoom=0.6)
img.set_offset((625,621))
ax.add_artist(img)

plt.show()


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
As we can see the shots in categorized as shots from the "Right Side(R)",
while to the viewers right, are actually to the left side of the hoop.
This is something we will need to fix when creating our final shot chart.
Drawing the Court
But first we need to figure out how to draw the court lines onto our plot. By looking at the first plot and
at the data we can roughly estimate that the center of the hoop is at the origin. We can also estimate that
every 10 units on either the x and y axes represents one foot. We can verify this by just look at the first
observation in our DataFrame . That shot was taken from the Right Corner 3 spot from a distance of 22 feet
with a LOC_X value of 226. So the shot was taken from about 22.6 feet to the right of the hoop. Now that we
know this we can actually draw the court onto our plot.
The dimensions of a basketball court can be seen here, and here.
Using those dimensions we can convert them to fit the scale of our plot and just draw them using
Matplotlib Patches. We'll be using and Arc objects to draw our court.
Now to create our function that draws our basketball court.
NOTE: While you can draw lines onto the plot using Lines2D I found it more convenient to use Rectangles (without a height or width) instead.
EDIT (Aug 4, 2015): I made a mistake in drawing the outerlines and the half court arcs. The outer courtlines height was changed from the incorrect value of 442.5 to 470. The y-values for the centers of the center court arcs were changed from 395 to 422.5. The ylim values for the plots were changed from (395, -47.5) to (422.5, -47.5)
Can you write Python code for it?

from matplotlib.patches import Circle, Rectangle, Arc
def draw_court(ax=None, color='black', lw=2, outer_lines=False):
    # If an axes object isn't provided to plot onto, just get current one
    if ax is None:
        ax = plt.gca()
    # Create the various parts of an NBA basketball court
    # Create the basketball hoop
    # Diameter of a hoop is 18" so it has a radius of 9", which is a value
    # 7.5 in our coordinate system
    hoop = Circle((0, 0), radius=7.5, linewidth=lw, color=color, fill=False)
    # Create backboard
    backboard = Rectangle((-30, -7.5), 60, -1, linewidth=lw, color=color)
    # The paint
    # Create the outer box 0f the paint, width=16ft, height=19ft
    outer_box = Rectangle((-80, -47.5), 160, 190, linewidth=lw, color=color,
                          fill=False)
    # Create the inner box of the paint, widt=12ft, height=19ft
    inner_box = Rectangle((-60, -47.5), 120, 190, linewidth=lw, color=color,
                          fill=False)
    # Create free throw top arc
    top_free_throw = Arc((0, 142.5), 120, 120, theta1=0, theta2=180,
                         linewidth=lw, color=color, fill=False)
    # Create free throw bottom arc
    bottom_free_throw = Arc((0, 142.5), 120, 120, theta1=180, theta2=0,
                            linewidth=lw, color=color, linestyle='dashed')
    # Restricted Zone, it is an arc with 4ft radius from center of the hoop
    restricted = Arc((0, 0), 80, 80, theta1=0, theta2=180, linewidth=lw,
                     color=color)
    # Three point line
    # Create the side 3pt lines, they are 14ft long before they begin to arc
    corner_three_a = Rectangle((-220, -47.5), 0, 140, linewidth=lw,
                               color=color)
    corner_three_b = Rectangle((220, -47.5), 0, 140, linewidth=lw, color=color)
    # 3pt arc - center of arc will be the hoop, arc is 23'9" away from hoop
    # I just played around with the theta values until they lined up with the
    # threes
    three_arc = Arc((0, 0), 475, 475, theta1=22, theta2=158,...


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Create a pandas DataFrame using the scraped shot chart data.
Can you write Python code for it?

shot_df = pd.DataFrame(shots, columns=headers)

# View the head of the DataFrame and all its columns
from IPython.display import display
with pd.option_context('display.max_columns', None):
    display(shot_df.head())


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Getting a Player's Image
We could also scrape Jame Harden's picture from stats.nba.com and place it on our plot.
We can find his image at this url.
To retrieve the image for our plot we can use urlretrieve from url.requests as follows:
Can you write Python code for it?

import urllib.request

# we pass in the link to the image as the 1st argument
# the 2nd argument tells urlretrieve what we want to scrape

pic = urllib.request.urlretrieve("http://stats.nba.com/media/players/230x185/"+playerID+".png",
                                 playerID+".png")

# urlretrieve returns a tuple with our image as the first
# element and imread reads in the image as a
# mutlidimensional numpy array so matplotlib can plot it
harden_pic = plt.imread(pic[0])
# plot the image
plt.imshow(harden_pic)
plt.show()


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Getting the data
Getting the data from stats.nba.com is pretty straightforward. While there isn't a a public API provided by the NBA,
we can actually access the API that the NBA uses for stats.nba.com using the requests library.
This blog post
by Greg Reda does a great job on explaining how to access this API (or finding an API to any web app for that matter).
Can you write Python code for it?


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Lets create a few shot charts using jointplot from seaborn .
Can you write Python code for it?


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Lets draw our court
Can you write Python code for it?

plt.figure(figsize=(12,11))
draw_court(outer_lines=True)
plt.xlim(-300,300)
plt.ylim(-100,500)
plt.show()


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Lets orient our shot chart with the hoop by the top of the chart, which is the same orientation as the shot charts on stats.nba.com. We do this by settting descending y-values from the bottom to the top of the y-axis. When we do this we no longer need to adjust the x-values of our plot.
Can you write Python code for it?

plt.figure(figsize=(12,11))
plt.scatter(shot_df.LOC_X, shot_df.LOC_Y)
draw_court(outer_lines=True)
# Adjust plot limits to just fit in half court
plt.xlim(-250,250)
# Descending values along th y axis from bottom to top
# in order to place the hoop by the top of plot
plt.ylim(422.5, -47.5)
# get rid of axis tick labels
plt.tick_params(labelbottom=False, labelleft=False)
plt.show()


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
Now to plot Harden's face on a jointplot we will import OffsetImage from matplotlib.
Offset, which will allow us to place the image at the top right corner of the plot.
So lets create our shot chart like we did above, but this time we will create a KDE jointplot and at the end add
on our image.
Can you write Python code for it?

from matplotlib.offsetbox import  OffsetImage

# create our jointplot
# get our colormap for the main kde plot
# Note we can extract a color from cmap to use for
# the plots that lie on the side and top axes
cmap=plt.cm.YlOrRd_r

# n_levels sets the number of contour lines for the main kde plot
joint_shot_chart = sns.jointplot(shot_df.LOC_X, shot_df.LOC_Y, stat_func=None,
                                 kind='kde', space=0, color=cmap(0.1),
                                 cmap=cmap, n_levels=50)
joint_shot_chart.fig.set_size_inches(12,11)
# A joint plot has 3 Axes, the first one called ax_joint
# is the one we want to draw our court onto and adjust some other settings
ax = joint_shot_chart.ax_joint
draw_court(ax,outer_lines=True)
# Adjust the axis limits and orientation of the plot in order
# to plot half court, with the hoop by the top of the plot
ax.set_xlim(-250,250)
ax.set_ylim(422.5, -47.5)
# Get rid of axis labels and tick marks
ax.set_xlabel('')
ax.set_ylabel('')
ax.tick_params(labelbottom='off', labelleft='off')
# Add a title
ax.set_title('James Harden FGA \n2015-16 Reg. Season',
             y=1.2, fontsize=18)
# Add Data Scource and Author
ax.text(-250,460,authors,fontsize=12)
# Add Harden's image to the top right
# First create our OffSetImage by passing in our image
# and set the zoom level to make the image small enough
# to fit on our plot
img = OffsetImage(harden_pic, zoom=0.6)
# Pass in a tuple of x,y coordinates to set_offset
# to place the plot where you want, I just played around
# with the values until I found a spot where I wanted
# the image to be
img.set_offset((625,621))
# add the image
ax.add_artist(img)
plt.show()


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
The above shot chart data contains all the the field goal attempts James Harden took during the 2014-15
regular season. They data we want is found in LOC_X and LOC_Y. These are coordinate values for each shot
attempt, which can then be plotted onto a set of axes that represent the basketball court.
Plotting the Shot Chart Data
Lets just quickly plot the data just too see how it looks.
Can you write Python code for it?

sns.set_style("white")
sns.set_color_codes()
plt.figure(figsize=(12,11))
plt.scatter(shot_df.LOC_X, shot_df.LOC_Y)
plt.show()


I am working on the file "NBA_stats_visualization/Shot charts Notebook.ipynb".
The first task is:
The above url sends us to the JSON file contatining the data we want.
Also note that the url contains the various API parameters used to access the data.
The PlayerID parameter in the url is set to 201935, which is James Harden's PlayerID.
Now lets use requests to get the data we want
Can you write Python code for it?


I am working on the file "NCDCWebServiceGenerateMonthlySummary.ipynb".
The first task is:
Let's manipulate start and end data a little so that we're able to pick out the normals corresponding to daily observations. Normals are formally provided with the year set to 2010.
Can you write Python code for it?

st = dt.datetime.strptime(startdate, "%Y-%m-%d")
ed = dt.datetime.strptime(enddate, "%Y-%m-%d")
normalsst = dt.datetime.strftime(dt.datetime(2010, st.month, st.day), "%Y-%m-%d")
normalsed = dt.datetime.strftime(dt.datetime(2010, ed.month, ed.day), "%Y-%m-%d")


I am working on the file "NCDCWebServiceGenerateMonthlySummary.ipynb".
The first task is:
Note on start and end date: This code is meant for beginning to end of a month. It should work for arbitrary periods, but if the end of February is enclosed in your interval, leap year offsets may have to be handled by the user and leap year related bugs fixed.
Can you write Python code for it?

baseurl = "https://www.ncdc.noaa.gov/cdo-web/api/v2/"
endpoint = "data"
custom_headers = {
    "token": credentials.token
}
datatypes = ["TAVG", "TMIN", "TMAX", "MDTX", 'MDTN', "PRCP", "SNOW"]
startdate = "2017-07-01"
enddate = "2017-07-31"
params = {
    "datatypeid": datatypes,
    "startdate": startdate,
    "enddate": enddate,
    "limit": 1000,
    "units": "standard",
}

STATIONFN = "AK_NCDC_FirstOrderStations.json"
with open(STATIONFN, "r") as source:
    stationdata = json.load(source)


I am working on the file "NCDCWebServiceGenerateMonthlySummary.ipynb".
The first task is:
Request 2: Daily data
Second, daily summaries.
Can you write Python code for it?

dataset = "GHCND"
fullbase = requests.compat.urljoin(baseurl, endpoint)


I am working on the file "NCDCWebServiceGetStations.ipynb".
The first task is:
Get information for all stations in list and write out to JSON file
Can you write Python code for it?

OUTFN = "AK_NCDC_FirstOrderStations.json"

SAVEDATA = False
stationdata = []
for station in all_stations:
    path = os.path.join(endpoint_stations, "GHCND:{}".format(station))
    fullbase = requests.compat.urljoin(baseurl, path)
    r = requests.get(
        fullbase,
        headers=custom_headers,
    )
    stationdata.append(json.loads(r.text))
if SAVEDATA:
    with open(OUTFN, "w") as fh:
        fh.write(json.dumps(stationdata, indent=2))

with open(OUTFN, "w") as fh:
    fh.write(json.dumps(stationdata, indent=2))


I am working on the file "NCDCWebServiceGetStations.ipynb".
The first task is:
Optional: Add short names and then save again
Can you write Python code for it?

[item["name"] for item in stationdata]

shortnamedic = {
    'FAIRBANKS INTERNATIONAL AIRPORT, AK US': "Fairbanks",
    'ANNETTE WEATHER SERVICE OFFICE AIRPORT, AK US': "Annette",
    'JUNEAU AIRPORT, AK US': "Juneau",
    'YAKUTAT AIRPORT, AK US': "Yakutat",
    'KODIAK AIRPORT, AK US': "Kodiak",
    'KING SALMON AIRPORT, AK US': "King Salmon",
    'HOMER AIRPORT, AK US': "Homer",
    'COLD BAY AIRPORT, AK US': "Cold Bay",
    'ANCHORAGE TED STEVENS INTERNATIONAL AIRPORT, AK US': "Anchorage",
    'ST PAUL ISLAND AIRPORT, AK US': "St. Paul",
    'BIG DELTA AIRPORT, AK US': "Big Delta",
    'GULKANA AIRPORT, AK US': "Gulkana",
    'VALDEZ WEATHER SERVICE OFFICE, AK US': "Valdez",
    'MCGRATH AIRPORT, AK US': "McGrath",
    'TALKEETNA AIRPORT, AK US': "Talkeetna",
    'BETTLES AIRPORT, AK US': "Bettles",
    'BETHEL AIRPORT, AK US': "Bethel",
    'KOTZEBUE RALPH WEIN MEMORIAL AIRPORT, AK ': "Kotzebue",
    'NOME MUNICIPAL AIRPORT, AK US': "Nome",
    'BARROW W POST W ROGERS AIRPORT, AK US': "Barrow (Utqiaġvik)"}

for item in stationdata:
    item['shortname'] = shortnamedic[item['name']]

with open(OUTFN, "w") as fh:
    fh.write(json.dumps(stationdata, indent=2))


I am working on the file "NCDCWebServiceTest.ipynb".
The first task is:
On a side note: Python integers that start with 0o (zero - lowercase o) are octal. 7 * 8 + 7 = 63.
Can you write Python code for it?


I am working on the file "NCDCWebServiceTest.ipynb".
The first task is:
Request 2: Data for a station
Can you write Python code for it?

fullbase = requests.compat.urljoin(baseurl, endpoint_data)
r = requests.get(
    fullbase,
    headers=custom_headers,
    params=params,
)

json.loads(r.text)

r.headers


I am working on the file "NCDCWebServiceTest.ipynb".
The first task is:
Request 3: Available datasets
Can you write Python code for it?

fullbase = requests.compat.urljoin(baseurl, endpoint_datasets)
r = requests.get(
    fullbase,
    headers=custom_headers,
)

json.loads(r.text)


I am working on the file "NCPSchool2021/Examples/CreateAHistogramPython.ipynb".
The first task is:
We can also fill a ROOT histogram from a numpy array.
Can you write Python code for it?

import numpy as np
x = np.random.normal(0.,2.,1000)
h.Reset()


I am working on the file "NCPSchool2021/Examples/CreateAHistogramPython.ipynb".
The first task is:
We now create our histogram
Can you write Python code for it?

h = ROOT.TH1D("h", "My Notebook Histo;x;#", 64, -4, 4)


I am working on the file "NCPSchool2021/Examples/CreateAHistogramPython.ipynb".
The first task is:
We now import the gauss generation from the random module of Python and fill the histogram
Can you write Python code for it?

from random import gauss
numbers = [gauss(0., 1.) for _ in range(1000)]
numbers

for i in numbers: h.Fill(i)


I am working on the file "NCPSchool2021/Examples/FittingExample.ipynb".
The first task is:
Construct the histogram containing the input data
Can you write Python code for it?


I am working on the file "NCPSchool2021/Examples/FittingExample.ipynb".
The first task is:
Create the function and try to fit it without setting any parameter
Note we are using Minuit2 as minimizer
Can you write Python code for it?

%jsroot on
c = ROOT.TCanvas()
nparams = 6
fitFcn = ROOT.TF1('fitFcn', ROOT.fitFunction, xlow, xup, nparams)

ROOT.Math.MinimizerOptions.SetDefaultMinimizer("Minuit2");

histo.Fit(fitFcn)
c.Draw()


I am working on the file "NCPSchool2021/Examples/FittingExample.ipynb".
The first task is:
Inject into the interpreter the functions.
Can you write Python code for it?

%%cpp -d
//Define functions for fitting
// Quadratic background function
double background(double *x, double *par) {
  return par[0] + par[1]*x[0] + par[2]*x[0]*x[0];
}

// Lorenzian Peak function
double lorentzianPeak(double *x, double *par) {
  return (0.5*par[0]*par[1]/TMath::Pi()) /
    TMath::Max(1.e-10,(x[0]-par[2])*(x[0]-par[2])
    + .25*par[1]*par[1]);
}

// Sum of background and peak function
double fitFunction(double *x, double *par) {
  return background(x, par) + lorentzianPeak(x, &par[3]);
}


I am working on the file "NCPSchool2021/Examples/FittingExample.ipynb".
The first task is:
Less than optimal. Set parameters and fit again, draw histogram with error bars
Can you write Python code for it?

fitFcn.SetParameter(4, 0.2); # width
fitFcn.SetParameter(5, 1);   # peak
histo.Fit(fitFcn)
histo.Draw('E')
c.Draw()


I am working on the file "NCPSchool2021/Examples/FittingExample.ipynb".
The first task is:
Much better. Now time to beautify the plot. Construct a TF1 for the background and Lorentzian functions and draw them in the same canvas.
We save the fit results and set the parameters of the functions accordingly
Can you write Python code for it?

pars = fitFcn.GetParameters()
backFcn = ROOT.TF1('backFcn', ROOT.background, xlow, xup, 3)
backFcn.SetLineColor(ROOT.kGreen)
backFcn.Draw('Same')
backFcn.SetParameters(pars[0], pars[1], pars[2])
c.Draw()

signalFcn = ROOT.TF1('signalFcn', ROOT.lorentzianPeak, xlow, xup, 3)
signalFcn.SetLineColor(ROOT.kBlue)
signalFcn.SetParameters(pars[3], pars[4], pars[5])
signalFcn.Draw('Same')
c.Draw()


I am working on the file "NCPSchool2021/Examples/FittingExample.ipynb".
The first task is:
We can now add a legend
Can you write Python code for it?

legend = ROOT.TLegend(0.45, 0.65, 0.73, 0.85)
legend.SetTextFont(72)
legend.SetTextSize(0.04)
legend.AddEntry(histo, 'Data', 'LE')
legend.AddEntry(backFcn, 'Background fit', 'L')
legend.AddEntry(signalFcn, 'Signal fit', 'L')
legend.AddEntry(fitFcn, 'Global Fit', 'L')
legend.Draw('Same')
c.Draw()


I am working on the file "NCPSchool2021/Examples/GraphDrawPython.ipynb".
The first task is:
Change marker style, colour as well as line colour and thickness. Make the plot interactive. Re-draw the plot and interact with it!
Can you write Python code for it?

%jsroot on
g.SetMarkerStyle(ROOT.kFullTriangleUp)
g.SetMarkerSize(3)
g.SetMarkerColor(ROOT.kAzure)
g.SetLineColor(ROOT.kRed - 2)
g.SetLineWidth(2)
g.SetLineStyle(3)
c.Draw()


I am working on the file "NCPSchool2021/Examples/GraphDrawPython.ipynb".
The first task is:
Now we set the title and the grid on the canvas.
Can you write Python code for it?

g.SetTitle("My Graph;The X;My Y")
c.SetGrid()
c.Draw()


I am working on the file "NCPSchool2021/Examples/GraphDrawPython.ipynb".
The first task is:
The simple graph
Can you write Python code for it?

g = ROOT.TGraph()
for i in range(5): g.SetPoint(i,i,i*i)
g.Draw("APL")
c.Draw()


I am working on the file "NCPSchool2021/Examples/GraphDrawPython.ipynb".
The first task is:
We will now add the symbol of the Calcium isotope
Can you write Python code for it?

txt = "#color[804]{My text #mu {}^{40}_{20}Ca}"
l = ROOT.TLatex(.2, 10, txt)
l.Draw()
c.Draw()


I am working on the file "NCPSchool2021/Examples/TMVA_CNN_Classification_py.ipynb".
The first task is:
Book Convolutional Neural Network in Keras using a generated model
Can you write Python code for it?

## to use tensorflow backend
import os
##os.environ["KERAS_BACKEND"] = "tensorflow"
import tensorflow as tf

from tensorflow.keras.models import Sequential
from tensorflow.keras.optimizers import Adam, SGD
from tensorflow.keras.initializers import TruncatedNormal
from tensorflow.keras.layers import Input, Dense, Dropout, Flatten, Conv2D, MaxPooling2D, Reshape, BatchNormalization

model = Sequential()
model.add(Reshape((8,8, 1), input_shape=(64,)))
model.add(Conv2D(10, kernel_size=(3,3), kernel_initializer='TruncatedNormal', activation='relu', padding='same' ) )
model.add(Conv2D(10, kernel_size=(3,3), kernel_initializer='TruncatedNormal', activation='relu', padding='same' ) )
#stride for maxpool is equal to pool size
model.add(MaxPooling2D(pool_size=(2, 2) ))
model.add(Flatten())
model.add(Dense(64, activation='tanh'))
#model.add(Dropout(0.2))
model.add(Dense(2, activation='sigmoid'))
model.compile(loss='binary_crossentropy', optimizer=Adam(lr=0.001), metrics=['accuracy'])
model.save('model_cnn.h5')
model.summary()

factory.BookMethod(loader, ROOT.TMVA.Types.kPyKeras,
                       "PyKeras","H:!V:VarTransform=None:FilenameModel=model_cnn.h5:"
                       "FilenameTrainedModel=trained_model_cnn.h5:NumEpochs=20:BatchSize=128");


I am working on the file "NCPSchool2021/Examples/TMVA_CNN_Classification_py.ipynb".
The first task is:
Book Convolutional Neural Network in TMVA
Can you write Python code for it?

#input layout
inputLayoutString = "InputLayout=1|8|8"

## Batch Layout
batchLayoutString = "BatchLayout=128|1|64"

layoutString = ("Layout=CONV|10|3|3|1|1|1|1|RELU,CONV|10|3|3|1|1|1|1|RELU,MAXPOOL|2|2|1|1,"
            "RESHAPE|FLAT,DENSE|64|TANH,DENSE|1|LINEAR")

##Training strategies.
training1 = ("LearningRate=1e-3,Momentum=0.9,Repetitions=1,"
                     "ConvergenceSteps=10,BatchSize=128,TestRepetitions=1,"
                     "MaxEpochs=20,WeightDecay=1e-4,Regularization=None,"
                     "Optimizer=ADAM,DropConfig=0.0+0.0+0.0+0.0")

trainingStrategyString = "TrainingStrategy=" + training1

## General Options.
cnnOptions = ("!H:V:ErrorStrategy=CROSSENTROPY:VarTransform=None:"
                       "WeightInitialization=XAVIERUNIFORM");

cnnOptions +=  ":" + inputLayoutString
cnnOptions +=  ":" + batchLayoutString
cnnOptions +=  ":" + layoutString
cnnOptions +=  ":" + trainingStrategyString
cnnOptions +=  ":Architecture=CPU"

##book CNN
factory.BookMethod(loader, ROOT.TMVA.Types.kDL, "DL_CNN", cnnOptions);