I am working on the file "Analysis/MSD_Analyzer/notebook.ipynb".
The first task is:
Confined motion (more work is needed here)
Simulate particle motion
Can you write Python code for it?

# Parameters
SPACE_UNITS = '\mu m'
TIME_UNITS = 's'

N_PARTICLES = 10
N_TIME_STEPS = 200
COORDS = ['x', 'y']
N_DIM = len(COORDS)

cm = plt.get_cmap('gist_rainbow')
COLORS = [cm(i/N_PARTICLES) for i in range(N_PARTICLES)]

# Typical values taken from studies of proteins diffusing in membranes:

# Diffusion coefficient
D  = 1e-3 # µm^2/s

# Time step between acquisition; fast acquisition!
dt = 0.05 # s

# Boltzman constant
kt = 4.2821e-21 # kBoltzman x T @ 37ºC

# Area size, just used to disperse particles in 2D. Has no impact on
# analysis
SIZE = 5 # µm

k = np.sqrt(2 * D * dt)

# Confined motion parameters

# Particle in a potential: settings the 'stiffness' of the energy potential
# Typical diameter of the trap (still in micron)
ltrap = 0.05 # µm
ktrap = kt / ltrap**2 # = thermal energy / trap size ^ 2

# Generate trajectories

def Fx(x, initial_position):
    return ktrap * (x - initial_position)

trajs = []

for i in range(N_PARTICLES):

    # Generate time vector
    time = np.arange(0, N_TIME_STEPS) * dt

    # Energy potential:
    #V = @(x) 0.5 * ktrap * sum (x .^ 2) # Unused, just to show

    p = np.zeros((N_TIME_STEPS, N_DIM))

    # Setup custom initial position
    initial_position = SIZE * np.random.rand(1, N_DIM)
    p[0] = initial_position

    for j in range(1, N_TIME_STEPS):
        dxtrap = D / kt * Fx(p[j-1], initial_position) * dt # ad hoc displacement
        dxbrownian = k * np.random.randn(1, N_DIM);

        p[j] = p[j-1] + dxtrap + dxbrownian

    # Convert to Dataframe
    p = pd.DataFrame({c: p[:, i] for i, c in enumerate(COORDS)})
    p['t'] = time

    trajs.append(p)

# Plot trajectories
fig, ax = plt.subplots()

for color, traj in zip(COLORS, trajs):
    traj.plot(x='x', y='y', color=color, ax=ax, legend=False)

ax.set_xlabel(COORDS[0])
ax.set_ylabel(COORDS[1])


I am working on the file "Analysis/MSD_Analyzer/notebook.ipynb".
The first task is:
Directed motion
Simulate particle motion
Can you write Python code for it?

# Parameters
SPACE_UNITS = '\mu m'
TIME_UNITS = 's'

N_PARTICLES = 10
N_TIME_STEPS = 100
COORDS = ['x', 'y']
N_DIM = len(COORDS)

cm = plt.get_cmap('gist_rainbow')
COLORS = [cm(i/N_PARTICLES) for i in range(N_PARTICLES)]

# Typical values taken from studies of proteins diffusing in membranes:

# Diffusion coefficient
D  = 1e-3 # µm^2/s

# Time step between acquisition; fast acquisition!
dt = 0.05 # s

# Mean velocity
vm = 0.05 # µm/s

# Area size, just used to disperse particles in 2D. Has no impact on
# analysis
SIZE = 2 # µm

# Generate trajectories

trajs = []

for i in range(N_PARTICLES):

    # Generate time vector
    time = np.arange(0, N_TIME_STEPS) * dt

    # Velocity orientation
    theta = 2 * np.pi * np.random.rand()

    # Mean velocity
    v = vm * (1 + 1/4 * np.random.randn())

    # Get random displacement
    dp = k * np.random.randn(N_TIME_STEPS, N_DIM)

    dp_brownian = k * np.random.randn(N_TIME_STEPS, N_DIM)
    dp_directed = v * dt * (np.cos(theta) * np.ones((N_TIME_STEPS, 1)) + np.sin(theta) * np.ones((N_TIME_STEPS, 1)))

    dp = dp_brownian + dp_directed

    # Setup custom initial position
    initial_position = SIZE * np.random.rand(1, N_DIM)
    dp[0] = initial_position

    # Get position
    p = np.cumsum(dp, axis=0)

    # Convert to Dataframe
    p = pd.DataFrame({c: p[:, i] for i, c in enumerate(COORDS)})
    p['t'] = time

    trajs.append(p)

# Plot trajectories
fig, ax = plt.subplots()

for color, traj in zip(COLORS, trajs):
    traj.plot(x='x', y='y', color=color, ax=ax, legend=False)

ax.set_xlabel(COORDS[0])
ax.set_ylabel(COORDS[1])


I am working on the file "Analysis/MSD_Analyzer/notebook.ipynb".
The first task is:
MSD analysis
Can you write Python code for it?

# Compute MSD
msd_mean, all_msd = compute_msd_mean(trajs, dt, N_TIME_STEPS, coords=COORDS)

# Fit model
def model(tau, D, v):
    return 2*D*N_DIM*tau + v*tau**2

clip_factor = 1
t_stamp = np.round(len(msd_mean) * clip_factor, 0)
(D, v), pcov = optimize.curve_fit(model, msd_mean.loc[:t_stamp, 'tau'], msd_mean.loc[:t_stamp, 'msds'])
print(D)
print(v)

# Plot all MSD

fig, ax = plt.subplots()

for color, msd in zip(COLORS, all_msd):
    msd.plot(x='tau', y='msds', color=color, ax=ax, legend=False)

ax.set_xlabel("Delay (${}$)".format(TIME_UNITS))
ax.set_ylabel("MSD (${}^2$)".format(SPACE_UNITS))

# Plot MSD mean

fig, ax = plt.subplots()

msd_mean.plot(x='tau', y='msds', color=color, ax=ax, legend=False)
ax.fill_between(msd_mean['tau'], msd_mean['msds'] - msd_mean['msds_sem'],
                msd_mean['msds'] + msd_mean['msds_sem'],  alpha=0.2)

ax.plot(msd_mean['tau'], model(msd_mean['tau'], D, v), color='red')

ax.set_xlabel("Delay (${}$)".format(TIME_UNITS))
ax.set_ylabel("MSD (${}^2$)em".format(SPACE_UNITS))


# Compute MSD
msd_mean, all_msd = compute_msd_mean(trajs, dt, N_TIME_STEPS, coords=COORDS)

# Plot all MSD

fig, ax = plt.subplots()

for color, msd in zip(COLORS, all_msd):
    msd.plot(x='tau', y='msds', color=color, ax=ax, legend=False)

ax.set_xlabel("Delay (${}$)".format(TIME_UNITS))
ax.set_ylabel("MSD (${}^2$)".format(SPACE_UNITS))


I am working on the file "Analysis/MSD_Analyzer/notebook.ipynb".
The first task is:
The Einstein equation tells us that displacements follow a Gaussian PDF with standard deviation given by :
Can you write Python code for it?

k = np.sqrt(2 * D * dt)
print(k)

# Generate trajectories

trajs = []

for i in range(N_PARTICLES):

    # Generate time vector
    time = np.arange(0, N_TIME_STEPS) * dt

    # Get random displacement
    dp = k * np.random.randn(N_TIME_STEPS, N_DIM)

    # Setup custom initial position
    initial_position = SIZE * np.random.rand(1, N_DIM)
    dp[0] = initial_position

    # Get position
    p = np.cumsum(dp, axis=0)

    # Convert to Dataframe
    p = pd.DataFrame({c: p[:, i] for i, c in enumerate(COORDS)})
    p['t'] = time

    trajs.append(p)

# Plot trajectories
fig, ax = plt.subplots()

for color, traj in zip(COLORS, trajs):
    traj.plot(x='x', y='y', color=color, ax=ax, legend=False)

ax.set_xlabel(COORDS[0])
ax.set_ylabel(COORDS[1])


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
Display all the MSD curves
Can you write Python code for it?

fig, ax = plt.subplots(figsize=(10, 8))
msds.plot(ax=ax, legend=False)

ax.set_xlabel('Delay (s)')
ax.set_ylabel('MSD ($\mu m^2.s^{-1}$)')


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
Display the average MSD (with std and sem)
Can you write Python code for it?

msd_mean = msds.mean(axis=1)
msd_std = msds.std(axis=1)
msd_sem = msds.sem(axis=1)

fig, ax = plt.subplots(figsize=(10, 8))
msd_mean.plot(ax=ax, lw=2)

# std
ax.fill_between(msd_mean.index, msd_mean, msd_mean + msd_std, alpha=0.1)
ax.fill_between(msd_mean.index, msd_mean, msd_mean - msd_std, alpha=0.1)

# sem
ax.fill_between(msd_mean.index, msd_mean, msd_mean + msd_sem, alpha=0.2)
ax.fill_between(msd_mean.index, msd_mean, msd_mean - msd_sem, alpha=0.2)

ax.set_xlabel('Delay (s)')
ax.set_ylabel('MSD ($\mu m^2.s^{-1}$)')


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
Display the covariance matrix.
Can you write Python code for it?


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
Fitting
In msd_fitting.m.
Brownian diffusion
Fit the following equation : $MSD(\tau) = 6D\tau$
Can you write Python code for it?

# Purely diffusive model

def msd_model(tau, D_coeff):
    return 6 * D_coeff * tau

msd_observed = msd_mean.copy()
tau = msd_mean.index

popt, pcov = optimize.curve_fit(msd_model, tau, msd_observed)
errors = np.sqrt(np.diag(pcov))

print("Estimate for D coeff is {:.2f} with variance = {:.5f}".format(popt[0], errors[0]))

# Constant model

def msd_model(tau, sigma_e):
    return 6 * sigma_e ** 2

msd_observed = msd_mean.copy()
tau = msd_mean.index

popt, pcov = optimize.curve_fit(msd_model, tau, msd_observed)
errors = np.sqrt(np.diag(pcov))

print("Estimate for sigma_e is {:.2f} with variance = {:.5f}".format(popt[0], errors[0]))

# Diffusive + error model

def msd_model(tau, D_coeff, sigma_e):
    return 6 * D_coeff * tau + 6 * sigma_e ** 2

msd_observed = msd_mean.copy()
tau = msd_mean.index

popt, pcov = optimize.curve_fit(msd_model, tau, msd_observed)
errors = np.sqrt(np.diag(pcov))

print("Estimate for D coeff is {:.2f} with variance = {:.5f}".format(popt[0], errors[0]))
print("Estimate for sigma_e is {:.2f} with variance = {:.5f}".format(popt[1], errors[1]))


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
Implementation with PyMC3
See https://pymc-devs.github.io/pymc3/getting_started/#a-motivating-example-linear-regression for an introduction to PyMC3.
Brownian diffusion
Fit the following equation : $MSD(\tau) = 6D\tau$
Can you write Python code for it?

# Purely diffusive model

msd_observed = msd_mean.copy()

with pm.Model() as model:

    D_coeff = pm.Uniform("D_coeff", lower=0, upper=1000)

    tau = msd_mean.index
    msd_model = 6 * D_coeff * tau

    observation = pm.Normal("observation", mu=msd_model, observed=msd_observed)

    step = pm.NUTS()
    trace = pm.sample(1000, step, )

print("\nEstimate for D coeff is {:.2f} with variance unknown".format(trace["D_coeff"][-1]))
pm.traceplot(trace)

# Constant model

msd_observed = msd_mean.copy()

with pm.Model() as model:

    sigma_e = pm.Uniform("sigma_e", lower=0, upper=10)

    tau = msd_mean.index
    msd_model = 6 * sigma_e ** 2

    observation = pm.Normal("observation", mu=msd_model, observed=msd_observed)

    step = pm.NUTS()
    trace = pm.sample(1000, step)

print("\nEstimate for sigma_e is {:.2f} with variance unknown".format(trace["sigma_e"][-1]))
pm.traceplot(trace)

# Diffusive + error model

msd_observed = msd_mean.copy()

with pm.Model() as model:

    D_coeff = pm.Uniform("D_coeff", lower=0, upper=1)
    sigma_e = pm.Uniform("sigma_e", lower=0, upper=10)

    tau = msd_mean.index
    msd_model = 6 * sigma_e ** 2 + 6 * D_coeff * tau

    observation = pm.Normal("observation", mu=msd_model, observed=msd_observed)

    step = pm.NUTS()
    trace = pm.sample(1000, step)

print("\nEstimate for D coeff is {:.2f} with variance unknown".format(trace["D_coeff"][-1]))
print("Estimate for sigma_e is {:.2f} with variance unknown".format(trace["sigma_e"][-1]))
pm.traceplot(trace)


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
Naive implementation from Matlab code
Matlab code is available here : http://msd-bayes.org/
Covariance matrix
In msd_curves_bayes.m.
Can you write Python code for it?

# Get the average MSD
msd_mean = msds.mean(axis=1)

# Get difference between each individual curve and the mean curve
errors = msds.copy()
for i, col in msds.iteritems():
    errors.loc[:, i] = col - msd_mean

# Calculate raw covariance matrix
error_cov_raw = np.cov(errors)

# Regularize covariance matrix (TODO)
error_cov = error_cov_raw.copy()

# Covariance of the mean curve
error_cov_raw /= errors.shape[0]
error_cov /= errors.shape[0]


I am working on the file "Analysis/MSD_Bayes/notebook.ipynb".
The first task is:
WIP
 IN CONSTRUCTION
based on http://nbviewer.ipython.org/github/CamDavidsonPilon/Probabilistic-Programming-and-Bayesian-Methods-for-Hackers/blob/master/Chapter1_Introduction/Chapter1.ipynb#Introducing-our-first-hammer:-PyMC
Can you write Python code for it?

%matplotlib qt
%load_ext autoreload
%autoreload 2

import numpy as np
import pandas as pd
from scipy import io
from scipy import optimize

import pymc3 as pm
import theano
import theano.tensor as t

import matplotlib.pyplot as plt

count_data = np.loadtxt("txtdata.csv")
plt.bar(np.arange(len(count_data)), count_data, color="#348ABD", edgecolor="none")

count_data = np.loadtxt("txtdata.csv")

alpha = 1.0 / count_data.mean()  # Recall count_data is the variable that holds our txt counts

with pm.Model() as model:

    lambda_1 = pm.Exponential("lambda_1", alpha)
    lambda_2 = pm.Exponential("lambda_2", alpha)

    tau = pm.DiscreteUniform("tau", lower=0, upper=1000)

    days = np.arange(len(count_data))
    lambda_ = pm.switch(tau >= days, lambda_1, lambda_2)

    observation = pm.Poisson("observation", mu=lambda_, observed=count_data)

    step = pm.Metropolis()
    trace = pm.sample(1000, step)

print()
print("tau", trace['tau'][-1])
print("lambda_1", trace['lambda_1'][-1])
print("lambda_2", trace['lambda_2'][-1])

pm.traceplot(trace)

tau = trace['tau'][-1]
lambda_1 = trace['lambda_1'][-1]
lambda_2 = trace['lambda_2'][-1]

mcount = np.zeros(count_data.shape)
mcount[:tau] = lambda_1
mcount[tau:] = lambda_2

plt.figure(figsize=(10, 6))
plt.bar(np.arange(len(count_data)), count_data, color="#348ABD", edgecolor="none")
plt.plot(mcount, lw=4, color="#E24A33")

count_data = np.loadtxt("txtdata.csv")

@theano.compile.ops.as_op(itypes=[t.lscalar, t.lscalar, t.dscalar, t.dscalar, t.dscalar], otypes=[t.dvector])
def lambda_(tau1, tau2, lambda_1, lambda_2, lambda_3):
    out = np.zeros(len(count_data))
    out[:tau1] = lambda_1  # lambda before tau is lambda1
    out[tau1:tau2] = lambda_2  # lambda before tau is lambda1
    out[tau2:] = lambda_3  # lambda after (and including) tau is lambda2
    return out

alpha = 1.0 / count_data.mean()  # Recall count_data is the variable that holds our txt counts

with pm.Model() as model:

    lambda_1 = pm.Exponential("lambda_1", alpha)
    lambda_2 =...


I am working on the file "Analysis/Sigmoids function to model connection probabilities.ipynb".
The first task is:

Chemical Synapses

Can you write Python code for it?

# Plot probabilities of inhibitory chemical synapses between inhibitory neurons
ichem_found  = np.array([10, 30, 29, 12, 2, 4], dtype=int)
ichem_tested = np.array([4,  14,  9,  2, 0, 0], dtype=float)

yval = (ichem_tested/ichem_found)*100
xval = arange(25,350,50) # bins
xval = array([ 25,  75, 125, 175,275, 325])
fit_sigmoid(xval, yval, 'chem_syn')


I am working on the file "Analysis/Sigmoids function to model connection probabilities.ipynb".
The first task is:

Electrical Synapses

Can you write Python code for it?

# Plot probabilities of inhibitory electrical synapses between inhibitory neurons
igap_found  = np.array([ 5, 15, 15, 6, 4, 1,  2], dtype=int)
igap_tested = np.array([ 2, 11,  6, 2, 0, 0,  0], dtype=float)

yval = (igap_tested/igap_found)*100
xval = arange(25,350,50) # bins

fit_sigmoid(xval, yval, 'elec_syn')


I am working on the file "Analysis/Uniform model for inhibitory synaptic connections-Copy1.ipynb".
The first task is:
<H2> Bar plots for estatistical significance</H2>
Can you write Python code for it?

# This is our null hypothesis
fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.nchem, n_found = mydataset.motif.ii_chem_found, larger=1);
ax.set_title('Chemical synapses', size=20);
ax.set_ylim(ymax=55);
ax.tick_params(labelsize=20)

fig.savefig('ii_chem.pdf')

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.nelec, n_found = mydataset.motif.ii_elec_found, larger=1);
ax.set_title('Electrical synapses',  size=20);
ax.set_ylim(ymax=45);
ax.tick_params(labelsize=20)

fig.savefig('ii_elec.pdf')

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.nbid, n_found = mydataset.motif.ii_c2_found, larger=1);
ax.set_title('Bidirectional chemical',  size=20);
ax.set_ylim(ymax=25);
ax.tick_params(labelsize=20)

fig.savefig('ii_c2.pdf')

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.ncon, n_found = mydataset.motif.ii_con_found, larger=0);
ax.set_title('Convergent inhibitory',  size=20);
ax.set_ylim(ymin=0, ymax=10);
ax.tick_params(labelsize=20)

fig.savefig('ii_con.pdf')

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.ndiv, n_found = mydataset.motif.ii_div_found, larger=1);
ax.set_title('Divergent inhibitory',  size=20);
ax.set_ylim(ymin=0, ymax=10);
ax.tick_params(labelsize=20)

fig.savefig('ii_div.pdf')

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.nlin, n_found = mydataset.motif.ii_lin_found, larger=0);
ax.set_title('Linear chains',  size=20);
ax.set_ylim(ymin=0, ymax=15);
ax.tick_params(labelsize=20)

fig.savefig('ii_chain.pdf')

#pii_chain # change this value in the plot!

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.nc1e, n_found = mydataset.motif.ii_c1e_found, larger=1);
ax.set_title('Electrical and one chemical',  size=20);
ax.set_ylim(ymax=35);
ax.tick_params(labelsize=20)

#fig.savefig('ii_c1e.pdf')

fig = figure(5)
ax = fig.add_subplot(111)

ax = barplot(simulation = unimodel.nc2e, n_found =...


I am working on the file "Analysis/Uniform model for inhibitory synaptic connections-Copy1.ipynb".
The first task is:
<H2> Calculate empirical probabilities of inhibitory connectivity motifs </H2>
<P> It will be used to create the null hypothesis</P>
Can you write Python code for it?

PC = mydataset.motif.ii_chem_found/mydataset.motif.ii_chem_tested
PE = mydataset.motif.ii_elec_found/mydataset.motif.ii_elec_tested

Pbid  = mydataset.motif.ii_c2_found/mydataset.motif.ii_c2_tested
Pdiv = mydataset.motif.ii_div_found/mydataset.motif.ii_div_tested
Pcon = mydataset.motif.ii_con_found/mydataset.motif.ii_con_tested
Plin = mydataset.motif.ii_lin_found/mydataset.motif.ii_lin_tested

PC1E = mydataset.motif.ii_c1e_found/mydataset.motif.ii_c1e_tested
PC2E = mydataset.motif.ii_c2e_found/mydataset.motif.ii_c2e_tested

info = [
    ['key', 'Type', 'Motif', 'Probability'],
    ['ii_chem', 'P(C)', 'chemical synapse',PC ],
    ['ii_elec', 'P(E)', 'electrical synapse',PE ],
    ['','',''],
    ['ii_c2', 'P(C U C)','bidirectional chemical synapse',Pbid],
    ['ii_con', 'Pcon', 'convergent inhibitory motifs', Pcon],
    ['ii_div', 'Pdiv', 'divergent inhibitory motifs', Pdiv],
    ['ii_lin', 'Plin', 'linear inhibitory motifs', Plin],
    ['',''],
    ['ii_c1e', 'P(C U E)', 'electrical and unidirectional chemical', PC1E],
    ['ii_c2e', 'P(2C U E):','electrical and bidirectional chemical', PC2E],

]
print(AsciiTable(info).table)


I am working on the file "Analysis/Uniform model for inhibitory synaptic connections-Copy1.ipynb".
The first task is:

 Simulate synapses with uniform distribution

We simulate a random but uniform connection probability, which is adjusted to the empirical probability found in the recordings. To evaluate that the simulations are correct, we calculate analytically the expected number of chemical and electrical connections in the simulations. The analytical solutions and the simulations must be very close.
Can you write Python code for it?

# %%timeit about 13 sec for 1,000 iterations
unimodel.run(n_iter = 1000, seed=0) # 1,000 random simulations


I am working on the file "Analysis/Uniform model for inhibitory synaptic connections-Copy1.ipynb".
The first task is:

Testing null hypothesis

 The null hypothesis is that all motifs emerge form the individual probabilities of chemical (PC) and electrical (PE) synapses

Can you write Python code for it?

info = [
    ['Type', 'Analytical', 'Simulation', 'Observation'],
    ['ii_chem', PC*mydataset.motif['ii_chem']['tested'], unimodel.nchem.mean(), mydataset.motif.ii_chem_found ],
    ['ii_elec', PE*mydataset.motif['ii_elec']['tested'], unimodel.nelec.mean(), mydataset.motif.ii_elec_found ],
    [''],
    ['ii_bid', PC*PC*mydataset.motif['ii_c2']['tested'],  unimodel.nbid.mean(), mydataset.motif.ii_c2_found  ],
    ['ii_con', PC*PC*mydataset.motif['ii_con']['tested'], unimodel.ncon.mean(), mydataset.motif.ii_con_found ],
    ['ii_div', PC*PC*mydataset.motif['ii_div']['tested'], unimodel.ndiv.mean(), mydataset.motif.ii_div_found ],
    ['ii_lin', PC*PC*mydataset.motif['ii_lin']['tested'], unimodel.nlin.mean(), mydataset.motif.ii_lin_found ],
    [''],
    ['ii_c1e', PC*PE*mydataset.motif['ii_c1e']['tested'],  unimodel.nc1e.mean(), mydataset.motif.ii_c1e_found ],
    ['ii_c2e', PC*PC*PE*mydataset.motif['ii_c2e']['tested'],  unimodel.nc2e.mean(), mydataset.motif.ii_c2e_found  ]
]
print(AsciiTable(info).table)


I am working on the file "Analysis/cantera_simulation_time_resolved.ipynb".
The first task is:
Load the species and reaction from the RMG-generated chemkin file chem_annotated.inp and species_dictionary.txt file found in your chemkin folder after running a job.
Can you write Python code for it?

folder = '/home/mjliu/Documents/Models/Naphthalene/acetylene/run3'
speciesList, reactionList = loadChemkinFile(os.path.join(folder, 'chem0252.inp'),
                                            os.path.join(folder, 'species_dictionary.txt'))


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Analysis of Step Response
The goal here is to understand the dynamics of the receptor state for a step response.
From the foregoing, we established that it's sufficient to look at the fraction of receptors that are in the phosphorylated state (since this correlates with Yp). Now we want to understand what substates contribute to phosphorylated receptors.
Spiro shows a step response plot with L going from 0 to 11uM and "perfect adaption of Yp. We do not see this. Possibly, this is because he used a model with more methylation levels. Below is Spiro's step response curve.
Very Large Step - 1 mM (1,000 times ligand-receptor $K_d$)
Can you write Python code for it?

# Spiro's plot for a step response with L=1mM
from IPython.display import Image, display
display(Image(filename='img/spiro_largestep.png'))
# The solid line is Yp

# The amount of ligand is 1000 times the amount of receptors
sim(elements=["at (time > 100): L = 1e-3"])
plotter.lines(["L", "Yp", "Bp"])


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Ligand Binding Analysis
This section provides an analytical validation of one quantity in the simulation, the fraction of ligand bound receptors. The analysis is based on a simple equilibrium analysis.
- $K_d = 1uM$ is the disassociation constant for the reaction $TL \rightleftharpoons T + L$, where $T$ is the receptor and $L$ is the ligand.
- $r$ is the ratio $\frac{L_{TOT}}{T_{TOT}}$, where $L_{TOT}$ is the total amount of ligand and $T_{TOT}$ is the total number of receptors.
Can you write Python code for it?

Kd = 1e-6 # 1 micromolar
TTOT = 8  # 8 micro molars
K = Kd/(TTOT*1e-6)

import numpy as np
def fBound(r, K=K):
    """
    Using steady state analysis to compute the fraction of ligand bound to receptors
    :param float r: ratio of total L to total T
    :param float K: ratio of Kd to total T
    :return float: fraction of ligand bound to receptors
    """
    result = None
    b = -(1 + r + K)
    term1 = -b/2
    term2 = np.sqrt(b**2-4*r)/2
    result1 = term1 - term2
    result2 = term1 + term2
    if result1 <= 1.0 and result1 >= 0:
        result = result1
    if result2 <= 1.0 and result2 >= 0:
        if result is not None:
            raise RuntimeError("Two valid solutions")
        else:
            result = result1
    if result is None:
        raise RuntimeError("No valid solution.")
    return result

def evaluateEstimateError(L):
    """
    Returns the error of expected fraction of ligand bound compared with the values
    obtained from simulation.
    """
    sim(elements=["at (time > 100): L = %s" % (L*1e-6)])
    actual = model.getVariable("fT___")[-1]  # Get the last (steady state) value
    expected = fBound(L/TTOT)
    return (expected -actual)/actual

evaluateEstimateError(4.1)

evaluateEstimateError(1.1)

evaluateEstimateError(0.1)


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Observations
 - Phosphorylation level of receptor does not recover after the Ligands are introduced.
 - See a shift to higher methylation states
Can you write Python code for it?

# Analyze the methylation levels
sim(elements=["at (time > 100): L = 11e-6"])
plotter.lines(["f___2", "f___3", "f___4"], yrange=[0,1])

# Analyze the methylation levels for ligand bound receptors
sim(elements=["at (time > 100): L = 4.1e-6"])
plotter.lines(["fT___", "fT__2", "fT__3", "fT__4"], yrange=[0,0.5])


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Observations
- Large fraction of LT2 means reduced phosphorylation
Questions
- Why doesn't the LT2 methylate?
Can you write Python code for it?

# Breakdown the state phosphorylations by ligand bound
sim(elements=["at (time > 100): L = 11e-6"])
plotter.lines(["f_T__", "fFT__", "fTT__"], yrange=[0, 0.2])


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Observations
- The steady state after the large step is about Yp=2.5 uM as opposed to Yp=6uM in Spiro.
Small Step - 0.11 uM  (11% of ligand-receptor $K_d$)
Can you write Python code for it?

from IPython.display import Image, display
display(Image(filename='img/spiro_0.11.png'))
# The solid line is Yp;

sim(elements=["at (time > 10): L = 0.11e-6"], end=50)
plotter.lines(["Yp"])


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Step Response - 11 uM (138% of ligand-receptor $K_d$)
Can you write Python code for it?

# System is flooded with more ligand than there are receptors. Much more ligand than receptors.
sim(elements=["at (time > 100): L = 11e-6"])
plotter.lines(["Yp"])


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Step Response - 4 uM (400% of ligand-receptor $K_d$)
Can you write Python code for it?

sim(elements=["at (time > 100): L = 4e-6"])
plotter.lines(["Yp"])


I am working on the file "Analysis/chemotaxis.ipynb".
The first task is:
Summary
Initially, we consider a step response. Later, we repeat the same analysis for a ramp. These correspond to the analyses done by Spiro.
Most of the analysis is done using fractional concentrations (variables that begin with "f"). We begin by justifying this.
Next, we analyze the effects of a step response. "Perfect adaptation" is possible if the step is small enough.
Software Dependencies
The model and various utility functions are in the python modules chemotaxis_model and data_plotter.
Can you write Python code for it?

import numpy as np
%matplotlib inline
import matplotlib.pyplot as plt
import tellurium as te
from chemotaxis_model import ChemotaxisModel, StateAggregationFactory
from data_plotter import DataPlotter
model = ChemotaxisModel()


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
Create reaction network
Can you write Python code for it?

import networkx as nx
import matplotlib.pyplot as plt
from rmgpy.tools.extractInfoFromckcsv import getConcentrationDictFromCKCSV, getROPFromCKCSV, getFluxGraphEdgesDict
from rmgpy.chemkin import getSpeciesIdentifier
from IPython.display import display
import numpy as np
from rmgpy.rmg.pdep import PDepReaction
%matplotlib inline


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
More detail for the H_Abstration and Disproportionation in last step of the pathway
Can you write Python code for it?

print ''
print 'The step you see currently is between the two species {0}'.format(step_pair)
print ''
print 'Positive flux in below means positive production of 1st node of the pair.'
print ''
print 'This step have {0} H_Abstration and Disproportionation in total.'.format(len(h_abs_rxns + disp_rxns))

flux_threshold = 1e-9

print ''
print 'TO avoid too much printout, the reactions shown below have fluxes above {0}.'.format(flux_threshold)

total_flux = 0
rxn_flux_tups = []
for rxn in h_abs_rxns + disp_rxns:
    if showIntegratedFlux:
    #integrate the flux using a trapezoidal rule
        flux = 0
        for t in range(timepoint_index):
            rxnROP0 = graph_edges_dict_simple[step_pair][rxn][t]
            rxnROP1 = graph_edges_dict_simple[step_pair][rxn][t+1]
            flux += 0.5 * (rxnROP1 * Vlist[t+1] + rxnROP0 * Vlist[t]) * (timeList[t+1] - timeList[t]) # in mole
    else:
        flux = graph_edges_dict_simple[step_pair][rxn][timepoint_index]
    rxn_flux_tups.append((rxn, flux))

rxn_flux_tups = sorted(rxn_flux_tups, key=lambda tup: tup[1], reverse=False)
for tup in rxn_flux_tups:
    rxn = tup[0]
    flux = tup[1]
    if flux > flux_threshold:
        total_flux += flux
        print ''
        print "**********************************************************************************"
        if showIntegratedFlux:
            print "rxn#{0}: {1}: Integrated Flux = {2:.3E} mole ".format(rxn.index, str(rxn), flux)
        else:
            print "rxn#{0}: {1}: Flux = {2:.3E} mole/cm3/s".format(rxn.index, str(rxn), flux)
        display(rxn)
print "***************************************"
print ''
if showIntegratedFlux:
    print "TOTAL integrated flux from h_abs and disp is {0:.3E} mole.".format(total_flux)
else:
    print "TOTAL flux from h_abs and disp is {0:.3E} mole/cm3/s.".format(total_flux)


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] Functionality 1: find most dominant species at the investigated time point
Pleae specify

dominant_list: If you want to visualize the first top 10 dominant species, specify the list as [0, 9]. If you want to visualize from the 5th most dominant species to the 10th most dominant species, specify the list as [4, 9].
Can you write Python code for it?

########## User Input ####################
dominant_list = [0, 9]
##########################################

# sort out the moleFraction_dict in the descending order
v = list(moleFraction_dict.values())
k = list(moleFraction_dict.keys())
index_list = [i[0] for i in sorted(enumerate(v), key=lambda x:x[1], reverse=True)]
print '=============================================================='
print 'Order \t Species \t Mole Frac. \t MW (g/mol)'
print '=============================================================='

for order in range(dominant_list[0], dominant_list[-1]+1):
    index_number = index_list[order]
    species_dominant = k[index_number]
    molecule_dominant = species_identifier_dict.get(species_dominant)
    mole_fraction = v[index_number]
    molecular_weight = molecule_dominant.molecule[0].getMolecularWeight()*1000
    if len(species_dominant) < 6: # if the species label is too short, put more space to allign it
        print '{0} \t {1} \t\t {2} \t {3:.0f}'.format(order, species_dominant, mole_fraction, molecular_weight)
    else:
        print '{0} \t {1} \t {2} \t {3:.0f}'.format(order, species_dominant, mole_fraction, molecular_weight)
    display(molecule_dominant)
    print '--------------------------------------------------------------'


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] Functionality 2: find paths between two species
Pleae specify

source species

target species
Can you write Python code for it?


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] Functionality 3: find paths from a particular species
Please specify

source species

depth of pathway you want search, e.g., depth = 2 means search pathways 2 steps down from source species

path_top_list, e.g., [0, 3] means 1st step of the pathway takes the most significant branch and 2nd step would be the third most siginificant branch
Can you write Python code for it?

########## User Input ####################
source = "S(16320)"
depth = 2
path_top_list = [0, 0]
##########################################

current_node = source

print ''
print '\t Pathway Report '
print '======================================'
print 'The pathway you are intested in has {0} steps.'.format(depth)
for step in range(depth):
    print "\n"
    nextNode_flux_list = [(next_node, G[current_node][next_node]['total_flux']) for next_node in G[current_node]]
    sorted_nextNode_flux_list = sorted(nextNode_flux_list, key=lambda tup: -tup[1])

    # choose the top one as next node
    tup = sorted_nextNode_flux_list[path_top_list[step]]
    next_node = tup[0]
    step_flux = tup[1]

    print ""
    if showIntegratedFlux:
        print "Step{0} \t\t\t\t Integrated Flux (mole)".format(step+1)
        print '************************************************************************'
        print "{0} --> {1} \t\t {2:.3E}".\
        format(current_node, next_node, step_flux)
        print '************************************************************************'
    else:
        print "Step{0} \t\t\t\t Flux (mole/cm3/s)".format(step+1)
        print '****************************************************'
        print "{0} --> {1} \t\t {2:.3E}".\
        format(current_node, next_node, step_flux)
        print '****************************************************'

    step_pair = (current_node, next_node)
    if step_pair not in graph_edges_dict_simple:
        step_pair = (next_node, current_node)

    h_abs_rxns = []
    disp_rxns = []
    Pdep_rxns = []
    for rxn in graph_edges_dict_simple[step_pair]:
        if isinstance(rxn, PDepReaction):
            Pdep_rxns.append(rxn)
        elif rxn.family == "H_Abstraction":
            h_abs_rxns.append(rxn)
        elif rxn.family == "Disproportionation":
            disp_rxns.append(rxn)
        else:
            print ''
            print "Example reaction: rxn#{0}".format(rxn.index)
            print ''
            if...


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] Functionality 4: find paths towards a particular species
Similar to functionality 2, please specify
Please specify

target species

depth of pathway you want search, e.g., depth = 2 means search pathways 2 steps up beyond target species

path_top_list, e.g., [0, 3] means 1st step of the pathway takes the most significant branch and 2nd step would be the third most siginificant branch
Can you write Python code for it?

########## User Input ####################
target = "CH3(3)"
depth = 5
path_top_list = [0, 0, 0, 0, 0]
##########################################

current_node = target
print ''
print '\t Pathway Report '
print '======================================'
print 'The pathway you are intested in has {0} steps.'.format(depth)
print ''
if showIntegratedFlux:
    product_flux = 0
    for t in range(timepoint_index):
        productROP0 = spc_total_rop_dict[target][1][t]
        productROP1 = spc_total_rop_dict[target][1][t+1]
        product_flux += 0.5 * (productROP1 * Vlist[t+1] + productROP0 * Vlist[t]) * (timeList[t+1] - timeList[t]) # in mole
    print "total integrated product flux for {0} is {1:.3E} mole.".format(target, product_flux)
else:
    print "total product flux for {0} is {1:.3E} mole/cm3/s.".format(target, spc_total_rop_dict[target][1][timepoint_index])
for step in range(depth):
    print "\n"
    prev_nodes = []
    for node1 in G:
        if current_node in G[node1]:
            prev_nodes.append(node1)
    prevNode_flux_list = [(prev_node, G[prev_node][current_node]['total_flux']) for prev_node in prev_nodes]
    sorted_prevNode_flux_list = sorted(prevNode_flux_list, key=lambda tup: -tup[1])

    # choose the top one as next node
    tup = sorted_prevNode_flux_list[path_top_list[step]]
    prev_node = tup[0]
    step_flux = tup[1]

    print ""
    if showIntegratedFlux:
        print "Step{0} \t\t\t\t Integrated Flux (mole)".format(step+1)
        print '************************************************************************'
        print "{0} <-- {1} \t\t {2:.3E}".\
        format(current_node, prev_node, step_flux)
        print '************************************************************************'
    else:
        print "Step{0} \t\t\t\t Flux (mole/cm3/s)".format(step+1)
        print '****************************************************'
        print "{0} <-- {1} \t\t {2:.3E}".\
        format(current_node, prev_node, step_flux)
        print...


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] Prepare files
Please add

annotated chemkin files (mechanism and species dictionary) and

chemkin simulation ckcsv file

to a folder with name stored in mech variable in the next cell.
Can you write Python code for it?

mechPath = '/home/mjliu/Documents/Models/Tetralin/acetylene/run1'
chemkinPath= os.path.join(mechPath, 'chem_annotated.inp')
dictionaryPath = os.path.join(mechPath, 'species_dictionary.txt')
ckcsvPath= os.path.join(mechPath, 'CKSoln.ckcsv')

model = CoreEdgeReactionModel()
model.core.species, model.core.reactions = loadChemkinFile(chemkinPath,dictionaryPath)

# generate paris for reactions that don't have flux pairs
for rxn in model.core.reactions:
    if not rxn.pairs: rxn.generatePairs()


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] output2: visualize the path of interest
Please specify

the path index you want to visualize
Can you write Python code for it?

########## User Input ####################
path_index_investigate = 1
##########################################

path = path_fluxes[path_index_investigate][0]
path_steps = len(path) - 1
print ''
print '\t Pathway Report '
print '======================================'
print 'The pathway you are intested in has {0} steps.'.format(len(path))
for step in range(path_steps):
    step_pair = (path[step], path[step+1])
    h_abs_rxns = []
    disp_rxns = []
    Pdep_rxns = []

    print ""
    if showIntegratedFlux:
        print "Step{0} \t\t\t\t Integrated Flux (mole)".format(step+1)
        print '***********************************************************************'
        print "{1} --> {2} \t\t {3:.3E}".\
        format(step+1, step_pair[0], step_pair[1], G[step_pair[0]][step_pair[1]]['total_flux'])
        print '***********************************************************************'
    else:
        print "Step{0} \t\t\t\t Flux (mole/cm3/s)".format(step+1)
        print '*****************************************************'
        print "{1} --> {2} \t\t {3:.3E}".\
        format(step+1, step_pair[0], step_pair[1], G[step_pair[0]][step_pair[1]]['total_flux'])
        print '*****************************************************'

    if step_pair not in graph_edges_dict_simple:
        step_pair = (step_pair[1], step_pair[0])

    print ''
    for rxn in graph_edges_dict_simple[step_pair]:
        if isinstance(rxn, PDepReaction):
            Pdep_rxns.append(rxn)
        elif rxn.family == "H_Abstraction":
            h_abs_rxns.append(rxn)
        elif rxn.family == "Disproportionation":
            disp_rxns.append(rxn)
        else:
            print "Example reaction: rxn#{0}".format(rxn.index)
            print ''
            print str(rxn)
            display(rxn)
            print 'Forward rate coefficient at {0} K: {1:.2E} [cm3, mole, s]'.format(T, rxn.getRateCoefficient(T))
            reverseRate = rxn.generateReverseRateCoefficient()
            print 'Reverse rate coefficient...


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
[user input] step2: pick a time to analyze pathways
Please specify

the time point you want to investigate

the chemkin name of your initial species/reactant

the reaction temperature in K

the reaction pressure in Pa if the mechanism includes pressure-dependent reactions

showIntegratedFlux: True if users want to use time-integrated fluxes (integrated up to the investigated time point) in mole for analysis. Flase if users want to use instantaneous fluxes in mole/cm3/s at the investigated time point for analysis
Can you write Python code for it?

########## User Input ####################
time_investigated = 1.0/3600 # hour
reactant_name = 'C10H12(1)'
T = 1200 # K
P = 16*101325 # Pa
showIntegratedFlux = True
##########################################

timepoint_index = (np.abs(firstColDict['Time_(sec)']-time_investigated*3600)).argmin()
timeList = firstColDict['Time_(sec)']
Plist = firstColDict['Pressure_(atm)']
Tlist = firstColDict['Temperature_(k)']
Vlist = firstColDict['Volume_(cm3)']

print "Investigated time point is {0} secs".format(timeList[timepoint_index])
print "At this moment, the reactant's mole fraction is {}.".format(spc_mf_dict[reactant_name][timepoint_index])

# Create moleFraction_dict at the investigated time point
moleFraction_dict = {}
for i, v in spc_mf_dict.items():
    moleFraction_dict.update({i:v[timepoint_index]})

# Create species_identifier_didct for the core species
species_identifier_dict = {}
for i, s in enumerate(model.core.species):
    species_identifier_dict.update({getSpeciesIdentifier(model.core.species[i]): s})

# find the total consumption of the reactant in mol (positive value means consumption)
reactantConsumption = 0 #
for t in range(timepoint_index):
    reactantROP0 = spc_total_rop_dict[reactant_name][1][t] # in mole/cm3*s
    reactantROP1 = spc_total_rop_dict[reactant_name][1][t+1] # in mole/cm3*s
    #integrate the flux using a trapezoidal rule
    reactantConsumption += 0.5 * (reactantROP1 * Vlist[t+1] + reactantROP0 * Vlist[t]) * (timeList[t+1] - timeList[t])
reactantConsumption = reactantConsumption * -1.0 # in mole

if showIntegratedFlux:
    print "At this moment, the reactant's total molar consumption is {0:.3E} mole.".format(reactantConsumption)

# create DiGraph for this moment
G = nx.DiGraph()
for pair in graph_edges_dict:
    node1 = getSpeciesIdentifier(pair[0])
    node2 = getSpeciesIdentifier(pair[1])
    e_rawdata = graph_edges_dict[pair]
    total_flux = 0
    for rxn in e_rawdata:
        if showIntegratedFlux:
            for t in range(timepoint_index):
                rxnROP0 =...


I am working on the file "Analysis/mechanism_analyzer.ipynb".
The first task is:
output1: sort out all the paths and most significant one
Can you write Python code for it?

paths = list(nx.all_simple_paths(G, source=src_species, target=tgt_species, cutoff=6))

path_fluxes = []
if showIntegratedFlux:
    print '========================================================================================================'
    print 'Path index \t Flux (mols)    Path steps'
else:
    print '================================================'
    print 'Path index \t Flux (mol/cm3/s)    Path steps'
print ''

path_fluxes = [(path, min([G[path[step]][path[step+1]]['total_flux'] for step in range(len(path)-1)])) for path in paths if 'H(5)' not in path]

path_fluxes.sort(key=lambda x: x[1], reverse=True)

for i, (path, flux) in enumerate(path_fluxes):
    if flux < 1e-10:
        break
    path_fluxes.append(flux)
    if showIntegratedFlux:
        print 'Path #{0}: \t {1:.3E} \t {2}'.format(i, flux, path)
    else:
        print 'Path #{0}: \t {1:.3E} \t {2}'.format(i, flux, path)


I am working on the file "Analysis/misc/Bidirectional connections as a function of distance.ipynb".
The first task is:

Distances in recurrently connected inhibitory neurons

Can you write Python code for it?

def read_rec_dist(fname):
    """
    get distances between bidirectionally connected interneurons
    from a matrix of intersomatic distances.

    Argument:
    fname: string
        the matrix name to that contains the connected synapses (*.syn)
    """
    # take all non-diagonal elements
    mydistpath = '../data/PV/' + fname[:-3] + 'dist'

    try:
        D = II_slice(np.loadtxt(mydistpath), int(fname[0]))

    except IOError:
        warnings.warn(mydistpath + ' not found!')
        return([])

    try:
        S = np.loadtxt('../data/PV/'+fname)

    except IOError:
        warnings.warn(fname + ' not found!')
        return([])

    S = II_slice(S, int(fname[0]))
    S[S==2] = 0 # remove gaps
    S[S==3] = 1 # remove gaps in chemical

    x,y = np.nonzero(S)
    ids = zip(x,y)
    mydist = list()
    if ids>0:
        for i,j in ids:
            if S[j,i] == 1:
                mydist.append(D[i,j])

    print(np.unique(np.abs(mydist)))
    return( np.unique(np.abs(mydist)).tolist() )

# Number of bidirectionally connected interneurons
mydataset.motif['ii_c2']

# select experiments with bidirectional motifs
mybidirec = [i for i in range(len(mydataset)) if mydataset.motifs(i)['ii_c2']['found']>0 ]

for i in mybidirec:
    print('Experiment {:3d}, filename: {}'.format(i, mydataset.filename(i)))

dist_found = list()
for i in mybidirec:
    dist_found += read_rec_dist(mydataset.filename(i))

dist_found

mybins = arange(0,550, 50)
bid_tested = np.unique(dist_tested)
plt.hist(bid_tested, bins = mybins, facecolor='white', lw=2);
plt.ylim(ymax=20);
plt.ylabel('Inhbitory chemical synapses');
plt.xlabel('Intersomatic distance ($\mu$m)');

plt.hist(dist_found, bins = mybins, facecolor='gray', lw=2);


I am working on the file "Analysis/misc/Counting CA3 synapses.ipynb".
The first task is:
the element in the list with IN[0] contains zero interneurons (all the rest
are principal neurons)
Can you write Python code for it?

mydataset.IN[0] # this is the whole data set


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
<H2> Calculate empirical probabilities </H2>
Can you write Python code for it?

PC = mydataset.motif.ii_chem_found/mydataset.motif.ii_chem_tested
PE = mydataset.motif.ii_elec_found/mydataset.motif.ii_elec_tested

PC2  = mydataset.motif.ii_c2_found/mydataset.motif.ii_c2_tested
Pdiv = mydataset.motif.ii_div_found/mydataset.motif.ii_div_tested
Pcon = mydataset.motif.ii_con_found/mydataset.motif.ii_con_tested
Plin = mydataset.motif.ii_lin_found/mydataset.motif.ii_lin_tested

PC1E = mydataset.motif.ii_c1e_found/mydataset.motif.ii_c1e_tested
PC2E = mydataset.motif.ii_c2e_found/mydataset.motif.ii_c2e_tested

info = [
    ['key', 'Probability', 'Motif', 'Value'],
    ['ii_chem', 'P(C)', 'chemical synapse',PC ],
    ['ii_elec', 'P(E)', 'electrical synapse',PE ],
    ['','',''],
    ['ii_c2', 'P(C U C)','bidirectional chemical synapse',PC2],
    ['ii_con', 'Pcon', 'convergent inhibitory motifs', Pcon],
    ['ii_div', 'Pdiv', 'divergent inhibitory motifs', Pdiv],
    ['ii_lin', 'Plin', 'linear inhibitory motifs', Plin],
    ['',''],
    ['ii_c1e', 'P(C U E)', 'electrical and unidirectional chemical', PC1E],
    ['ii_c2e', 'P(2C U E):','electrical and bidirectional chemical', PC2E],

]
print(AsciiTable(info).table)


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
<H2> Plot chemical synapses alone</H2>
Can you write Python code for it?

# This is our null hypothesis
fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_chem, n_found = mydataset.motif.ii_chem_found, larger=1);
ax.set_title('Chemical synapses', size=20);
ax.set_ylim(ymax=40);
ax.tick_params(labelsize=20)

fig.savefig('ii_chem.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
<H2> Plot divergent inhibitory </H2>
Can you write Python code for it?

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_div, n_found = mydataset.motif.ii_div_found, larger=1);
ax.set_title('Divergent inhibitory',  size=20);
ax.set_ylim(ymin=0, ymax=5);
ax.tick_params(labelsize=20)

fig.savefig('ii_div.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
<H2> Simulate random electrical synapses</H2>
from a random distribution whose probability is adjusted to the empirical probability found in the recordings.
Can you write Python code for it?

def myelec_simulation():
    """
    simulate inhibitory electrical connections of the dataset

    Return
    ------
    A iicounter object
    """
    mycount = iicounter()
    for _ in range(PV2):
        mycount += iicounter(elec_squarematrix(size=2,prob = PE))

    for _ in range(PV3):
        mycount += iicounter(elec_squarematrix(size=3, prob = PE))

    return(mycount)

print(myelec_simulation()) # one simulation, test the number of connection tested

# must contain the same number of tested connections
for key in myelec_simulation().keys():
    print(key, mydataset.motif[key])

n_elec = list()
for _ in range(1000):
    syn_elec = myelec_simulation()
    n_elec.append(syn_elec['ii_elec']['found'])


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

Calculating P-Values

Can you write Python code for it?

# operate with NumPY arrays rather than with lists
n_chem = np.array(n_chem)
n_elec = np.array(n_elec)

n_c2 = np.array(n_c2)
n_con = np.array(n_con)
n_div = np.array(n_div)
n_chain = np.array(n_chain)

n_c1e = np.array(n_c1e)
n_c2e = np.array(n_c2e)

pii_chem = len(n_chem[n_chem>mydataset.motif.ii_chem_found]) / n_chem.size
pii_elec = len(n_elec[n_elec>mydataset.motif.ii_elec_found])/ n_elec.size

pii_c2 =  len(n_c2[n_c2 > mydataset.motif.ii_c2_found])/ n_c2.size
pii_con = len(n_con[n_con < mydataset.motif.ii_con_found])/n_con.size # under-rep
pii_div = len(n_div[n_div > mydataset.motif.ii_div_found])/n_div.size
pii_chain = len(n_chain[n_chain < mydataset.motif.ii_lin_found])/n_chain.size # under-rep

pii_c1e = len(n_c1e[n_c1e > mydataset.motif.ii_c1e_found])/ n_c1e.size
pii_c2e = len(n_c2e[n_c2e > mydataset.motif.ii_c2e_found])/ n_c2e.size

info = [
    ['Syn Motif', 'Simulations', 'Empirical', 'P(Simulations)', 'alpha'],
    ['chemical', np.mean(n_chem),  mydataset.motif.ii_chem_found, pii_chem],
    ['electrical', np.mean(n_elec),  mydataset.motif.ii_elec_found, pii_elec],
    [''],
    ['2 chem bidirect', np.mean(n_c2),  mydataset.motif.ii_c2_found, pii_c2,alpha_rec],
    ['convergent', np.mean(n_con),  mydataset.motif.ii_con_found, pii_con, alpha_con],
    ['divergent', np.mean(n_div), mydataset.motif.ii_div_found, pii_div, alpha_div],
    ['chain', np.mean(n_chain), mydataset.motif.ii_lin_found, pii_chain, alpha_chain],
    [''],
    ['1 chem + elec', np.mean(n_c1e),  mydataset.motif.ii_c1e_found, pii_c1e],
    ['2 chem + elec', np.mean(n_c2e),  mydataset.motif.ii_c2e_found, pii_c2e],
     ]
print(AsciiTable(info).table)

from inet.plots import barplot


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

Plot bidirectional chemical synapses

Can you write Python code for it?

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_c2, n_found = mydataset.motif.ii_c2_found, larger=1);
ax.set_title('Bidirectional chemical',  size=20);
ax.set_ylim(ymax=10);
ax.tick_params(labelsize=20)

fig.savefig('ii_c2.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

Plot convergent inhibitory

Can you write Python code for it?

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_con, n_found = mydataset.motif.ii_con_found, larger=False);
ax.set_title('Convergent inhibitory',  size=20);
ax.set_ylim(ymin=0, ymax=4);
ax.tick_params(labelsize=20)

fig.savefig('ii_con.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
<H2>Plot electrical and one chemical synapse alone </H2>
Can you write Python code for it?

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_c1e, n_found = mydataset.motif.ii_c1e_found, larger=1);
ax.set_title('Electrical and one chemical',  size=20);
ax.set_ylim(ymax=25);
ax.tick_params(labelsize=20)

fig.savefig('ii_c1e.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

Plot electrical and two chemical

Can you write Python code for it?

fig = figure(5)
ax = fig.add_subplot(111)

ax = barplot(simulation = n_c2e, n_found = mydataset.motif.ii_c2e_found, larger=1);
ax.set_title('Electrical and two chemical',  size=20);
ax.set_ylim(ymin  = 0, ymax=10);
ax.tick_params(labelsize=20)

fig.savefig('ii_c2d.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
<H2>Plot electrical synapses alone </H2>
Can you write Python code for it?

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_elec, n_found = mydataset.motif.ii_elec_found, larger=1);
ax.set_title('Electrical synapses',  size=20);
ax.set_ylim(ymax=40);
ax.tick_params(labelsize=20)

fig.savefig('ii_elec.pdf')


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

Plot linear chains

Can you write Python code for it?

fig = figure()
ax = fig.add_subplot(111)

ax = barplot(simulation = n_chain, n_found = mydataset.motif.ii_lin_found, larger=False);
ax.set_title('Linear chains',  size=20);
ax.set_ylim(ymin=0, ymax=10);
ax.tick_params(labelsize=20)

fig.savefig('ii_chain.pdf')

#pii_chain # change this value in the plot!


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

C) Only electrical synapses *ii_elec*

Can you write Python code for it?

mydataset.motif['ii_elec']

PE = mydataset.motif.ii_elec_found /mydataset.motif.ii_elec_tested
PE

# definition of the null hypothese
# if this value is close to the simulation, we accept the null hypothese
(PE)*mydataset.motif.ii_elec_tested #


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

D) Unidirectionaly chemmical only *ii_chem*

Can you write Python code for it?

mydataset.motif['ii_chem']

PC = mydataset.motif.ii_chem_found /mydataset.motif.ii_chem_tested
PC

# definition of the null hypothese
# if this value is close to the simulation, we accept the null hypothese
(PC)*mydataset.motif.ii_chem_tested #


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

E) Bidirectional chemical connections only *ii_c2*

Can you write Python code for it?

mydataset.motif['ii_c2']

PC1 = mydataset.motif.ii_chem_found /mydataset.motif.ii_chem_tested
PC1

# definition of the null hypothese
# if this value is close to the simulation, we accept the null hypothese
(PC1*PC1)*mydataset.motif.ii_c2_tested

# calculate alfa_levels according to Zhao et al., 2001
alpha_rec = (PC1/(PC*PC))-1
print(alpha_rec)


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:

F) Divergent inhibitory connections *ii_div*

Can you write Python code for it?

mydataset.motif['ii_div']

Pdiv = mydataset.motif.ii_div_found /mydataset.motif.ii_div_tested
Pdiv

# definition of the null hypothese
# if this value is close to the simulation, we accept the null hypothese
(PC*PC)*mydataset.motif.ii_div_tested

# calculate alfa_levels according to Zhao et al., 2001
alpha_div = (Pdiv/(PC*PC))-1
print(alpha_div)


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
If the null hypothesis is correctly implemented, we should see almost the same
number of chemical synases as in the experiments.
Can you write Python code for it?

np.mean(n_chem) # on average the same number of unidirectional connections

mydataset.motif['ii_chem']['found']


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
If we found a number which is different form the empirical, we must revise our
null hypothese.
Can you write Python code for it?

np.mean(n_bichem) # on average the same number of bidirectional connections????


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
Let's see if the connections found correspond to the theoretical values for the complex motifs.

A) Unidirectional chemical connections in the presence of an electrical synapse *ii_c1e*

Can you write Python code for it?

mydataset.motif['ii_c1e']

PCE1 = mydataset.motif.ii_c1e_found /mydataset.motif.ii_c1e_tested
PCE1

# definition of the null hypothese
# if this value is close to the simulation, we accept the null hypothese
(PC*PE)*mydataset.motif.ii_c1e_tested


I am working on the file "Analysis/misc/Counting inhibitory connectivity motifs.ipynb".
The first task is:
Similarly, we must see almost the same
number of electrical connections as with the experiments
Can you write Python code for it?

np.mean(n_elec)

mydataset.motif.ii_elec_found # voila!


I am working on the file "Analysis/misc/Distance between a point and identity line.ipynb".
The first task is:

Distance from a point to a line

\begin{equation}
\frac{|Ax+By+C|}{\sqrt{A^2+B^2}}.
\end{equation}
Can you write Python code for it?

def distance(mypoint, myline):
    """
    Calculates the distance from a point to a line
    """

    x, y = mypoint

    A, B, C = myline

    return np.abs(A*x + B*y + C) / np.sqrt(np.power(A,2)+np.power(B,2))

mypoint = (5,1)
myline = (3,-1, 1)
distance(mypoint, myline) # 15/np.sqrt(10) = 4.743416

mypoint = (0.25, 0.50)
myline = (1, -1, 0) # identity line
distance(mypoint, myline)


I am working on the file "Analysis/misc/How to use DataLoader.ipynb".
The first task is:
<H2> Descriptive statistics </H2>
The stats attribute will return basis statistics of the whole dataset
Can you write Python code for it?

y = mydataset.stats()

print AsciiTable(y).table

mymotifs = mydataset.motif
info = [
        ['Connection type', 'Value'],
        ['PV-PV chemical synapses', mymotifs.ii_chem_found],
        ['PV-PV electrical synapses', mymotifs.ii_elec_found],
        [' ',' '],
        ['PV-PV bidirectional chemical', mymotifs.ii_c2_found],
        ['PV-PV divergent chemical', mymotifs.ii_div_found],
        ['PV-PV convergent chemical', mymotifs.ii_con_found],
        ['PV-PV linear chemical', mymotifs.ii_lin_found],
        [''],
        ['PV-PV one chemical with electrical', mymotifs.ii_c1e_found],
        ['PV-PV bidirectional chemical with electrical', mymotifs.ii_c2e_found],
        [' ',' '],
        ['P(PV-PV) chemical synapse', mymotifs.ii_chem_found/mymotifs.ii_chem_tested],
        ['P(PV-PV) electrical synapse', mymotifs.ii_elec_found/mymotifs.ii_elec_tested],
        [''],
        ['P(PV-PV) bidirectional chemical synapse', mymotifs.ii_c2_found/mymotifs.ii_c2_tested],
        ['P(div) divergent chemical motifs', mymotifs.ii_div_found/mymotifs.ii_div_tested],
        ['P(div) convergent chemical motifs', mymotifs.ii_con_found/mymotifs.ii_con_tested],
        ['P(chain) linear chain motifs', mymotifs.ii_lin_found/mymotifs.ii_lin_tested],
        [' ',' '],
        ['P(PV-PV) one chemical with electrical', mymotifs.ii_c1e_found/mymotifs.ii_c1e_tested],
        ['P(PV-PV) bidirectional chemical with electrical', mymotifs.ii_c2e_found/mymotifs.ii_c2e_tested],
        [' ',' '],
        ['PV-GC chemical synapses', mymotifs.ie_found],
        ['GC-PC chemical synapses', mymotifs.ei_found],
        [' ',' '],
        ['P(PV-GC) chemical synapse',mymotifs.ie_found/mymotifs.ie_tested],
        ['P(GC-PC) chemical synapse', mymotifs.ei_found/mymotifs.ei_tested],
        [' ',' '],
        ]

table = AsciiTable(info)
print (table.table)


I am working on the file "Analysis/misc/How to use DataLoader.ipynb".
The first task is:

Object atttributes

The object contains a list with all experiments loaded
Can you write Python code for it?

len(mydataset.experiment)

mydataset.nIN, mydataset.nPC # number of PV cells and GC cells recorded

mydataset.configuration # number of recording configurations

print(mydataset.motif) # number of connections tested and found for every type


I am working on the file "Analysis/misc/How to use DataLoader.ipynb".
The first task is:

Object creation

The object loads the connectivity matrices in .syn format and reports the number of files loaded at construction.
Can you write Python code for it?

mydataset = DataLoader('../../data/PV') # create an object with information of all connections


I am working on the file "Analysis/misc/How to use DataLoader.ipynb".
The first task is:
Details of every experiments are given in a list
Can you write Python code for it?

mydataset.experiment[0] # example of the data from the first experiment


I am working on the file "Analysis/misc/How to use DataLoader.ipynb".
The first task is:
and details fo the recording configurations are provided
Can you write Python code for it?

mydataset.experiment[12]['fname'] # mydataset.filename(12)

mydataset.filename(12)

mydataset.experiment[12]['matrix']

mydataset.matrix(12)

print(mydataset.experiment[12]['motif'])

mydataset.motifs(12)


I am working on the file "Analysis/misc/Patterns.ipynb".
The first task is:
<H2> Percentage of separated patterns</H2>
It is the number of patterns whose input similarity is larger than the output similarity.
Can you write Python code for it?

in_sim = np.array([similarity(x,y) for x,y in inputpattern])
out_sim = np.array([similarity(x,y) for x,y in outpattern])

separated = in_sim[in_sim>out_sim]
prop = len(separated)/len(in_sim)
print('Percentage of separated patterns {:2.2f} %.'.format(prop*100))


I am working on the file "Analysis/misc/Patterns.ipynb".
The first task is:

Definition of random pattern $\boldsymbol{z}$

A memory random pattern $\boldsymbol{z}$ is the representation of the activity of all the neurons in the network. It is a vector of lenght n, being n the total number of neurons in the network.

$\boldsymbol{z}_i: i = {0, 1, \cdots, n},$

The i-th element of the pattern is an independent random variable of probability a to be one when the neuron is active,  and zero if not (1-a).

\begin{equation}
    \Pr(\boldsymbol{z}_i) =
        \begin{cases}
        1, & \text{if} \ \ \boldsymbol{z}_i < a,\
        0, & \text{if} \ \ \boldsymbol{z}_i \geq 1-a\
        \end{cases}
\end{equation}
Can you write Python code for it?

def randpattern(size, prob):
    """
    Creates a pattern of activities (ones) with a given probability.
    Activity is 1 if the cell is active, zero otherwise.

    Parameters
    ----------
    size : ndarray
           vector size
    prob : float
            probability of having ones
    """
    z = np.zeros(size, dtype=int)
    mysize = int(size*prob)

    z[np.random.choice(size, mysize, replace=False)]=1 # without replacement. Thank you Clau!!!!

    return(z)


I am working on the file "Analysis/misc/Patterns.ipynb".
The first task is:

Magnitude of separation

Is the averge distance to the identity line from the patterns that were separated
Can you write Python code for it?

point = [(x,y) for x,y in zip(out_sim, in_sim) if x<y] # patterns separated
# test that the proportion remains the same
prop = len(point)/len(in_sim)
print('Percentage of separated patterns {:2.2f} %'.format(prop*100))

dist = list()
for x,y in point:
    dist.append( np.abs(x-y)/np.sqrt(2))

mean_separation = np.mean(dist)
max_separation = (mean_separation*100)/(1/np.sqrt(2))
print('Magnitude of separation {:2.4f} '.format(mean_separation))

print('Percentage of max separation {:2.4f} %'.format(max_separation))

fig = plt.figure()
ax = fig.add_subplot(111)

ax = plot_pattern_separation(inputpattern, outpattern)


I am working on the file "Analysis/misc/Patterns.ipynb".
The first task is:

Vector norm

The most commonly encountered vector norm (often simply called "the norm" of a vector, or sometimes the magnitude of a vector) is the L2-norm, given by
\begin{equation}
\left\| \boldsymbol{z} \right\|_2 := \sqrt{z_1^2 + \cdots + z_n^2}.\end{equation}
Can you write Python code for it?

z = randpattern(size=10, prob=0.5) # example for testing
mycount = list()
for _ in range(100):
    mycount.append(np.count_nonzero(randpattern(10,.5)) )

np.mean(mycount)

np.linalg.norm(z, ord=2) #norm2 of a vector

np.linalg.norm([1,1],2) # must be sqrt(2)


I am working on the file "Analysis/misc/Slicing matrices.ipynb".
The first task is:

Select connections from excitatory to inhibitory neurons (EI)

Can you write Python code for it?

utils.EI_slice(matrix,2)

matrix[2:][:,(0,1)] # EI = matrix[nPV:][,:range(nPV)]


I am working on the file "Analysis/misc/Slicing matrices.ipynb".
The first task is:
<H2>Select only reciprocal inhibition (II) </H2>
Reciprocal connections are bidirectional connections between PV interneurons
Can you write Python code for it?

matrix[:2][:,(0,1)] # II = matrix[ :nPV][ :,range(nPV)]

utils.II_slice(matrix,2)


I am working on the file "Analysis/regression.ipynb".
The first task is:
The following function runs a random model with a random independent variable y and four random covariates, using both the statsmodels and scikit-learn packages. The user can compare output from the two tools.
Can you write Python code for it?

from regression import compare_OLS
compare_OLS(dat)


I am working on the file "Analysis/regression.ipynb".
The first task is:
The two models produce the same results.
There is no standard regression table type output from sklearn. However, sklearn offers greater features for prediction, by incorporating machine learning functionality. For that reason, we will likely wish to use both packages, for different purposes.
The user_model function prompts the user to input a model formula for an OLS regression, then runs the model in statsmodel, and outputs model results and a plot of y data vs. model fitted values.
At the prompt, you may either input your own model formula, or copy and paste the following formula as an example:
CO2uM ~ pH + TempC + ChlAugL
Can you write Python code for it?

%matplotlib inline
from regression import user_model
user_model(data=dat)

%matplotlib inline
import pandas as pd
from regression import plot_pairs
plot_pairs(data=dat[['XCO2Dpp', 'XCH4Dpp', 'TempC', 'ChlAugL', 'TurbFNU',
       'fDOMQSU', 'ODOmgL', 'pH', 'CH4uM', 'CO2uM']], minCorr=0.1, maxCorr=0.95)

dat.columns

dat.shape[1]


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Cluster identification
After dimensionality reduction, clusters are identified using a variety of approaches. We will use a simple algorithm called DBSCAN to identify clusters
This jupyter notebook contains 1 functions for dimensionality reduction:
1. DBSCAN
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Dimensionality reduction
After gene selection, the high dimensionality of single cell RNA-seq data is commnly reduced to cluster similar cells together.
This jupyter notebook contains 2 functions for dimensionality reduction:
1. PCA
2. tSNE - for the purposes of the demonstration, we will use tSNE and reduce data to 3 dimensions
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Feature Selection
One of the most important steps in single cell RNA seq processing, is selecting genes that describe most of the biological variance. However, this is confounded by the high levels of technical noise associated with single cell RNA-seq data.
This jupyter notebook contains 2 functions to enable feature selection:
1. variance - select the top variable genes in the dataset
2. overdispersion - select the top variable genes in the dataset corrected for technical variance
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Make .Rds file for CellView
And finally, let's summarize this analysis into an .Rds file that we can share with others
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Marker Identification
Identifying genes that differentiate each of these cell populations is an important aspect of single cell RNA-seq data. There are many different methods to do this type of analysis. Given the size of the dataset ome of these are compute heavy. For the sake of brevity, we will use AUROC classification of differentially expressed genes.
Can you write Python code for it?

aurocScoresAll=pd.DataFrame()
for cluster in range(1,8,1):

    cells1=tsnedata_dbscan[tsnedata_dbscan['dbCluster']==cluster].index
    cells2=tsnedata_dbscan.index.difference(cells1)

    data1=cpt.loc[cpt.index,cells1].apply(expMean,1)
    data2=cpt.loc[cpt.index,cells2].apply(expMean,1)
    totaldiff=(data1-data2)
    genes=totaldiff[totaldiff>1.].index

    aurocScores=findMarkers(cpt,
                  cells1,
                  cells2,
                  genes
                 )
    aurocScores['Associated.Gene.Name']=featuredata['Associated.Gene.Name']
    aurocScores['dbCluster']=cluster
    aurocScoresAll=aurocScoresAll.append(aurocScores)


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Read Input data
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Shape of dataset: Genes, Cells
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Visualization
Let's make a heatmap of all markergenes
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Visualization
Lets take a look at our dimensionality reduction by plotting cells, but this time color each cell by the cluster assignment as determined by DBSCAN
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Visualization
Visualization is an important part of an single cell experiment. Exploring data with genes of interest helps validate clustering as wells begins the process of identifying the cell type of each cluster
Lets take a look at our dimensionality reduction by plotting cells, but this time color each cell by the expression of particular gene. Pick from Emcn, Olig1, Olig2, Pdgra, Fyn, Aqp4,Mog,Slc32a1,Slc17a6,Cx3cr1.
Can you write Python code for it?


I am working on the file "AnalysisNotebook.ipynb".
The first task is:
Visualization
Visualization is an important part of an single cell experiment. Exploring data with genes of interest helps validate clustering as wells begins the process of identifying the cell type of each cluster
Lets take a look at our dimensionality reduction by plotting cells.
Can you write Python code for it?


I am working on the file "Analysis_2020/Match_NvsO_SSCv2.ipynb".
The first task is:
**SELECT GOOD G- AND I-MAG OBJECTS
These are the three mags used in the comparison with Gaia
g-mag and g-err
Can you write Python code for it?

# get a df of obj with good phot
# g_mag > 0
new_df3 = new_df2[(new_df2['Ng_mMed'] > 0)]
print(new_df3.shape)

nrows3,ncols3 = new_df3.shape
print('New df, GMAG > 0: num rows, cols: ',nrows3,ncols3)
outlog.write('*************************\n')
outlog.write('New df, GMAG > 0: num rows %d, cols %d\n' % (nrows3,ncols3));

# g_err
new_df3 = new_df3[(new_df3['Ng_mErr'] <= max_MAGerr)]
print(new_df3.shape)

nrows3,ncols3 = new_df3.shape
print('New df, GERR < MAX ERR: num rows, cols: ',nrows3,ncols3)
outlog.write('*************************\n')
outlog.write('New df, GERR < MAX ERR: num rows %d, cols %d\n' % (nrows3,ncols3));


I am working on the file "Analysis_2020/Match_NvsO_SSCv2.ipynb".
The first task is:
DEFINE INPUT FILENAMES AND PROG CONSTANTS
Can you write Python code for it?

# NEPOCH THRESHOLD FOR CLEANING NEW SSC
IZ_low_epoch = 5

# MATCH RAD FOR PYSPHEREMATCH
tol_asec = 2. # matching radius in arc.sec
tol_deg = tol_asec/3600.

# MAX PERMITTED MAG ERROR
max_MAGerr = 0.05

# The new SSC
nSSC = 'NEW_stripe82calibStars_v0.dat'

# The new PREMATCH SSC
nSSC_prematch = 'NEW_stripe82calibStars_prematch.csv'

# The new MATCHED SSC
nSSC_matched = 'NEW_stripe82calibStars_matched.csv'

# The old SSC
oSSC = 'stripe82calibStars_v2.6.dat'

# The old MATCHED SSC
oSSC_matched = 'stripe82calibStars_matched.csv'


I am working on the file "Analysis_2020/Match_NvsO_SSCv2.ipynb".
The first task is:
LIST OF NAMES OF DF AND THEIR CUTS
new_df = as read in KT 2020 cat
new_df1 = after removing null and low nepoch obj
new_df2 = select good r-mag and r-err
new_df3 = select good g-mag and g-err
new_df4 = select good i-mag and i-err

old_df = as read in ZI 2007 cat

new_prematch_df = prior to matching cleaned 2020
old_prematch_df = prior to matching as read 2007

new_matched_df = matched new df
old_matched_df = matched old df

The following df are written out to csv files
new_prematch_df
new_matched_df
old_matched_df

OPEN A LOG FILE
BEFORE QUITTING
REMEMBER TO RUN THE LAST CELL TO CLOSE THE LOG FILE
Can you write Python code for it?

log_fil = 'Match_NvsO_SSCv2.log'
outlog = open(log_fil,'w')

from datetime import date

today = date.today()
outlog.write("Running Match_NvsO_SSCv1 on %s\n" % (today));


I am working on the file "Analysis_2020/Match_NvsO_SSCv2.ipynb".
The first task is:
READ IN THE NEW SSC INTO DASK DF
Can you write Python code for it?

%%time
Ncolnames = ['Ncalib_fla', 'Nra', 'Ndec', 'NraRMS', 'NdecRMS', 'NnEpochs', 'NAR_val',
                'Nu_Nobs', 'Nu_mMed', 'Nu_mMean', 'Nu_mErr', 'Nu_rms_scatt', 'Nu_chi2',
                'Ng_Nobs', 'Ng_mMed', 'Ng_mMean', 'Ng_mErr', 'Ng_rms_scatt', 'Ng_chi2',
                'Nr_Nobs', 'Nr_mMed', 'Nr_mMean', 'Nr_mErr', 'Nr_rms_scatt', 'Nr_chi2',
                'Ni_Nobs', 'Ni_mMed', 'Ni_mMean', 'Ni_mErr', 'Ni_rms_scatt', 'Ni_chi2',
                'Nz_Nobs', 'Nz_mMed', 'Nz_mMean', 'Nz_mErr', 'Nz_rms_scatt', 'Nz_chi2']

# USING DASK DATAFRAME
df_nSSC = dd.read_csv(nSSC,delimiter=",",header=None,names=Ncolnames,
                      assume_missing=True,low_memory=False,comment='#')
new_df = df_nSSC.compute()
new_df.head()
nrows,ncols = new_df.shape
print('New df, as read: num rows, cols: ',nrows,ncols)
outlog.write('New df, as read: num rows %d, cols %d\n' % (nrows,ncols));


I am working on the file "Analysis_2020/Match_NvsO_SSCv2.ipynb".
The first task is:
SAVE PREMATCH NEW DF TO CSV
Can you write Python code for it?

%%time
paths = nSSC_prematch
prematch_csv = new_prematch_df.to_csv(paths,na_rep=-99.99,index=False)

print('Prematch csv: ',nSSC_prematch)
outlog.write('*************************\n')
outlog.write('Prematch csv %s\n' % nSSC_prematch);


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
ALSO WRITE OUT ONLY THE SDSS-GAIA MAGS/ERRS
MAKE A SUBSET OF THE DF WITH ONLY GAIA RA, DEC, SDSS GRIZ MED MAGS, ERRS, GAIA GMAG, FLUX AND ERR
Can you write Python code for it?

# GET A SUBSET OF MAG/ERR COLS ONLY
cols_needed = ['ra_gaia','dec_gaia','Gmag','flux','fluxErr',
               'g_mMed','g_mErr','r_mMed','r_mErr','i_mMed','i_mErr','z_mMed','z_mErr']
gaia_matchedOK2_subset = gaia_matchedOK2[cols_needed]

# WRITE OUT A CSV FILE
paths = SDSS2GAIA_magerr
gaia_matchedOK2_subset.write(paths,format='ascii.csv',
                      delimiter=',', comment='#',overwrite=True)


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
CCD gi vs Gr
Can you write Python code for it?

# plot
fig,ax = plt.subplots(1,1,figsize=(8,6))
ax.scatter(gi_all, Gr_all, s=0.01, c='green')
ax.scatter(gi_pk1, Gr_pk1, s=0.01, c='blue')
ax.scatter(gi_pk2, Gr_pk2, s=0.01, c='red')
ax.set_xlim(-0.5,3.5)
ax.set_ylim(-1.5,1.)
ax.set_xlabel('SDSS(g-i)')
ax.set_ylabel('Gaia G - SDSS r')
ax.set_title('CCD: gi vs Gr all-pk1-pk2',fontdict={'fontsize': 14, 'fontweight': 'bold'})


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
CCD gr vs ri
Can you write Python code for it?

# plot
fig,ax = plt.subplots(1,1,figsize=(8,6))
ax.scatter(gr_all, ri_all, s=0.01, c='green')
ax.scatter(gr_pk1, ri_pk1, s=0.01, c='blue')
ax.scatter(gr_pk2, ri_pk2, s=0.01, c='red')
ax.set_xlim(-0.7,2.2)
ax.set_ylim(-0.7,2.4)
ax.set_xlabel('SDSS(g-r)')
ax.set_ylabel('SDSS(r-i)')
ax.set_title('CCD: gr vs ri all-pk1-pk2',fontdict={'fontsize': 14, 'fontweight': 'bold'})


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
DEFINE PROG CONSTS
Can you write Python code for it?

# CONVERT IQD TO STD DEV
IQD2STD = 0.741

# GAIA ZEROPOINT
GAIA_ZP = 25.525

# DEFINE POLYNOMIAL DEGREES
deg1,deg2,deg3,deg5,deg7 = 1,2,3,5,7


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
FUNCTION DEFINITIONS
fitMedians for fitting med and std dev in bins over a x,y distribution
Can you write Python code for it?

# given vectors x and y, fit medians in bins from xMin to xMax, with Nbin steps,
# and return xBin, medianBin, medianErrBin
def fitMedians(x, y, xMin, xMax, Nbin, verbose=1):

    # first generate bins
    xEdge = np.linspace(xMin, xMax, (Nbin+1))
    xBin = np.linspace(0, 1, Nbin)
    nPts = 0*np.linspace(0, 1, Nbin)
    medianBin = 0*np.linspace(0, 1, Nbin)
    sigGbin = -1+0*np.linspace(0, 1, Nbin)
    for i in range(0, Nbin):
        xBin[i] = 0.5*(xEdge[i]+xEdge[i+1])
        yAux = y[(x>xEdge[i])&(x<=xEdge[i+1])]
        if (yAux.size > 0):
            nPts[i] = yAux.size
            medianBin[i] = np.median(yAux)
            # robust estimate of standard deviation: 0.741*(q75-q25)
            sigmaG = 0.741*(np.percentile(yAux,75)-np.percentile(yAux,25))
            # uncertainty of the median: sqrt(pi/2)*st.dev/sqrt(N)
            sigGbin[i] = np.sqrt(np.pi/2)*sigmaG/np.sqrt(nPts[i])
        else:
            nPts[i], medianBin[i], sigGBin[i] = 0

    if (verbose):
        print('median:', np.median(medianBin[Npts>0]))

    return xBin, nPts, medianBin, sigGbin


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
GET ALL THE REQUIRED QUANTITIES

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

_all
Can you write Python code for it?

r_all = gaia_sdss['r_mMed']
G_all = gaia_sdss['Gmag']

# sdss colors
gr_all = gaia_sdss['g_mMed'] - gaia_sdss['r_mMed']
gi_all = gaia_sdss['g_mMed'] - gaia_sdss['i_mMed']
gz_all = gaia_sdss['g_mMed'] - gaia_sdss['z_mMed']
ri_all = gaia_sdss['r_mMed'] - gaia_sdss['i_mMed']
# Gaia colors
Gg_all = gaia_sdss['Gmag'] - gaia_sdss['g_mMed']
Gr_all = gaia_sdss['Gmag'] - gaia_sdss['r_mMed']
Gi_all = gaia_sdss['Gmag'] - gaia_sdss['i_mMed']

ra_all = gaia_sdss['ra_gaia']
raW_all = np.where(ra_all > 180, ra_all-360, ra_all)
dec_all = gaia_sdss['dec_gaia']


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
PICK ALL THE REQUIRED QUANTITIES WITH SELECTION I

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

_pk1
Can you write Python code for it?

## MAGS AND COLORS
r_pk1 = gaia_matched['r_mMed']
G_pk1 = gaia_matched['Gmag']
# sdss colors
gr_pk1 = gaia_matched['g_mMed'] - gaia_matched['r_mMed']
gi_pk1 = gaia_matched['g_mMed'] - gaia_matched['i_mMed']
gz_pk1 = gaia_matched['g_mMed'] - gaia_matched['z_mMed']
ri_pk1 = gaia_matched['r_mMed'] - gaia_matched['i_mMed']
# gaia colors
Gg_pk1 = gaia_matched['Gmag'] - gaia_matched['g_mMed']
Gr_pk1 = gaia_matched['Gmag'] - gaia_matched['r_mMed']
Gi_pk1 = gaia_matched['Gmag'] - gaia_matched['i_mMed']

# POSITIONS BASED ON GAIA
ra_pk1 = gaia_matched['ra_gaia']
raW_pk1 = np.where(ra_pk1 > 180, ra_pk1-360, ra_pk1)
dec_pk1 = gaia_matched['dec_gaia']


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
PICK ALL THE REQUIRED QUANTITIES WITH SELECTION II**

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

_pk2
Can you write Python code for it?

g_pk2 = gaia_matchedOK['g_mMed']
r_pk2 = gaia_matchedOK['r_mMed']
i_pk2 = gaia_matchedOK['i_mMed']
z_pk2 = gaia_matchedOK['z_mMed']

G_pk2 = gaia_matchedOK['Gmag']
# sdss colors
gr_pk2 = gaia_matchedOK['g_mMed'] - gaia_matchedOK['r_mMed']
gi_pk2 = gaia_matchedOK['g_mMed'] - gaia_matchedOK['i_mMed']
gz_pk2 = gaia_matchedOK['g_mMed'] - gaia_matchedOK['z_mMed']
ri_pk2 = gaia_matchedOK['r_mMed'] - gaia_matchedOK['i_mMed']
# Gaia colors
Gg_pk2 = gaia_matchedOK['Gmag'] - gaia_matchedOK['g_mMed']
Gr_pk2 = gaia_matchedOK['Gmag'] - gaia_matchedOK['r_mMed']
Gi_pk2 = gaia_matchedOK['Gmag'] - gaia_matchedOK['i_mMed']

ra_pk2 = gaia_matchedOK['ra_gaia']
raW_pk2 = np.where(ra_pk2 > 180, ra_pk2-360, ra_pk2)
dec_pk2 = gaia_matchedOK['dec_gaia']


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
PICK ALL THE REQUIRED QUANTITIES WITH SELECTION III

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

NOTE Also get the mag errs for sdss griz and Gaia Gmag. These are needed for the next selection cut
_pk3
Can you write Python code for it?

g_pk3 = gaia_matchedOK2['g_mMed']
r_pk3 = gaia_matchedOK2['r_mMed']
i_pk3 = gaia_matchedOK2['i_mMed']
z_pk3 = gaia_matchedOK2['z_mMed']

G_pk3 = gaia_matchedOK2['Gmag']

# also get mag errors for the next selection cut
# Gaia err
Gflux_pk3 = gaia_matchedOK2['flux']
Gfluxerr_pk3 = gaia_matchedOK2['fluxErr']
G_err_pk3 = -2.5*np.log10(1.+ (Gfluxerr_pk3/Gflux_pk3))

# sdss errs
g_err_pk3 = gaia_matchedOK2['g_mErr']
r_err_pk3 = gaia_matchedOK2['r_mErr']
i_err_pk3 = gaia_matchedOK2['i_mErr']
z_err_pk3 = gaia_matchedOK2['z_mErr']

# sdss colors
gr_pk3 = gaia_matchedOK2['g_mMed'] - gaia_matchedOK2['r_mMed']
gi_pk3 = gaia_matchedOK2['g_mMed'] - gaia_matchedOK2['i_mMed']
gz_pk3 = gaia_matchedOK2['g_mMed'] - gaia_matchedOK2['z_mMed']
ri_pk3 = gaia_matchedOK2['r_mMed'] - gaia_matchedOK2['i_mMed']
# Gaia colors
Gg_pk3 = gaia_matchedOK2['Gmag'] - gaia_matchedOK2['g_mMed']
Gr_pk3 = gaia_matchedOK2['Gmag'] - gaia_matchedOK2['r_mMed']
Gi_pk3 = gaia_matchedOK2['Gmag'] - gaia_matchedOK2['i_mMed']

ra_pk3 = gaia_matchedOK2['ra_gaia']
raW_pk3 = np.where(ra_pk3 > 180, ra_pk3-360, ra_pk3)
dec_pk3 = gaia_matchedOK2['dec_gaia']


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
PLOTS
ONLY CCD
Can you write Python code for it?

# FUNCTION TO PLOT THE CCD, FIT THE POLY, AND OVERPLOT THE FITTED VALUES
def CCDpltNfit(clr1a,clr1b,clr1c,clr1d,clr2a,clr2b,clr2c,clr2d,labls,
               xlim,ylim,ax_labl,plt_titl,fitclr1,fitclr2,Binz,degr=3):
    # clr1a,clr1b,clr1c,clr1d = the four cuts on clr on x-axis
    # clr2a,clr2b,clr2c,clr2d = the four cuts on clr on y-axis
    # labls = labels for the datasets given as a 4-element list
    # xlim, ylim = plot limits on x and y as 2-element lists
    # ax_labl,plt_titl = axes labels, and plot title
    # fitclr1,fitclr2 = cut on clr1, clr2 with which to comp medians and fit poly
    # Binz = minx, maxx and nbins for fitting range
    # degr = degree of fit poly, default = 3

    # plot the base CCD
    fig,ax = plt.subplots(1,1,figsize=(8,6))
    ax.scatter(clr1a,clr2a, s=0.01, c='orange',label=labls[0])
    ax.scatter(clr1b,clr2b, s=0.01, c='green',label=labls[1])
    ax.scatter(clr1c,clr2c, s=0.01, c='blue',label=labls[2])
    ax.scatter(clr1d,clr2d, s=0.01, c='red',label=labls[3])
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)
    ax.set_xlabel(ax_labl[0])
    ax.set_ylabel(ax_labl[1])
    ax.set_title(plt_titl,fontdict={'fontsize': 14, 'fontweight': 'bold'})
    ax.legend(loc='best')
    ax.grid(True)

    plt.show()

    # now fit the medians to the specified data set
    minx,maxx,nbin = Binz[0],Binz[1],Binz[2]
    xBin, nPts, medBin, sigBin = fitMedians(fitclr1,fitclr2,minx,maxx,nbin,0)

    # now fit the poly of spec degr
    data = np.array([xBin,medBin,sigBin])
    theta = best_theta(data,degr)

    return xBin, nPts, medBin, sigBin,theta

clr1a,clr1b,clr1c,clr1d = gr_pk1,gr_pk2,gr_pk3,gr_pk4
clr2a,clr2b,clr2c,clr2d = Gg_pk1,Gg_pk2,Gg_pk3,Gg_pk4
labls = ['pk1','pk2','pk3','pk4']
xlim = (-0.6,1.8)
ylim = (-3.0,0.5)
ax_labl = ['(g-r)','(G-g)']
plt_titl = 'CCD SDSS gr VS Gaia Gg'
fitclr1,fitclr2 = gr_pk4,Gg_pk4
Binz = [0.2,1.2,20]
xBin, nPts, medBin, sigBin,theta = CCDpltNfit(clr1a,clr1b,clr1c,clr1d,clr2a,clr2b,clr2c,clr2d,labls,
              ...


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
Read the sdss and gaia catalogs
Goes quite slowly, consider DASK?
Can you write Python code for it?

%%time
colnames = ['calib_fla', 'ra', 'dec', 'raRMS', 'decRMS', 'nEpochs', 'AR_val',
                'u_Nobs', 'u_mMed', 'u_mMean', 'u_mErr', 'u_rms_scatt', 'u_chi2',
                'g_Nobs', 'g_mMed', 'g_mMean', 'g_mErr', 'g_rms_scatt', 'g_chi2',
                'r_Nobs', 'r_mMed', 'r_mMean', 'r_mErr', 'r_rms_scatt', 'r_chi2',
                'i_Nobs', 'i_mMed', 'i_mMean', 'i_mErr', 'i_rms_scatt', 'i_chi2',
                'z_Nobs', 'z_mMed', 'z_mMean', 'z_mErr', 'z_rms_scatt', 'z_chi2']

# sdss = Table.read(SDSS_CAT, format='ascii', names=colnames)
sdss = Table.read(SDSS_CAT, format='csv', fast_reader=False, names=colnames)

# USING DASK DATAFRAME
# df_nSSC = dd.read_csv(SDSS_CAT,delimiter=",",header=None,names=colnames,
#                      assume_missing=True,low_memory=False,comment='#')
# sdss = df_nSSC.compute()
# sdss.head()
# Snrows,Sncols = sdss.shape
# print('SDSS, as read: num rows, cols: ',Snrows,Sncols)
#outlog.write('New df, as read: num rows %d, cols %d\n' % (nrows,ncols));

%%time
colnames = ['ra', 'dec', 'nObs', 'Gmag', 'flux', 'fluxErr']
# gaia = Table.read('Stripe82_GaiaDR1_small.dat', format='ascii', names=colnames)
gaia = Table.read(GAIA_CAT, format='ascii', names=colnames)

# USING DASK DATAFRAME
# df_gaia = dd.read_csv(GAIA_CAT,sep="\s+",
#                       header=None,names=colnames,
#                       assume_missing=True,low_memory=False,comment='#')
#df_gaia = dd.read_table(GAIA_CAT,sep="\s+",
#                        header=None,names=colnames,
#                        assume_missing=True,low_memory=False,comment='#')

#gaia = df_gaia.compute()
# gaia = gaia_table.to_pandas()
# gaia.head()
# Gnrows,Gncols = gaia.shape
# print('Gaia, as read: num rows, cols: ',Gnrows,Gncols)


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
SELECTION II: BY RA, RMAG AND SDSS GI COLOR
flagOK = ((raW_pk1 > -10) & (raW_pk1 < 50) & (r_pk1>16) & (r_pk1<19) & (gi_pk1>0) & (gi_pk1<3.0))
gaia_matchedOK
Can you write Python code for it?

# flagOK = ((raW > -10) & (raW < 50) & (rMed>15) & (rMed<20))
# flagOK = ((raW > -10) & (raW < 50) & (rMed>16) & (rMed<19))
flagOK = ((raW_pk1 > -10) & (raW_pk1 < 50) & (r_pk1>16) & (r_pk1<19) & (gi_pk1>0) & (gi_pk1<3.0))
gaia_matchedOK = gaia_matched[flagOK]
print('Num obj after select by RA, rMag, gi color: %d' % (len(gaia_matchedOK)))


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
USE POLYNOMIAL FITS OF DIFFERENT ORDERS
USE _PK2
ORDERS TESTED: 3,5,7
Can you write Python code for it?

%%time
data = np.array([xBinOK2, medianBinOK2, sigGbinOK2])
# x, y, sigma_y = data
Ndata = xBinOK2.size
# get best-fit parameters for linear, quadratic and cubic models
theta1 = best_theta(data,deg3)
theta2 = best_theta(data,deg5)
theta3 = best_theta(data,deg7)


I am working on the file "Analysis_2020/NEWsdss_gaia_matching_v0.ipynb".
The first task is:
best_theta = fits a polynomial for a given (x, y, sigma_y)
Can you write Python code for it?

# this function computes polynomial models given some data x
# and parameters theta
def polynomial_fit(theta, x):
    """Polynomial model of degree (len(theta) - 1)"""
    return sum(t * x ** n for (n, t) in enumerate(theta))

# compute the data log-likelihood given a model
def logLv0(data, theta, model=polynomial_fit):
    """Gaussian log-likelihood of the model at theta"""
    x, y, sigma_y = data
    y_fit = model(theta, x)
    return sum(stats.norm.logpdf(*args)
               for args in zip(y, y_fit, sigma_y))

# a direct optimization approach is used to get best model
# parameters (which minimize -logL)
def best_theta(data, degree, model=polynomial_fit):
    theta_0 = (degree + 1) * [0]
    neg_logLv0 = lambda theta: -logLv0(data, theta, model)
    return optimize.fmin_bfgs(neg_logLv0, theta_0, disp=False)


I am working on the file "Analysis_2020/compSDSS_PS_KTv2.ipynb".
The first task is:
Define header, format, etc
NOTE: PS1 header is given in the first row
Can you write Python code for it?

colnames = ['ra','dec','nEpochs','g_Nobs','g_mMed','g_mErr',
            'r_Nobs','r_mMed','r_mErr','i_Nobs','i_mMed','i_mErr',
            'z_Nobs','z_mMed','z_mErr','raMean','decMean',
            'gMeanPSFMag','gMeanPSFMagErr','gMeanPSFMagNpt',
            'rMeanPSFMag','rMeanPSFMagErr','rMeanPSFMagNpt',
            'iMeanPSFMag','iMeanPSFMagErr','iMeanPSFMagNpt',
            'zMeanPSFMag','zMeanPSFMagErr','zMeanPSFMagNpt']

Ndtype = {'nEpochs':'int64','g_Nobs':'int64',\
          'r_Nobs':'int64','i_Nobs':'int64','z_Nobs':'int64',\
          'gMeanPSFMagNpt':'int64','rMeanPSFMagNpt':'int64',\
          'iMeanPSFMagNpt':'int64','zMeanPSFMagNpt':'int64'}


I am working on the file "Analysis_2020/compSDSS_PS_KTv2.ipynb".
The first task is:
Define prog constants
Can you write Python code for it?

# MATCH RAD FOR PYSPHEREMATCH
tol_asec = 3. # matching radius in arc.sec
tol_deg = tol_asec/3600.

# MAX PERMITTED MAG ERROR
max_MAGerr = 0.05


I am working on the file "Analysis_2020/compSDSS_PS_KTv2.ipynb".
The first task is:
Function to plot one color (y-axis) versus various others pars (x-axis)
Can you write Python code for it?

def doOneColor(d, kw):
    print('=========== WORKING ON:', kw['Ystr'], '===================')
    xVec = d[kw['Xstr']]
    yVec = d[kw['Ystr']]
    #
    # this is where the useable range of color specified by Xstr is defined
    # it's really a hack - these limits should be passed via kw...
    xMin = 0.5
    xMax = 3.1
    nBinX = 52
    xBin, nPts, medianBin, sigGbin = ktt.fitMedians(xVec, yVec, xMin, xMax,nBinX, 0)
    fig,ax = plt.subplots(1,1,figsize=(8,6))
    # This is for a scatter plot
    # ax.scatter(xVec, yVec, s=0.01, c='blue')
    # Replace with hist2D
    # get x,y-binning
    yMax,yMin = np.quantile(yVec, [0.99,0.01])
    yMax,yMin,nBinY = round(yMax,2),round(yMin,2),25
    xedges = np.linspace(xMin,xMax,nBinX+1,endpoint=False,dtype=float)
    yedges = np.linspace(yMin,yMax,nBinY+1,endpoint=False,dtype=float)
    Histo2D, xedges, yedges = np.histogram2d(xVec,yVec, bins=(xedges, yedges))
    Histo2D = Histo2D.T
    X, Y = np.meshgrid(xedges, yedges)
    # cs = ax.pcolormesh(X,Y, Histo2D, cmap='Greys')
    # cs = ax.pcolormesh(X,Y, Histo2D, cmap='plasma')
    cs = ax.pcolormesh(X,Y, Histo2D, cmap=kw['cmap'])

    #ax.scatter(xBin, medianBin, s=5.2, c='yellow')
    ax.scatter(xBin, medianBin, s=5.2, c='black', marker='+')
    # ax.set_xlim(0.4,3.2)
    # ax.set_ylim(-0.5,0.5)
    ax.set_xlim(xMin,xMax)
    ax.set_ylim(yMin,yMax)
    ax.set_xlabel(kw['Xstr'])
    ax.set_ylabel(kw['Ystr'])
    fig.colorbar(cs, ax=ax, shrink=0.9)
    # THERE IS NO ANALYTIC COLOR TERM: linear interpolation of the binned medians!
    d['colorfit'] = np.interp(xVec, xBin, medianBin)
    # the following line corrects the trend given by the binned medians
    d['colorresid'] = d[kw['Ystr']] - d['colorfit']
    # note that we only use the restricted range of color for ZP analysis
    # 0.3 mag limit is to reject gross outliers
    goodC = d[(np.abs(d['colorresid'])<0.3)&(xVec>xMin)&(xVec<xMax)]

    ### plots
    # RA
    print(' stats for RA binning medians:')
    plotNameRoot = kw['plotNameRoot'] +...


I am working on the file "Analysis_2020/compSDSS_PS_KTv2.ipynb".
The first task is:
Get value added cols
IMPORTANT: ORIGINAL DF IS MODIFIED IN THIS CELL
Can you write Python code for it?


I am working on the file "Analysis_2020/compSDSS_PS_KTv2.ipynb".
The first task is:
Plot color offsets versus (g-i) color, also against ra, dec
Can you write Python code for it?

# These keywords what you want to plot on x
keywords = {"Xstr":'gi', "plotNameRoot":'colorResidPS12_'}

# These keywords are for plot style
keywords["symbSize"] = 0.05
keywords["cmap"] = 'plasma' # gives a good dark background for the overplots
# keywords["cmap"] = 'PuBuGn' # the overplots look washed out

# create a series of plots
# for color in ('dg', 'dr', 'di', 'dz', 'dgr', 'dri', 'diz'):
for color in ('dr', 'dgr'): # use this for testing
    keywords["Ystr"] = color
    doOneColor(n2020PS1, keywords)


I am working on the file "Analysis_2020/compSDSS_PS_KTv2.ipynb".
The first task is:
Read in the matched 2020 SSC + PS1 cat
Can you write Python code for it?

%%time
n2020PS1 = pd.read_csv(nSSC2PS1,header=0,dtype=Ndtype,comment='#')
nrows,ncols = n2020PS1.shape
print('N2020+PS1, as read: num rows, cols: ',nrows,ncols)


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:

Define paths and catalogs
Can you write Python code for it?

ZIdataDir = "/Users/ivezic/Work/Science/CalibrationV2/SDSS_SSC/Data"

# the original SDSS catalog from 2007
sdssOldCat = ZIdataDir + "/" + "stripe82calibStars_v2.6.dat"
# INPUT: Karun's new catalog from 2020
sdssNewCatIn = ZIdataDir + "/" + "N2020_stripe82calibStars.dat"
readFormat = 'csv'
# OUTPUT: with Gmag-based gray corrections
sdssNewCatOut = ZIdataDir + "/" + "stripe82calibStars_v3.1.dat"
# Gaia DR2
GaiaDR2Cat = ZIdataDir + "/" + "Stripe82_GaiaDR2.dat"
# Gaia DR2 with BP and RP data
GaiaDR2CatBR = ZIdataDir + "/" + "Stripe82_GaiaDR2_BPRP.dat"

# both new and old files use identical data structure
colnamesSDSS = ['calib_fla', 'ra', 'dec', 'raRMS', 'decRMS', 'nEpochs', 'AR_val',
                'u_Nobs', 'u_mMed', 'u_mMean', 'u_mErr', 'u_rms_scatt', 'u_chi2',
                'g_Nobs', 'g_mMed', 'g_mMean', 'g_mErr', 'g_rms_scatt', 'g_chi2',
                'r_Nobs', 'r_mMed', 'r_mMean', 'r_mErr', 'r_rms_scatt', 'r_chi2',
                'i_Nobs', 'i_mMed', 'i_mMean', 'i_mErr', 'i_rms_scatt', 'i_chi2',
                'z_Nobs', 'z_mMed', 'z_mMean', 'z_mErr', 'z_rms_scatt', 'z_chi2']

%%time
# old
sdssOld = Table.read(sdssOldCat, format='ascii', names=colnamesSDSS)
np.size(sdssOld)

%%time
# new
sdssNew = Table.read(sdssNewCatIn, format=readFormat, names=colnamesSDSS)
np.size(sdssNew)


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
Final Figures for the Paper
with Karun's 2D histogram implementation
Can you write Python code for it?

def plotGmag2Ddiagrams(d):
    ### plots
    plotNameRoot = 'GrVSgi'
    plotName = plotNameRoot + '.png'
    kw = {"Xstr":'gi', "Xmin":0.0, "Xmax":3.5, "Xlabel":'SDSS g-i', \
          "Ystr":'gGr_new', "Ymin":-1.25, "Ymax":0.25, "Ylabel":'Gaia Gmag - SDSS r', \
          "XminBin":-0.5, "XmaxBin":4.0, "nBin":90, \
          "plotName":plotName, "Nsigma":3, "offset":0.0, "symbSize":0.05}
    kw["nBinX"] = 90
    kw["nBinY"] = 40
    kw["cmap"] = 'plasma'
    ktt.plotdelMagBW_KT(d, kw)
    print('made plot', plotName)

def plotGmag2DdiagramsX(d, kw):
    # Gaia G
    print('-----------')
    print(' stats for SDSS r binning medians:')
    plotName = plotNameRoot + '_Gmag.png'
    kwOC = {"Xstr":'Gmag', "Xmin":14.3, "Xmax":21.01, "Xlabel":'Gaia G (mag)', \
          "Ystr":kw['Ystr'], "Ymin":-0.06, "Ymax":0.06, "Ylabel":Ylabel, \
          "XminBin":14.5, "XmaxBin":21.0, "nBin":130, \
          "plotName":plotName, "Nsigma":3, "offset":0.01, "symbSize":kw['symbSize']}
    zit.plotdelMag(goodC, kwOC)
    plotName = plotNameRoot + '_Gmag_Hess.png'
    kwOC["plotName"] = plotName
    kwOC["nBinX"] = 130
    kwOC["nBinY"] = 50
    kwOC["cmap"] = 'plasma'
    ktt.plotdelMagBW_KT(goodC, kwOC)
    print('made plot', plotName)
    print('------------------------------------------------------------------')

def plotGmagCorrections(d, kw):
    ### REDEFINE residuals to correspond to "SDSS-others", as other cases
    d['redef'] = -1*d[kw['Ystr']]
    kw['Ystr'] = 'redef'
    goodC = d[np.abs(d['redef'])<0.1]

    ### plots
    plotNameRoot = kw['plotNameRoot']
    # RA
    print(' stats for RA binning medians:')
    plotName = plotNameRoot + '_RA.png'
    Ylabel = 'residuals for (Gmag$_{SDSS}$ - Gmag$_{GaiaDR2}$)  '
    kwOC = {"Xstr":'raW', "Xmin":-52, "Xmax":60.5, "Xlabel":'R.A. (deg)', \
          "Ystr":kw['Ystr'], "Ymin":-0.07, "Ymax":0.07, "Ylabel":Ylabel, \
          "XminBin":-51.5, "XmaxBin":60, "nBin":112, \
          "plotName":plotName, "Nsigma":3, "offset":0.01, "symbSize":kw['symbSize']}
...


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
Final Gmag-based recalibration
Recalibrate R.A. residuals
Can you write Python code for it?

RAbin, RAnPts, RAmedianBin, RAsigGbin = zit.fitMedians(mOKcBok['raW'], mOKcBok['GrResid'], -51.5, 60.0, 112, 1)


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
Now save correction arrays, then apply to original file, and then test
Can you write Python code for it?

# final refers to the July 2020 analysis, before the paper submission
#np.savetxt('ZPcorrectionsRA_v3.1_final.dat', (RAbin, RAmedianBin))
#np.savetxt('ZPcorrectionsDec_v3.1_final.dat', (decCorrGrid, ZPcorr))

sdssOut = sdss[sdss['ra_new']<1]
sdssOut = zit.selectCatalog(sdss, sdssOut)

sdssOut.sort('calib_fla_new')

# read back gray zero point recalibration files
zpRAgrid, zpRA = np.loadtxt('ZPcorrectionsRA_v3.1_final.dat')
zpDecgrid, zpDec = np.loadtxt('ZPcorrectionsDec_v3.1_final.dat')

sdssOut

# Dec correction
decGrid2correct = sdssOut['dec_new']
ZPcorrectionsDec = np.interp(decGrid2correct, zpDecgrid, zpDec)

# RA correction
ra = sdssOut['ra_new']
raWGrid2correct = np.where(ra > 180, ra-360, ra)
ZPcorrectionsRA = np.interp(raWGrid2correct, zpRAgrid, zpRA)
print('gray std RA/Dec:', np.std(ZPcorrectionsRA), np.std(ZPcorrectionsDec))

for b in ('u', 'g', 'r', 'i', 'z'):
    for mtype in ('_mMed_new', '_mMean_new'):
        mstr = b + mtype
        # applying here gray corrections
        sdssOut[mstr] = sdssOut[mstr] + ZPcorrectionsRA + ZPcorrectionsDec

SSCindexRoot = 'CALIBSTARS_'
outFile = ZIdataDir + "/" + "stripe82calibStars_v3.1_noheader_final.dat"
newSSC = open(outFile,'w')
df = sdssOut
Ngood = 0
for i in range(0, np.size(df)):
    Ngood += 1
    NoldCat = df['calib_fla_new'][i]
    strNo = f'{Ngood:07}'
    SSCindex = SSCindexRoot + strNo
    SSCrow = zit.getSSCentry(df, i)
    zit.SSCentryToOutFileRow(SSCrow, SSCindex, newSSC)
newSSC.close()
print(Ngood, 'rows in file', outFile)