I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
Recalibrate Dec residuals
Can you write Python code for it?

decOK = mOKcBok['dec_new']
GrResid =  mOKcBok['GrResid']
fig,ax = plt.subplots(1,1,figsize=(8,6))
ax.scatter(decOK, GrResid, s=0.01, c='blue')
ax.set_xlim(-1.3,1.3)
ax.set_ylim(-0.06,0.06)
ax.set_ylim(-0.04,0.04)

ax.set_xlabel('Declination (deg)')
ax.set_ylabel('Gaia G - SDSS G')
xBin, nPts, medianBin, sigGbin = zit.fitMedians(decOK, GrResid, -1.266, 1.264, 252, 0)
ax.scatter(xBin, medianBin, s=30.0, c='black', alpha=0.9)
ax.scatter(xBin, medianBin, s=15.0, c='yellow', alpha=0.5)
TwoSigP = medianBin + 2*sigGbin
TwoSigM = medianBin - 2*sigGbin
ax.plot(xBin, TwoSigP, c='yellow')
ax.plot(xBin, TwoSigM, c='yellow')
xL = np.linspace(-100,100)
ax.plot(xL, 0*xL+0.00, c='yellow')
ax.plot(xL, 0*xL+0.01, c='red')
ax.plot(xL, 0*xL-0.01, c='red')
dCleft = -1.3
ax.plot(0*xL+dCleft, xL, c='red')
alltheta = []
for i in range(0,12):
    decCol = -1.2655 + (i+1)*0.2109
    ax.plot(0*xL+decCol, xL, c='red')
    xR = xBin[(xBin>dCleft)&(xBin<decCol)]
    yR = medianBin[(xBin>dCleft)&(xBin<decCol)]
    dyR = sigGbin[(xBin>dCleft)&(xBin<decCol)]
    data = np.array([xR, yR, dyR])
    theta2 = zit.best_theta(data,5)
    alltheta.append(theta2)
    yfit = zit.polynomial_fit(theta2, xR)
    ax.plot(xR, yfit, c='cyan', lw=2)
    dCleft = decCol
    rrr = yR - yfit
    # print(i, np.median(rrr), np.std(rrr))  # 2 milimag scatter
    # print(i, theta2)
plt.savefig('GmagDecCorrections.png')

# let's now correct all mags with this correction
thetaRecalib = alltheta

decLeft = -1.3
for i in range(0,12):
    decRight = -1.2655 + (i+1)*0.2109
    decArr = np.linspace(decLeft, decRight, 100)
    thetaBin = thetaRecalib[i]
    ZPfit = zit.polynomial_fit(thetaBin, decArr)
    if (i==0):
        decCorrGrid = decArr
        ZPcorr = ZPfit
    else:
        decCorrGrid = np.concatenate([decCorrGrid, decArr])
        ZPcorr = np.concatenate([ZPcorr, ZPfit])
    decLeft = decRight

mOKtest = mOK[mOK['r_Nobs_new']>3]

# Dec correction
decGrid2correct = mOKtest['dec_new']
ZPcorrectionsDec = np.interp(decGrid2correct, decCorrGrid, ZPcorr)
# RA...


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
Select good matches and compare both catalogs to Gaia DR2
Can you write Python code for it?

# doGaiaAll(mOK)
def doGaiaGmagCorrection(d, Cstr, Gmax=20.0, yMax=0.03):
    # Cstr = 'gGr_old' or 'gGr_new'
    gi = d['gi']
    Gr = d[Cstr]
    Gmag = d['Gmag']
    zit.qpBM(d, 'gi', -1, 4.5, Cstr, -2, 1.0, 56)

    xBin, nPts, medianBin, sigGbin = zit.fitMedians(gi, Gr, -0.7, 4.0, 47, 0)
    data = np.array([xBin, medianBin, sigGbin])
    Ndata = xBin.size
    ### HERE WE ARE FITTING 7-th ORDER POLYNOMIAL TO Gmag-rSDSS vs. g-i ###
    # get best-fit parameters
    thetaCloc = zit.best_theta(data,7)
    # generate best fit lines on a fine grid
    xfit = np.linspace(-1.1, 4.3, 1000)
    yfit = zit.polynomial_fit(thetaCloc, xfit)
    ## added "Poly" because switched to piecewise linear interpolation below
    d['gGrFitPoly'] = zit.polynomial_fit(thetaCloc, gi)
    d['dgGrPoly'] = d[Cstr] - d['gGrFitPoly']
    ### PIECEWISE LINEAR INTERPOLATION (AS FOR ALL OTHER COLORS AND SURVEYS)
    d['gGrFit'] = np.interp(gi, xBin, medianBin)
    d['dgGr'] = d[Cstr] - d['gGrFit']

    # SELECT FOR RECALIBRATION wrt RA and Dec
    giMin = 0.4
    giMax = 3.0
    Dc = d[(d['gi']>giMin)&(d['gi']<giMax)]
    print('N before and after color cut:', np.size(d), np.size(Dc))
    DcB = Dc[(Dc['Gmag']>14.5)&(Dc['Gmag']<Gmax)]
    DcB['GrResid'] = DcB['dgGr'] - np.median(DcB['dgGr'])
    zit.printStats(DcB['dgGr'])
    DcBok = DcB[np.abs(DcB['dgGr'])<0.1]
    print(np.size(DcB), np.size(DcBok))

    zit.qpBM(DcBok, 'Gmag', 14.5, Gmax, 'GrResid', -1*yMax, yMax, 56)
    zit.qpBM(DcBok, 'dec', -1.3, 1.3, 'GrResid', -1*yMax, yMax, 126)
    zit.qpBM(DcBok, 'raW', -51.5, 60, 'GrResid', -1*yMax, yMax, 112)

    return thetaCloc, DcBok

## first limit astrometric distance and
## require at least 4 epochs as in the old catalog
MAX_DISTANCE_ARCSEC = 0.5
m1 = gaia_sdss[(gaia_sdss['sepSG_2d_arcsec'] < MAX_DISTANCE_ARCSEC)]
a1 = m1['g_Nobs_new']
a2 = m1['r_Nobs_new']
a3 = m1['i_Nobs_new']
mOK = m1[(a1>3)&(a2>3)&(a3>3)]
print(len(new_old))
print(len(m1))
print(len(mOK))

def plotAstro2Ddiagrams(d):
    ### plots
   ...


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
TEMP code for color corrections to go from 3.1 to 3.2 and 3.3
Can you write Python code for it?

### need to figure out where were ZPcorrections2_rz_Dec.dat etc produced ...
## color corrections
for mtype in ('_mMed', '_mMean'):
    ## u band from u-r color
    color = 'ur'
    zpcFilename = 'ZPcorrections_' + color + '_RA.dat'
    zpcRAgrid, zpcRA = np.loadtxt(zpcFilename)
    zpcFilename = 'ZPcorrections_' + color + '_Dec.dat'
    zpcDecgrid, zpcDec = np.loadtxt(zpcFilename)
    ZPcorrectionsRA = np.interp(raWGrid2correct, zpcRAgrid, zpcRA)
    ZPcorrectionsDec = np.interp(decGrid2correct, zpcDecgrid, zpcDec)
    print('u-r std RA/Dec:', np.std(ZPcorrectionsRA), np.std(ZPcorrectionsDec))
    mstr = 'u' + mtype
    sdssOut[mstr] = sdssOut[mstr] - ZPcorrectionsRA - ZPcorrectionsDec
    ## g band from g-r color
    color = 'gr'
    zpcFilename = 'ZPcorrections_' + color + '_RA.dat'
    zpcRAgrid, zpcRA = np.loadtxt(zpcFilename)
    zpcFilename = 'ZPcorrections_' + color + '_Dec.dat'
    zpcDecgrid, zpcDec = np.loadtxt(zpcFilename)
    ZPcorrectionsRA = np.interp(raWGrid2correct, zpcRAgrid, zpcRA)
    ZPcorrectionsDec = np.interp(decGrid2correct, zpcDecgrid, zpcDec)
    print('g-r std RA/Dec:', np.std(ZPcorrectionsRA), np.std(ZPcorrectionsDec))
    mstr = 'g' + mtype
    sdssOut[mstr] = sdssOut[mstr] - ZPcorrectionsRA - ZPcorrectionsDec
    ## i band from r-i color
    color = 'ri'
    zpcFilename = 'ZPcorrections_' + color + '_RA.dat'
    zpcRAgrid, zpcRA = np.loadtxt(zpcFilename)
    zpcFilename = 'ZPcorrections_' + color + '_Dec.dat'
    zpcDecgrid, zpcDec = np.loadtxt(zpcFilename)
    ZPcorrectionsRA = np.interp(raWGrid2correct, zpcRAgrid, zpcRA)
    ZPcorrectionsDec = np.interp(decGrid2correct, zpcDecgrid, zpcDec)
    mstr = 'i' + mtype
    print('r-i std RA/Dec:', np.std(ZPcorrectionsRA), np.std(ZPcorrectionsDec))
    sdssOut[mstr] = sdssOut[mstr] + ZPcorrectionsRA + ZPcorrectionsDec
    ## i band from r-z color
    color = 'rz'
    zpcFilename = 'ZPcorrections_' + color + '_RA.dat'
    zpcRAgrid, zpcRA = np.loadtxt(zpcFilename)
    zpcFilename = 'ZPcorrections_' +...


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
now match to Gaia DR2...
Can you write Python code for it?

colnamesGaia = ['ra', 'dec', 'nObs', 'Gmag', 'flux', 'fluxErr', 'pmra', 'pmdec']
colnamesGaia = colnamesGaia + ['BPmag', 'BPeI', 'RPmag', 'RPeI', 'BRef']
gaia = Table.read(GaiaDR2CatBR, format='ascii', names=colnamesGaia)
gaia['raG'] = gaia['ra']
gaia['decG'] = gaia['dec']
gaia['GmagErr'] = gaia['fluxErr'] / gaia['flux']
gaia['BR'] = gaia['BPmag'] - gaia['RPmag']
gaia['GBP'] = gaia['Gmag'] - gaia['BPmag']
gaia['GRP'] = gaia['Gmag'] - gaia['RPmag']

sdss_coords = SkyCoord(ra = sdss['ra_old']*u.degree, dec= sdss['dec_old']*u.degree)
gaia_coords = SkyCoord(ra = gaia['raG']*u.degree, dec= gaia['decG']*u.degree)

# this is matching gaia to sdss, so that indices are into sdss catalog
# makes sense in this case since the sdss catalog is bigger than gaia
idxG, d2dG, d3dG = gaia_coords.match_to_catalog_sky(sdss_coords)

# object separation is an object with units,
# I add that as a column so that one can
# select based on separation to the nearest matching object
gaia_sdss = hstack([gaia, sdss[idxG]], table_names = ['gaia', 'sdss'])
gaia_sdss['sepSG_2d_arcsec'] = d2dG.arcsec

### code for generating new quantities, such as dra, ddec, colors, differences in mags, etc
def derivedColumns(matches):
    matches['dra'] = (matches['ra_new']-matches['ra_old'])*3600
    matches['ddec'] = (matches['dec_new']-matches['dec_old'])*3600
    matches['ra'] = matches['ra_old']
    ra = matches['ra']
    matches['raW'] = np.where(ra > 180, ra-360, ra)
    matches['dec'] = matches['dec_old']
    matches['u'] = matches['u_mMed_old']
    matches['g'] = matches['g_mMed_old']
    matches['r'] = matches['r_mMed_old']
    matches['i'] = matches['i_mMed_old']
    matches['z'] = matches['z_mMed_old']
    matches['ug'] = matches['u_mMed_old'] - matches['g_mMed_old']
    matches['gr'] = matches['g_mMed_old'] - matches['r_mMed_old']
    matches['ri'] = matches['r_mMed_old'] - matches['i_mMed_old']
    matches['gi'] = matches['g_mMed_old'] - matches['i_mMed_old']
    matches['du'] = matches['u_mMed_old'] - matches['u_mMed_new']
   ...


I am working on the file "Analysis_2020/recalibration_gray.ipynb".
The first task is:
paper plot showing the jump in Gaia Gmag
Can you write Python code for it?


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
DEFINE CAT NAMES, ETC.
Can you write Python code for it?

SDSS_CAT = 'stripe82calibStars_v2.6.dat'
GAIA_CAT = 'Stripe82_GaiaDR1.dat'

# MATCHED GAIA-SDSS OBJECTS
SDSS2GAIA = 'S82_SDSS2GAIA_matchkln.csv'

# MAGS/ERRS OF MATCHED GAIA-SDSS OBJECTS
SDSS2GAIA_magerr = 'S82_SDSS2GAIA_matchkln_magerr.csv'


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
DEFINE PROG CONSTS
Can you write Python code for it?

# CONVERT IQD TO STD DEV
IQD2STD = 0.741

# GAIA ZEROPOINT
GAIA_ZP = 25.525

# DEFINE POLYNOMIAL DEGREES
deg1,deg2,deg3,deg5,deg7 = 1,2,3,5,7


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
FIT MEDIANS IN BINS TO gi on the CCD
Can you write Python code for it?

%%time
# medians
#xBin, nPts, medianBin, sigGbin = fitMedians(gi, Gr, -0.7, 4.0, 47, 0)
#xBinOK, nPtsOK, medianBinOK, sigGbinOK = fitMedians(giOK, GrOK, -0.2, 3.2, 34, 0)
xBin, nPts, medianBin, sigGbin = fitMedians(gi_all, Gr_all, 0.0, 3.0, 30, 0)
xBinOK1, nPtsOK1, medianBinOK1, sigGbinOK1 = fitMedians(gi_pk1, Gr_pk1, 0.0, 3.0, 30, 0)
xBinOK2, nPtsOK2, medianBinOK2, sigGbinOK2 = fitMedians(gi_pk2, Gr_pk2, 0.0, 3.0, 30, 0)
xBinOK3, nPtsOK3, medianBinOK3, sigGbinOK3 = fitMedians(gi_pk3, Gr_pk3, 0.0, 3.0, 30, 0)

#print xBin, nPts, medianBin, sigGbin
medOK1 = medianBinOK1[(xBinOK1>2)&(xBinOK1<3)]
medOK2 = medianBinOK2[(xBinOK2>2)&(xBinOK2<3)]
medOK3 = medianBinOK3[(xBinOK3>2)&(xBinOK3<3)]
dmedOK21 = medOK2 - medOK1
dmedOK32 = medOK3 - medOK2
print('Med offset pk2 - pk1: ',dmedOK21)
print('Med offset pk3 - pk2: ',dmedOK32)


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
FIT POLYNOMIAL TO THE gi MEDIANS
TRY ORDER = 1
Can you write Python code for it?

%%time
data = np.array([xBinOK1, medianBinOK1, sigGbinOK1])
#x, y, sigma_y = data
theta1 = best_theta(data,deg1)
print('Fit values: ',theta1)


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
FUNCTION DEFINITIONS
fitMedians for fitting med and std dev in bins over a x,y distribution
Can you write Python code for it?

# given vectors x and y, fit medians in bins from xMin to xMax, with Nbin steps,
# and return xBin, medianBin, medianErrBin
def fitMedians(x, y, xMin, xMax, Nbin, verbose=1):

    # first generate bins
    xEdge = np.linspace(xMin, xMax, (Nbin+1))
    xBin = np.linspace(0, 1, Nbin)
    nPts = 0*np.linspace(0, 1, Nbin)
    medianBin = 0*np.linspace(0, 1, Nbin)
    sigGbin = -1+0*np.linspace(0, 1, Nbin)
    for i in range(0, Nbin):
        xBin[i] = 0.5*(xEdge[i]+xEdge[i+1])
        yAux = y[(x>xEdge[i])&(x<=xEdge[i+1])]
        if (yAux.size > 0):
            nPts[i] = yAux.size
            medianBin[i] = np.median(yAux)
            # robust estimate of standard deviation: 0.741*(q75-q25)
            sigmaG = 0.741*(np.percentile(yAux,75)-np.percentile(yAux,25))
            # uncertainty of the median: sqrt(pi/2)*st.dev/sqrt(N)
            sigGbin[i] = np.sqrt(np.pi/2)*sigmaG/np.sqrt(nPts[i])
        else:
            nPts[i], medianBin[i], sigGBin[i] = 0

    if (verbose):
        print('median:', np.median(medianBin[Npts>0]))

    return xBin, nPts, medianBin, sigGbin


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
GET ALL THE REQUIRED QUANTITIES

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

_all
Can you write Python code for it?

r_all = gaia_sdss['r_mMed']
G_all = gaia_sdss['Gmag']

# sdss colors
gr_all = gaia_sdss['g_mMed'] - gaia_sdss['r_mMed']
gi_all = gaia_sdss['g_mMed'] - gaia_sdss['i_mMed']
gz_all = gaia_sdss['g_mMed'] - gaia_sdss['z_mMed']
ri_all = gaia_sdss['r_mMed'] - gaia_sdss['i_mMed']
# Gaia colors
Gg_all = gaia_sdss['Gmag'] - gaia_sdss['g_mMed']
Gr_all = gaia_sdss['Gmag'] - gaia_sdss['r_mMed']
Gi_all = gaia_sdss['Gmag'] - gaia_sdss['i_mMed']

ra_all = gaia_sdss['ra_gaia']
raW_all = np.where(ra_all > 180, ra_all-360, ra_all)
dec_all = gaia_sdss['dec_gaia']


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
Match gaia to sdss, since here sdss is much larger
Can you write Python code for it?

# PRINT OUT NUMBER OF OBJ READ
print('Num 2007 cat obj: {0}'.format(len(sdss['ra'])))
print('Num gaia obj: {0}'.format(len(gaia['ra'])))

%%time
sdss_coords = SkyCoord(ra = sdss['ra']*u.degree, dec= sdss['dec']*u.degree)
gaia_coords = SkyCoord(ra = gaia['ra']*u.degree, dec= gaia['dec']*u.degree)

# this is matching gaia to sdss, so that indices are into sdss catalog
# makes sense in this case since the sdss catalog is bigger than gaia
idx, d2d, d3d = gaia_coords.match_to_catalog_sky(sdss_coords)


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
PICK ALL THE REQUIRED QUANTITIES WITH SELECTION I

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

_pk1
Can you write Python code for it?

## MAGS AND COLORS
r_pk1 = gaia_matched['r_mMed']
G_pk1 = gaia_matched['Gmag']
# sdss colors
gr_pk1 = gaia_matched['g_mMed'] - gaia_matched['r_mMed']
gi_pk1 = gaia_matched['g_mMed'] - gaia_matched['i_mMed']
gz_pk1 = gaia_matched['g_mMed'] - gaia_matched['z_mMed']
ri_pk1 = gaia_matched['r_mMed'] - gaia_matched['i_mMed']
# gaia colors
Gg_pk1 = gaia_matched['Gmag'] - gaia_matched['g_mMed']
Gr_pk1 = gaia_matched['Gmag'] - gaia_matched['r_mMed']
Gi_pk1 = gaia_matched['Gmag'] - gaia_matched['i_mMed']

# POSITIONS BASED ON GAIA
ra_pk1 = gaia_matched['ra_gaia']
raW_pk1 = np.where(ra_pk1 > 180, ra_pk1-360, ra_pk1)
dec_pk1 = gaia_matched['dec_gaia']


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
PICK ALL THE REQUIRED QUANTITIES WITH SELECTION II**

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

_pk2
Can you write Python code for it?

g_pk2 = gaia_matchedOK['g_mMed']
r_pk2 = gaia_matchedOK['r_mMed']
i_pk2 = gaia_matchedOK['i_mMed']
z_pk2 = gaia_matchedOK['z_mMed']

G_pk2 = gaia_matchedOK['Gmag']
# sdss colors
gr_pk2 = gaia_matchedOK['g_mMed'] - gaia_matchedOK['r_mMed']
gi_pk2 = gaia_matchedOK['g_mMed'] - gaia_matchedOK['i_mMed']
gz_pk2 = gaia_matchedOK['g_mMed'] - gaia_matchedOK['z_mMed']
ri_pk2 = gaia_matchedOK['r_mMed'] - gaia_matchedOK['i_mMed']
# Gaia colors
Gg_pk2 = gaia_matchedOK['Gmag'] - gaia_matchedOK['g_mMed']
Gr_pk2 = gaia_matchedOK['Gmag'] - gaia_matchedOK['r_mMed']
Gi_pk2 = gaia_matchedOK['Gmag'] - gaia_matchedOK['i_mMed']

ra_pk2 = gaia_matchedOK['ra_gaia']
raW_pk2 = np.where(ra_pk2 > 180, ra_pk2-360, ra_pk2)
dec_pk2 = gaia_matchedOK['dec_gaia']


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
PICK ALL THE REQUIRED QUANTITIES WITH SELECTION III

MAGS AND COLORS FROM SDSS AND GAIA
POSITIONS FROM GAIA ONLY

NOTE Also get the mag errs for sdss griz and Gaia Gmag. These are needed for the next selection cut
_pk3
Can you write Python code for it?

g_pk3 = gaia_matchedOK2['g_mMed']
r_pk3 = gaia_matchedOK2['r_mMed']
i_pk3 = gaia_matchedOK2['i_mMed']
z_pk3 = gaia_matchedOK2['z_mMed']

G_pk3 = gaia_matchedOK2['Gmag']

# also get mag errors for the next selection cut
# Gaia err
Gflux_pk3 = gaia_matchedOK2['flux']
Gfluxerr_pk3 = gaia_matchedOK2['fluxErr']
G_err_pk3 = -2.5*np.log10(1.+ (Gfluxerr_pk3/Gflux_pk3))

# sdss errs
g_err_pk3 = gaia_matchedOK2['g_mErr']
r_err_pk3 = gaia_matchedOK2['r_mErr']
i_err_pk3 = gaia_matchedOK2['i_mErr']
z_err_pk3 = gaia_matchedOK2['z_mErr']

# sdss colors
gr_pk3 = gaia_matchedOK2['g_mMed'] - gaia_matchedOK2['r_mMed']
gi_pk3 = gaia_matchedOK2['g_mMed'] - gaia_matchedOK2['i_mMed']
gz_pk3 = gaia_matchedOK2['g_mMed'] - gaia_matchedOK2['z_mMed']
ri_pk3 = gaia_matchedOK2['r_mMed'] - gaia_matchedOK2['i_mMed']
# Gaia colors
Gg_pk3 = gaia_matchedOK2['Gmag'] - gaia_matchedOK2['g_mMed']
Gr_pk3 = gaia_matchedOK2['Gmag'] - gaia_matchedOK2['r_mMed']
Gi_pk3 = gaia_matchedOK2['Gmag'] - gaia_matchedOK2['i_mMed']

ra_pk3 = gaia_matchedOK2['ra_gaia']
raW_pk3 = np.where(ra_pk3 > 180, ra_pk3-360, ra_pk3)
dec_pk3 = gaia_matchedOK2['dec_gaia']


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
PLOT FiT VS MEASURED GMAG AND RESIDUAL
Can you write Python code for it?

# Convert flux to mag
Gmag_fit = GAIA_ZP - 2.5*np.log10(ffit)
Gmag_msr = GAIA_ZP - 2.5*np.log10(Gflux_pk4)
Gmag_resd = Gmag_fit - Gmag_msr

# Plot train and val losses
%matplotlib inline
# plot
fig,ax = plt.subplots(1,2,figsize=(10,5))
# Plt 1: mag vs mag
xmin,xmax = 16,16.5
ax[0].plot(Gmag_msr,Gmag_fit,'mo',markersize=2)
ax[0].plot([xmin,xmax],[xmin,xmax],'k-')
ax[0].set_xlabel('Measured Gmag')
ax[0].set_ylabel('Fit Gmag')
ax[0].set_xlim(xmin,xmax)
ax[0].set_ylim(xmin,xmax)

# Plt 2: mag vs resd
ax[1].plot(Gmag_msr,Gmag_resd,'bo')
ax[1].plot([xmin,xmax],[0,0],'k-')
ax[1].set_xlabel('Measured Gmag')
ax[1].set_ylabel('Gmag Resd (= Pred - Meas)')
ax[1].set_xlim(xmin,xmax)
ax[1].set_ylim(-0.15,0.15)


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
PLOTS
ONLY CCD
Can you write Python code for it?

# FUNCTION TO PLOT THE CCD, FIT THE POLY, AND OVERPLOT THE FITTED VALUES
def CCDpltNfit(clr1a,clr1b,clr1c,clr1d,clr2a,clr2b,clr2c,clr2d,labls,
               xlim,ylim,ax_labl,plt_titl,fitclr1,fitclr2,Binz,degr=3):
    # clr1a,clr1b,clr1c,clr1d = the four cuts on clr on x-axis
    # clr2a,clr2b,clr2c,clr2d = the four cuts on clr on y-axis
    # labls = labels for the datasets given as a 4-element list
    # xlim, ylim = plot limits on x and y as 2-element lists
    # ax_labl,plt_titl = axes labels, and plot title
    # fitclr1,fitclr2 = cut on clr1, clr2 with which to comp medians and fit poly
    # Binz = minx, maxx and nbins for fitting range
    # degr = degree of fit poly, default = 3

    # plot the base CCD
    fig,ax = plt.subplots(1,1,figsize=(8,6))
    ax.scatter(clr1a,clr2a, s=0.01, c='orange',label=labls[0])
    ax.scatter(clr1b,clr2b, s=0.01, c='green',label=labls[1])
    ax.scatter(clr1c,clr2c, s=0.01, c='blue',label=labls[2])
    ax.scatter(clr1d,clr2d, s=0.01, c='red',label=labls[3])
    ax.set_xlim(xlim)
    ax.set_ylim(ylim)
    ax.set_xlabel(ax_labl[0])
    ax.set_ylabel(ax_labl[1])
    ax.set_title(plt_titl,fontdict={'fontsize': 14, 'fontweight': 'bold'})
    ax.legend(loc='best')
    ax.grid(True)

    plt.show()

    # now fit the medians to the specified data set
    minx,maxx,nbin = Binz[0],Binz[1],Binz[2]
    xBin, nPts, medBin, sigBin = fitMedians(fitclr1,fitclr2,minx,maxx,nbin,0)

    # now fit the poly of spec degr
    data = np.array([xBin,medBin,sigBin])
    theta = best_theta(data,degr)

    return xBin, nPts, medBin, sigBin,theta

clr1a,clr1b,clr1c,clr1d = gr_pk1,gr_pk2,gr_pk3,gr_pk4
clr2a,clr2b,clr2c,clr2d = Gg_pk1,Gg_pk2,Gg_pk3,Gg_pk4
labls = ['pk1','pk2','pk3','pk4']
xlim = (-0.6,1.8)
ylim = (-3.0,0.5)
ax_labl = ['(g-r)','(G-g)']
plt_titl = 'CCD SDSS gr VS Gaia Gg'
fitclr1,fitclr2 = gr_pk4,Gg_pk4
Binz = [0.2,1.2,20]
xBin, nPts, medBin, sigBin,theta = CCDpltNfit(clr1a,clr1b,clr1c,clr1d,clr2a,clr2b,clr2c,clr2d,labls,
              ...


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
Read the sdss and gaia catalogs
Goes quite slowly, consider DASK?
Can you write Python code for it?

%%time
colnames = ['calib_fla', 'ra', 'dec', 'raRMS', 'decRMS', 'nEpochs', 'AR_val',
                'u_Nobs', 'u_mMed', 'u_mMean', 'u_mErr', 'u_rms_scatt', 'u_chi2',
                'g_Nobs', 'g_mMed', 'g_mMean', 'g_mErr', 'g_rms_scatt', 'g_chi2',
                'r_Nobs', 'r_mMed', 'r_mMean', 'r_mErr', 'r_rms_scatt', 'r_chi2',
                'i_Nobs', 'i_mMed', 'i_mMean', 'i_mErr', 'i_rms_scatt', 'i_chi2',
                'z_Nobs', 'z_mMed', 'z_mMean', 'z_mErr', 'z_rms_scatt', 'z_chi2']

sdss = Table.read(SDSS_CAT, format='ascii', names=colnames)

%%time
colnames = ['ra', 'dec', 'nObs', 'Gmag', 'flux', 'fluxErr']
# gaia = Table.read('Stripe82_GaiaDR1_small.dat', format='ascii', names=colnames)
gaia = Table.read(GAIA_CAT, format='ascii', names=colnames)


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
SELECTION I:  BASED ON MATCH DISTANCE
SELECT WITH DIST < 0.5 ARC.SEC
gaia_matched
Can you write Python code for it?

# I would call good match to be within a certain limit
# there is no built-in boundary - match_to_catalog_sky()
# will find the nearest match, regardless if it's an arcsecond
# or five degrees to the nearest one.

# gaia sources that have a good sdss match
flag = (gaia_sdss['sep_2d_arcsec'] < 0.5)  # 486812 for <1 arcsec
gaia_matched = gaia_sdss[flag]
print('Num matched obj: %d' % len(gaia_sdss))
print('Num dist < 0.5 arc.sec: %d' % len(gaia_matched))


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
SELECTION II: BY RA, RMAG AND SDSS GI COLOR
flagOK = ((raW_pk1 > -10) & (raW_pk1 < 50) & (r_pk1>16) & (r_pk1<19) & (gi_pk1>0) & (gi_pk1<3.0))
gaia_matchedOK
Can you write Python code for it?

# flagOK = ((raW > -10) & (raW < 50) & (rMed>15) & (rMed<20))
# flagOK = ((raW > -10) & (raW < 50) & (rMed>16) & (rMed<19))
flagOK = ((raW_pk1 > -10) & (raW_pk1 < 50) & (r_pk1>16) & (r_pk1<19) & (gi_pk1>0) & (gi_pk1<3.0))
gaia_matchedOK = gaia_matched[flagOK]
print('Num obj after select by RA, rMag, gi color: %d' % (len(gaia_matchedOK)))


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
SELECTION III: INCLUDE GAIA GMAG WITH SELECTION II
flagOK = ((raW_pk2 > -10) & (raW_pk2 < 50) & (r_pk2>16) & (r_pk2<19) & (gi_pk2>0) & (gi_pk2<3.0))
flagOK2 = (flagOK & (G_pk2 > 16) & (G_pk2 < 16.5))
gaia_matchedOK2
Can you write Python code for it?

flagOK = ((raW_pk2 > -10) & (raW_pk2 < 50) & (r_pk2>16) & (r_pk2<19) & (gi_pk2>0) & (gi_pk2<3.0))

flagOK2 = (flagOK & (G_pk2 > 16) & (G_pk2 < 16.5))
gaia_matchedOK2 = gaia_matchedOK[flagOK2]
print('Num obj after select by RA, rMag, gi color: %d' % len(gaia_matchedOK))
print('Num obj after Gaia G-mag cut: %d' % len(gaia_matchedOK2))

print(len(flagOK2))


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
STOPPED HERE
CCD gi vs Gr
Can you write Python code for it?

# plot
fig,ax = plt.subplots(1,1,figsize=(8,6))
ax.scatter(gi_all, Gr_all, s=0.01, c='green')
ax.scatter(gi_pk1, Gr_pk1, s=0.01, c='blue')
ax.scatter(gi_pk2, Gr_pk2, s=0.01, c='red')
ax.set_xlim(-0.5,3.5)
ax.set_ylim(-1.5,1.)
ax.set_xlabel('SDSS(g-i)')
ax.set_ylabel('Gaia G - SDSS r')
ax.set_title('CCD: gi vs Gr all-pk1-pk2',fontdict={'fontsize': 14, 'fontweight': 'bold'})


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
USE COEFFTS TO GENERATE FITTED VALUES
COMPARE WITH Y-VALUES, ESTIMATE CHI2
Can you write Python code for it?

# generate best fit lines on a fine grid
xfit = np.linspace(-1.1, 4.3, 1000)
yfit1 = polynomial_fit(theta1, xfit)
yfit2 = polynomial_fit(theta2, xfit)
yfit3 = polynomial_fit(theta3, xfit)

# and compute chi2 per degree of freedom: sum{[(y-yfit)/sigma_y]^2}
x, y, sigma_y = xBinOK2, medianBinOK2, sigGbinOK2
chi21 = np.sum(((y-polynomial_fit(theta1, x))/sigma_y)**2)
chi22 = np.sum(((y-polynomial_fit(theta2, x))/sigma_y)**2)
chi23 = np.sum(((y-polynomial_fit(theta3, x))/sigma_y)**2)
# the number of fitted parameters is 2, 3, 4
chi2dof1 = chi21/(Ndata - deg3)
chi2dof2 = chi22/(Ndata - deg5)
chi2dof3 = chi23/(Ndata - deg7)

print("CHI2:")
print('   Model deg, chi2 :', deg3,chi21)
print('Model deg, chi2 ::', deg5, chi22)
print('    Model deg, chi2 ::', deg7, chi23)
print("CHI2 per degree of freedom:")
print('   best model 1:', chi2dof1)
print('best model 2:', chi2dof2)
print('    best model 3:', chi2dof3)

# Plot a (gaia - r)  vs (g-i)  for photometric transformation
%matplotlib inline
# plot
fig,ax = plt.subplots(1,1,figsize=(12,8))
ax.scatter(gi_pk1, Gr_pk1, s=0.01, c='green')
ax.scatter(gi_pk2, Gr_pk2, s=0.01, c='blue')
ax.scatter(gi_pk3, Gr_pk3, s=0.01, c='red')
# medians
ax.scatter(xBinOK1, medianBinOK1, s=30.0, c='black', alpha=0.5)
ax.scatter(xBinOK2, medianBinOK2, s=30.0, c='yellow', alpha=0.5)
ax.scatter(xBinOK2, medianBinOK2, s=30.0, c='green', alpha=0.5)
ax.set_xlim(-1,4)
ax.set_ylim(-2.0,0.5)
ax.set_xlabel('SDSS(g-i)')
ax.set_ylabel('Gaia G - SDSS r')
ax.errorbar(x, y, sigma_y, fmt='ok', ecolor='gray')
ax.plot(xfit, polynomial_fit(theta1, xfit), label='best P3 model')
ax.plot(xfit, polynomial_fit(theta2, xfit), label='best P5 model')
ax.plot(xfit, polynomial_fit(theta3, xfit), label='best P7 model')
ax.legend(loc='best', fontsize=14)

print('Coeffts 3rd order fit: ',theta3)

# GrModel = -0.06348 -0.03111*gi +0.08643*gi*gi -0.05593*gi*gi*gi
GrModel = sum(t * gi_pk2 ** n for (n, t) in enumerate(theta3))

GrResid = Gr_pk2 - GrModel
minx,maxx,nx = min(xBinOK2),max(xBinOK2),10*len(xBinOK2)
xBinM, nPtsM,...


I am working on the file "Analysis_2020/sdss_gaia_matching_IZv2.ipynb".
The first task is:
WRITE THIS MATCHED, CLEANED SDSS-GAIA DF TO FILE
Can you write Python code for it?

%%time
paths = SDSS2GAIA
gaia_matchedOK2.write(paths,format='ascii.csv',
                      delimiter=',', comment='#',overwrite=True)


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Between Experiment (Stroop vs Navon)
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr_orig']
n_grp = len(df['cfg_key_label'][()]['Experiment_ID'])
n_subj = len(df['cfg_key_label'][()]['Subject_ID'])

grp_edge_wt = np.zeros((n_grp, n_subj))
for grp_ii in xrange(n_grp):
    for subj_ii in xrange(n_subj):
        grp_ix = np.array(cfg_obs_lut[subj_ii, :, :, :, :, :][grp_ii, :, :, :, :].reshape(-1), dtype=int)
        grp_edge_wt[grp_ii, subj_ii] = np.mean(np.mean(cfg_matr[grp_ix, :], axis=1))

print(stats.ttest_rel(*(grp_edge_wt)))

### Plot
plt.figure(figsize=(3,3), dpi=300.0)
ax = plt.subplot(111)
bp = ax.boxplot(grp_edge_wt.T, patch_artist=True)
Echobase.Plotting.fig_format.set_box_color(bp, [0.0, 0.0, 0.0], [[0.2, 0.2, 0.2] for iii in xrange(n_grp)])

ax.set_ylim(ymin=0)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
ax.set_xticklabels(df['cfg_key_label'][()]['Experiment_ID'])

ax.set_xlabel('')
ax.set_ylabel('Weighted Edge Density')

plt.savefig('{}/Wgt_Edge_Density.Expriment.svg'.format(path_Figures))
plt.show()


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Check Contrasts
Stroop vs Navon
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr']

df_to_yeo = np.load('{}/Lausanne125_to_Yeo.npz'.format(path_ExpData))
n_laus = len(df_to_yeo['yeo_lbl'])

coef_ix = np.array(cfg_obs_lut, dtype=int)
cfg_matr_reshape = cfg_matr[coef_ix, :]

for sgn_ii, sgn_id in enumerate(df['cfg_key_label'][()]['CorSign_ID']):
    sel_cfg_matr = (cfg_matr_reshape[:, :, :, 1, sgn_ii, :, :]).mean(axis=-2).mean(axis=-2)
    sel_cfg_matr_tv = np.nan*np.zeros(cfg_matr.shape[1])
    sel_cfg_matr_pv = np.nan*np.zeros(cfg_matr.shape[1])
    for cc in xrange(cfg_matr.shape[1]):
        tv, pv = stats.ttest_rel(*sel_cfg_matr[:, :, cc].T)

        mean_stroop = np.mean(sel_cfg_matr[:, :, cc], axis=0)[0]
        mean_navon = np.mean(sel_cfg_matr[:, :, cc], axis=0)[1]
        dv = (mean_stroop - mean_navon) / np.std(sel_cfg_matr[:, :, cc].reshape(-1))

        sel_cfg_matr_tv[cc] = dv
        sel_cfg_matr_pv[cc] = pv

    sig_pv = Echobase.Statistics.FDR.fdr.bhp(sel_cfg_matr_pv, alpha=0.05, dependent=True)
    sel_cfg_matr_tv[sig_pv == False] = 0.0

    adj = convert_conn_vec_to_adj_matr(sel_cfg_matr_tv)
    adj_yeo = adj[df_to_yeo['sort_laus_to_yeo'], :][:, df_to_yeo['sort_laus_to_yeo']]
    adj_yeo[np.diag_indices_from(adj_yeo)] = np.nan

    # Plot
    plt.figure(figsize=(3,3), dpi=300.0)
    ax = plt.subplot(111)
    mat = ax.matshow(adj_yeo,
                     cmap='PuOr', vmin=-1.0, vmax=1.0)
    plt.colorbar(mat, ax=ax, fraction=0.046, pad=0.04)

    for xx in df_to_yeo['yeo_adj_demarc']:
        ax.vlines(xx, 0, n_laus, color='k', lw=0.5)
        ax.hlines(xx, 0, n_laus, color='k', lw=0.5)

    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_tick_params(width=0)
    ax.xaxis.set_tick_params(width=0)
    ax.grid(False)
    ax.tick_params(axis='both', which='major', pad=-3)

    ax.set_xticks((df_to_yeo['yeo_adj_demarc'][:-1] +...


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Checking Correlation Biases
Across Subjects
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr_orig']
n_grp = len(df['cfg_key_label'][()]['Subject_ID'])

grp_edge_wt = []
for grp_ii in xrange(n_grp):
    grp_ix = np.array(cfg_obs_lut[grp_ii, :, :, :, :, :].reshape(-1), dtype=int)
    grp_edge_wt.append(np.mean(cfg_matr[grp_ix, :], axis=1))
grp_edge_wt = np.array(grp_edge_wt)

mean_grp_edge_wt = np.mean(grp_edge_wt, axis=1)
grp_ord_ix = np.argsort(mean_grp_edge_wt)[::-1]

### Plot Subject Distribution
print(stats.f_oneway(*(grp_edge_wt)))

plt.figure(figsize=(3,3), dpi=300.0)
ax = plt.subplot(111)
bp = ax.boxplot(grp_edge_wt[grp_ord_ix, :].T, sym='', patch_artist=True)
Echobase.Plotting.fig_format.set_box_color(bp, [0.0, 0.0, 0.0], [[0.2, 0.2, 0.2] for iii in xrange(n_grp)])

ax.set_ylim(ymin=0)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
ax.set_xticklabels([])

ax.set_xlabel('Subjects')
ax.set_ylabel('Weighted Edge Density')

plt.savefig('{}/Wgt_Edge_Density.Subjects.svg'.format(path_Figures))
plt.show()


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Compute Functional Connectivity
Functional Connectivity FuncDef
Can you write Python code for it?

def comp_fconn(bold, alpha=0.05, dependent=False):
    n_roi, n_tr = bold.shape

    adj = np.arctanh(np.corrcoef(bold))
    cfg_vec = convert_adj_matr_to_cfg_matr(adj.reshape(-1, n_roi, n_roi))[0, :]

    # Separate edges based on sign
    cfg_vec_pos = cfg_vec.copy()
    cfg_vec_pos[cfg_vec_pos < 0] = 0

    cfg_vec_neg = -1*cfg_vec.copy()
    cfg_vec_neg[cfg_vec_neg < 0] = 0

    adj_pos = convert_conn_vec_to_adj_matr(cfg_vec_pos)
    adj_neg = convert_conn_vec_to_adj_matr(cfg_vec_neg)

    return adj_pos, adj_neg


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Construct System Adjacency Matrices
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_matr = df['cfg_matr']

# Compute Brain System Adjacency Matrices
sys_adj_matr = np.zeros((cfg_matr.shape[0], n_system, n_system))
for sys_ii, (sys_ix, sys_iy) in enumerate(zip(sys_triu_ix, sys_triu_iy)):
    sys1 = system_name[sys_ix]
    sys2 = system_name[sys_iy]
    sys1_ix = np.flatnonzero(system_lbl[triu_ix] == sys1)
    sys2_iy = np.flatnonzero(system_lbl[triu_iy] == sys2)
    inter_sys_ii = np.intersect1d(sys1_ix, sys2_iy)
    if len(inter_sys_ii) == 0:
        sys1_ix = np.flatnonzero(system_lbl[triu_ix] == sys2)
        sys2_iy = np.flatnonzero(system_lbl[triu_iy] == sys1)
        inter_sys_ii = np.intersect1d(sys1_ix, sys2_iy)

    mean_conn_sys1_sys2 = np.mean(cfg_matr[:, inter_sys_ii], axis=1)

    sys_adj_matr[:, sys_ix, sys_iy] = mean_conn_sys1_sys2
    sys_adj_matr[:, sys_iy, sys_ix] = mean_conn_sys1_sys2

np.savez('{}/Full_Adj.Yeo2011_7Networks.npz'.format(path_ExpData),
         sys_adj_matr=sys_adj_matr,
         cfg_obs_lut=df['cfg_obs_lut'],
         cfg_key_label=df['cfg_key_label'],
         cfg_key_names=df['cfg_key_names'])


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Create Lookup-Table and Full Configuration Matrix
Can you write Python code for it?

# Generate a dictionary of all key names
cfg_key_names = ['Subject_ID', 'Experiment_ID', 'Condition_ID', 'Task_ID', 'CorSign_ID', 'Block_ID']
cfg_key_label = {'Subject_ID': np.arange(n_subj),
                 'Experiment_ID': ['Stroop', 'Navon'],
                 'Condition_ID': ['lo', 'hi'],
                 'Task_ID': ['fix', 'task'],
                 'CorSign_ID': ['pos', 'neg'],
                 'Block_ID': np.arange(n_tsk_block)}

cfg_obs_lut = np.zeros((len(cfg_key_label[cfg_key_names[0]]),
                        len(cfg_key_label[cfg_key_names[1]]),
                        len(cfg_key_label[cfg_key_names[2]]),
                        len(cfg_key_label[cfg_key_names[3]]),
                        len(cfg_key_label[cfg_key_names[4]]),
                        len(cfg_key_label[cfg_key_names[5]])))

# Iterate over all cfg key labels and generate a LUT matrix and a config matrix
key_cnt = 0
cfg_matr = []
for key_0_ii, key_0_id in enumerate(cfg_key_label[cfg_key_names[0]]):
    for key_1_ii, key_1_id in enumerate(cfg_key_label[cfg_key_names[1]]):

        adj_dict = np.load(expr_dict[key_0_id][key_1_id])['adj_dict'][()]

        for key_2_ii, key_2_id in enumerate(cfg_key_label[cfg_key_names[2]]):
            for key_3_ii, key_3_id in enumerate(cfg_key_label[cfg_key_names[3]]):
                for key_4_ii, key_4_id in enumerate(cfg_key_label[cfg_key_names[4]]):
                    for key_5_ii, cfg_vec in enumerate(convert_adj_matr_to_cfg_matr(adj_dict[key_2_id][key_3_id][key_4_id])):
                        cfg_obs_lut[key_0_ii, key_1_ii, key_2_ii,
                                    key_3_ii, key_4_ii, key_5_ii] = key_cnt
                        cfg_matr.append(cfg_vec)
                        key_cnt += 1
cfg_matr = np.array(cfg_matr)
cfg_matr_orig = cfg_matr.copy()

# Normalize sum of edge weights to 1
cfg_L1 = np.linalg.norm(cfg_matr, axis=1, ord=1)
cfg_L1[cfg_L1 == 0] = 1.0
cfg_matr = (cfg_matr.T / cfg_L1).T

# Rescale edge weight to unit L2-Norm
cfg_L2 =...


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Fixation vs Task
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr_orig']
n_grp = len(df['cfg_key_label'][()]['Task_ID'])
n_subj = len(df['cfg_key_label'][()]['Subject_ID'])

grp_edge_wt = np.zeros((n_grp, n_subj))
for grp_ii in xrange(n_grp):
    for subj_ii in xrange(n_subj):
        grp_ix = np.array(cfg_obs_lut[subj_ii, :, :, :, :, :][:, :, grp_ii, :, :].reshape(-1), dtype=int)
        grp_edge_wt[grp_ii, subj_ii] = np.mean(np.mean(cfg_matr[grp_ix, :], axis=1))

print(stats.ttest_rel(*(grp_edge_wt)))

### Plot
plt.figure(figsize=(3,3), dpi=300.0)
ax = plt.subplot(111)
bp = ax.boxplot(grp_edge_wt.T, patch_artist=True)
Echobase.Plotting.fig_format.set_box_color(bp, [0.0, 0.0, 0.0], [[0.2, 0.2, 0.2] for iii in xrange(n_grp)])

ax.set_ylim(ymin=0)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
ax.set_xticklabels(df['cfg_key_label'][()]['Task_ID'])

ax.set_xlabel('')
ax.set_ylabel('Weighted Edge Density')

plt.savefig('{}/Wgt_Edge_Density.Task.svg'.format(path_Figures))
plt.show()


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Generate Population Configuration Matrix
Dictionary of all adjacency matrices
Can you write Python code for it?

expr_dict = {}
for expr_id in ['Stroop', 'Navon']:
    df_list = glob.glob('{}/Subject_*.{}.npz'.format(path_ExpData, expr_id))

    for df_subj in df_list:
        subj_id = int(df_subj.split('/')[-1].split('.')[0].split('_')[1])
        if subj_id not in expr_dict.keys():
            expr_dict[subj_id] = {}

        expr_dict[subj_id][expr_id] = df_subj


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Lo vs Hi
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr']

df_to_yeo = np.load('{}/Lausanne125_to_Yeo.npz'.format(path_ExpData))
n_laus = len(df_to_yeo['yeo_lbl'])

for expr_ii, expr_id in enumerate(df['cfg_key_label'][()]['Experiment_ID']):
    for sgn_ii, sgn_id in enumerate(df['cfg_key_label'][()]['CorSign_ID']):
        coef_ix = np.array(cfg_obs_lut, dtype=int)
        cfg_matr_reshape = cfg_matr[coef_ix, :]

        sel_cfg_matr = cfg_matr_reshape[:, expr_ii, :, 1, sgn_ii, :, :].mean(axis=-2)
        sel_cfg_matr_tv = np.nan*np.zeros(cfg_matr.shape[1])
        sel_cfg_matr_pv = np.nan*np.zeros(cfg_matr.shape[1])
        for cc in xrange(cfg_matr.shape[1]):
            tv, pv = stats.ttest_rel(*sel_cfg_matr[:, :, cc].T)

            mean_lo = np.mean(sel_cfg_matr[:, :, cc], axis=0)[0]
            mean_hi = np.mean(sel_cfg_matr[:, :, cc], axis=0)[1]
            dv = (mean_hi - mean_lo) / np.std(sel_cfg_matr[:, :, cc].reshape(-1))

            sel_cfg_matr_tv[cc] = dv
            sel_cfg_matr_pv[cc] = pv

        sig_pv = Echobase.Statistics.FDR.fdr.bhp(sel_cfg_matr_pv, alpha=0.05, dependent=True)
        sel_cfg_matr_tv[sig_pv == False] = np.nan

        adj = convert_conn_vec_to_adj_matr(sel_cfg_matr_tv)
        adj_yeo = adj[df_to_yeo['sort_laus_to_yeo'], :][:, df_to_yeo['sort_laus_to_yeo']]
        adj_yeo[np.diag_indices_from(adj_yeo)] = np.nan

        # Plot
        plt.figure(figsize=(3,3), dpi=300)
        ax = plt.subplot(111)
        mat = ax.matshow(adj_yeo,
                         cmap='coolwarm', vmin=-0.5, vmax=0.5)
        plt.colorbar(mat, ax=ax, fraction=0.046, pad=0.04)

        for xx in df_to_yeo['yeo_adj_demarc']:
            ax.vlines(xx, 0, n_laus, color='k', lw=0.5)
            ax.hlines(xx, 0, n_laus, color='k', lw=0.5)

        ax.yaxis.set_ticks_position('left')
        ax.xaxis.set_ticks_position('bottom')
        ax.yaxis.set_tick_params(width=0)
       ...


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Positive vs Negative
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr_orig']
n_grp = len(df['cfg_key_label'][()]['CorSign_ID'])
n_subj = len(df['cfg_key_label'][()]['Subject_ID'])

grp_edge_wt = np.zeros((n_grp, n_subj))
for grp_ii in xrange(n_grp):
    for subj_ii in xrange(n_subj):
        grp_ix = np.array(cfg_obs_lut[subj_ii, :, :, :, :, :][:, :, :, grp_ii, :].reshape(-1), dtype=int)
        grp_edge_wt[grp_ii, subj_ii] = np.mean(np.mean(cfg_matr[grp_ix, :], axis=1))

print(stats.ttest_rel(*(grp_edge_wt)))

""
### Plot
plt.figure(figsize=(3,3), dpi=300.0)
ax = plt.subplot(111)
bp = ax.boxplot(grp_edge_wt.T, patch_artist=True)
Echobase.Plotting.fig_format.set_box_color(bp, [0.0, 0.0, 0.0], [[0.2, 0.2, 0.2] for iii in xrange(n_grp)])

ax.set_ylim(ymin=0)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
ax.set_xticklabels(df['cfg_key_label'][()]['CorSign_ID'])

ax.set_xlabel('')
ax.set_ylabel('Weighted Edge Density')

plt.savefig('{}/Wgt_Edge_Density.CorSign.svg'.format(path_Figures))
plt.show()


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Process Navon
Can you write Python code for it?

for subj_id in xrange(n_subj):
    proc_item = '{}/Subject_{}.Navon'.format(path_ExpData, subj_id)
    print(proc_item)

    adj_dict = {'lo': {'fix': {'pos': np.zeros((n_tsk_block, n_good_roi, n_good_roi)),
                               'neg': np.zeros((n_tsk_block, n_good_roi, n_good_roi))},
                       'task': {'pos': np.zeros((n_tsk_block, n_good_roi, n_good_roi)),
                                'neg': np.zeros((n_tsk_block, n_good_roi, n_good_roi))}
                      },
                'hi': {'fix': {'pos': np.zeros((n_tsk_block, n_good_roi, n_good_roi)),
                               'neg': np.zeros((n_tsk_block, n_good_roi, n_good_roi))},
                       'task': {'pos': np.zeros((n_tsk_block, n_good_roi, n_good_roi)),
                                'neg': np.zeros((n_tsk_block, n_good_roi, n_good_roi))},
                      }}

    # Process Fixation Blocks
    cnt = 0
    for fix_block in xrange(n_fix_block):
        data = np.array(df_navon['data'][subj_id][fix_block].NFix, dtype='f').T
        data = data[np.setdiff1d(np.arange(n_roi), bad_roi), :]

        if (fix_block % 2) == 0:
            adj_dict['lo']['fix']['pos'][cnt, :, :], adj_dict['lo']['fix']['neg'][cnt, :, :] = comp_fconn(data)

        if (fix_block % 2) == 1:
            adj_dict['hi']['fix']['pos'][cnt, :, :], adj_dict['hi']['fix']['neg'][cnt, :, :] = comp_fconn(data)
            cnt += 1

    # Process Task Blocks
    cnt = 0
    for tsk_block in xrange(n_tsk_block):

        # Low demand
        data = np.array(df_navon['data'][subj_id][tsk_block].NS, dtype='f').T
        data = data[np.setdiff1d(np.arange(n_roi), bad_roi), :]
        adj_dict['lo']['task']['pos'][cnt, :, :], adj_dict['lo']['task']['neg'][cnt, :, :] = comp_fconn(data)

        # High demand
        data = np.array(df_navon['data'][subj_id][tsk_block].S, dtype='f').T
        data = data[np.setdiff1d(np.arange(n_roi), bad_roi), :]
        adj_dict['hi']['task']['pos'][cnt, :, :],...


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
System-Level Adjacency Matrices
Can you write Python code for it?

# Read in Yeo Atlas
df_laus_yeo = pd.read_csv('{}/LausanneScale125.csv'.format(path_CoreData))
df_laus_yeo = df_laus_yeo[df_laus_yeo.Label_ID != bad_roi[0]+1]
system_lbl = np.array(df_laus_yeo['Yeo2011_7Networks'].as_matrix())
system_name = np.unique(df_laus_yeo['Yeo2011_7Networks'])
n_system = len(system_name)
n_roi = len(system_lbl)
triu_ix, triu_iy = np.triu_indices(n_roi, k=1)
sys_triu_ix, sys_triu_iy = np.triu_indices(n_system, k=0)

# Reorder System Labels and Count ROIs per System
system_srt_ix = np.argsort(system_lbl)
system_cnt = np.array([len(np.flatnonzero(system_lbl == sys_name))
                       for sys_name in system_name])
system_demarc = np.concatenate(([0], np.cumsum(system_cnt)))

np.savez('{}/Lausanne125_to_Yeo.npz'.format(path_ExpData),
         df_laus_yeo=df_laus_yeo,
         yeo_lbl=system_lbl,
         yeo_name=system_name,
         sort_laus_to_yeo=system_srt_ix,
         yeo_adj_demarc=system_demarc,
         laus_triu=np.triu_indices(n_roi, k=1),
         yeo_triu=np.triu_indices(n_system, k=0))


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
System-Level Connectivity
Assign Lausanne to Yeo Systems
Can you write Python code for it?

import nibabel as nib

df_yeo_atlas = nib.load('{}/Yeo_JNeurophysiol11_MNI152/Yeo2011_7Networks_MNI152_FreeSurferConformed1mm_LiberalMask.nii.gz'.format(path_AtlasData))
yeo_matr = df_yeo_atlas.get_data()[..., 0]
yeo_roi = np.unique(yeo_matr)[1:]
yeo_names = ['VIS', 'SMN', 'DAN', 'VAN', 'LIM', 'FPN', 'DMN']

yeo_xyz = {}
M = df_yeo_atlas.affine[:3, :3]
abc = df_yeo_atlas.affine[:3, 3]
for yeo_id in yeo_roi:
    yeo_ijk = np.array(np.nonzero(yeo_matr == yeo_id)).T
    yeo_xyz[yeo_id] = M.dot(yeo_ijk.T).T + abc.T

df_laus_atlas = nib.load('{}/Lausanne/ROIv_scale125_dilated.nii.gz'.format(path_AtlasData))
laus_matr = df_laus_atlas.get_data()
laus_roi = np.unique(laus_matr)[1:]

laus_xyz = {}
M = df_laus_atlas.affine[:3, :3]
abc = df_laus_atlas.affine[:3, 3]
for laus_id in laus_roi:
    laus_ijk = np.array(np.nonzero(laus_matr == laus_id)).T
    laus_xyz[laus_id] = M.dot(laus_ijk.T).T + abc.T

laus_yeo_assign = []
for laus_id in laus_roi:
    dists = []

    for yeo_id in yeo_roi:
        dists.append(np.min(np.sum((yeo_xyz[yeo_id] - laus_xyz[laus_id].mean(axis=0))**2, axis=1)))

    laus_yeo_assign.append(yeo_names[np.argmin(dists)])
laus_yeo_assign = np.array(laus_yeo_assign)

pd.DataFrame(laus_yeo_assign).to_csv('{}/Lausanne/ROIv_scale125_dilated.Yeo2011_7Networks_MNI152.csv'.format(path_AtlasData))

# Manually replaced subcortical and cerebellar structures as SUB and CBR, respectively.


I am working on the file "Analysis_Notebooks/e01-Measure_Dynamic_Functional_Networks.ipynb".
The first task is:
Within Experiment (Hi vs Lo)
Can you write Python code for it?

df = np.load('{}/Population.Configuration_Matrix.npz'.format(path_ExpData))
cfg_obs_lut = df['cfg_obs_lut']
cfg_matr = df['cfg_matr_orig']
n_grp = len(df['cfg_key_label'][()]['Condition_ID'])
n_subj = len(df['cfg_key_label'][()]['Subject_ID'])

grp_edge_wt = np.zeros((n_grp, n_subj))
for grp_ii in xrange(n_grp):
    for subj_ii in xrange(n_subj):
        grp_ix = np.array(cfg_obs_lut[subj_ii, :, :, :, :, :][:, grp_ii, :, :, :].reshape(-1), dtype=int)
        grp_edge_wt[grp_ii, subj_ii] = np.mean(np.mean(cfg_matr[grp_ix, :], axis=1))

print(stats.ttest_rel(*(grp_edge_wt)))

### Plot
plt.figure(figsize=(3,3), dpi=300.0)
ax = plt.subplot(111)
bp = ax.boxplot(grp_edge_wt.T, patch_artist=True)
Echobase.Plotting.fig_format.set_box_color(bp, [0.0, 0.0, 0.0], [[0.2, 0.2, 0.2] for iii in xrange(n_grp)])

ax.set_ylim(ymin=0)
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
ax.set_xticklabels(df['cfg_key_label'][()]['Condition_ID'])

ax.set_xlabel('')
ax.set_ylabel('Weighted Edge Density')

plt.savefig('{}/Wgt_Edge_Density.Condition.svg'.format(path_Figures))
plt.show()


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Circle Plot
Can you write Python code for it?

from matplotlib.offsetbox import AnnotationBbox, OffsetImage, TextArea
from scipy import ndimage

# Assign color for each circle node based on associated brain system color
node_clr = np.array([brain_system_pixmap['brain_system_rgbcol'][()][sys_lbl][:3] / 255
                     for sys_lbl in df_to_yeo['yeo_lbl'][df_to_yeo['sort_laus_to_yeo']]])

# Arrange nodes around the circle
system_pos = []
node_rads = np.linspace(0, 2*np.pi - (2*np.pi/n_laus), n_laus)
for sys_ii, sys_nm in enumerate(df_to_yeo['yeo_name']):
    sys_rad = np.mean(node_rads[df_to_yeo['yeo_lbl'] == sys_nm])
    dd = 16
    system_pos.append((sys_rad, dd))

# Render the circle plot
for fac_ii in xrange(n_fac):
    sel_subnet = df_subg['system_subgraph'][fac_ii]['subnet_roi']
    sys_con = convert_adj_matr_to_cfg_matr(np.expand_dims(sel_subnet, axis=0)).squeeze()

    fig, ax = Echobase.Plotting.render_circle_connectivity.draw(conn_list=sys_con,
                                                                conn_pct=[99, 100],
                                                                conn_cmap='YlGnBu',
                                                                conn_linewidth=0.5,
                                                                node_color=node_clr)

    fig.savefig('{}/Circle_Subgraph.{}.svg'.format(path_Figures, fac_ii+1))
    plt.close()


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Generate Surrogate Subgraphs using Consensus Subgraphs
Can you write Python code for it?

# Map NMF surrogate
n_opt = 1000
job_str = './NMF_surrogate_map.py {} {} {}'.format(echobase_path, path_InpData, path_ExpData)
qsub_str = 'qsub -t 1-{} {}'.format(n_opt, job_str)

os.chdir('./e02-SGE_Scripts/')
!sh {qsub_str}
os.chdir('../')

"""
for nn in xrange(n_opt):
    if not os.path.exists('{}/NMF_Surrogate.Param.{}.npz'.format(path_ExpData, nn)):
        qsub_str = 'qsub -t {}-{} {}'.format(nn+1, nn+1, job_str)

        os.chdir('./e02-SGE_Scripts/')
        !sh {qsub_str}
        os.chdir('../')
"""


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Plot Subgraph Adjacency Matrices
Can you write Python code for it?

# Plot the subgraphs
plt.figure(figsize=(5,5), dpi=300);
for ii, fac_ii in enumerate(xrange(n_fac)): #enumerate(sort_fac):
    sel_fac_subnet = fac_subnet_B[fac_ii, :]

    adj = convert_conn_vec_to_adj_matr(sel_fac_subnet)
    adj_yeo = adj[df_to_yeo['sort_laus_to_yeo'], :][:, df_to_yeo['sort_laus_to_yeo']]

    # Plot
    ax = plt.subplot(4, 4, ii+1)
    mat = ax.matshow(adj_yeo,
                     cmap='magma', vmin=1.1*sel_fac_subnet.min());
    #plt.colorbar(mat, ax=ax, fraction=0.046, pad=0.04)

    for xx in df_to_yeo['yeo_adj_demarc']:
        ax.vlines(xx, 0, n_laus, color='w', lw=0.25)
        ax.hlines(xx, 0, n_laus, color='w', lw=0.25)

    ax.yaxis.set_ticks_position('left')
    ax.xaxis.set_ticks_position('bottom')
    ax.yaxis.set_tick_params(width=0)
    ax.xaxis.set_tick_params(width=0)
    ax.grid(False)
    ax.tick_params(axis='both', which='major', pad=-3)

    ax.set_xticks((df_to_yeo['yeo_adj_demarc'][:-1] + (np.diff(df_to_yeo['yeo_adj_demarc']) * 0.5)));
    ax.set_xticklabels(df_to_yeo['yeo_name'], fontsize=3.0, rotation=45)

    ax.set_yticks((df_to_yeo['yeo_adj_demarc'][:-1] + (np.diff(df_to_yeo['yeo_adj_demarc']) * 0.5)));
    ax.set_yticklabels(df_to_yeo['yeo_name'], fontsize=3.0, rotation=45)

plt.show()


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Plot Subgraphs
Can you write Python code for it?

%matplotlib inline

# Load the consensus data
data = np.load("{}/NMF_Consensus.Param.All.npz".format(path_ExpData), mmap_mode='r')
#data = np.load("{}/NMF_Surrogate.Param.50.npz".format(path_ExpData), mmap_mode='r')
fac_subnet = data['fac_subnet']
fac_coef = data['fac_coef']

n_fac = fac_subnet.shape[0]
n_conn = fac_subnet.shape[1]
n_win = fac_coef.shape[1]

# Plot subgraph matrix
plt.figure()
ax = plt.subplot(111)
mat = ax.matshow(fac_subnet.T, aspect=float(n_fac)/n_conn, cmap='rainbow')
#plt.colorbar(mat, ax=ax)

ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
#ax.set_xticks(np.linspace(0, 80, 5))
ax.set_ylabel('Functional Interactions')
ax.set_xlabel('Subgraphs')

plt.savefig('{}/Subgraph-Cfg_Matrix.svg'.format(path_Figures))
plt.show()
plt.close()

# Plot subgraph adjacency
plt.figure()
n_row = np.floor(np.sqrt(n_fac))
n_col = np.ceil(n_fac / n_row)
for ii, subg in enumerate(fac_subnet):
    adj = convert_conn_vec_to_adj_matr(subg)

    ax = plt.subplot(n_row, n_col, ii+1)
    mat = ax.matshow(adj, cmap='rainbow') #, vmin=0, vmax=1)
    #plt.colorbar(mat, ax=ax)
    ax.set_axis_off()

plt.savefig('{}/Subgraph-Adj_Matrices.svg'.format(path_Figures))
plt.show()
plt.close()

# Plot Coefficients
plt.figure()
ax = plt.subplot(111)

fac_coef = fac_coef.T
norm_fac = fac_coef - fac_coef.mean(axis=0)
for ff in xrange(n_fac):
    ax.plot(ff + norm_fac[:, ff] / (3*np.std(norm_fac[:, ff])), color=[66/256., 152/256., 221./256])

# Axis Settings
ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')
ax.set_ylim([-1, n_fac+1])
ax.set_xlim([0, int(n_win/28*3)])
ax.set_ylabel('Subgraphs')
ax.set_xlabel('Time Windows')

plt.savefig('{}/Subgraph-Coefs.svg'.format(path_Figures))
plt.show()
plt.close()


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Quality Measures in Parameter Space
Can you write Python code for it?

param_list = np.load('{}/NMF_CrossValidation.Param_List.npz'.format(path_ExpData))['param_list']
all_param, opt_param = Echobase.Network.Partitioning.Subgraph.optimize_nmf.find_optimum_xval_paramset(param_list, qmeas_list, search_pct=5.0)

n_bin = 100
srch_pct = 25.0
for param in ['rank', 'alpha', 'beta']:
    err_mean = stats.binned_statistic(all_param[param], all_param[qmeas],
                                      statistic=np.nanmean, bins=n_bin)
    err_std = stats.binned_statistic(all_param[param], all_param[qmeas],
                                     statistic=np.nanstd, bins=n_bin)
    mean_ix = np.flatnonzero(err_mean[0] < np.nanpercentile(err_mean[0], srch_pct))
    std_ix = np.flatnonzero(err_std[0] < np.nanpercentile(err_std[0], srch_pct))
    opt_param[param] = np.nanmean(err_mean[1][np.intersect1d(mean_ix, std_ix)])

    if param == 'rank':
        opt_param[param] = int(np.round(opt_param[param]))

print
print('Optimal Rank: {}'.format(opt_param['rank']))
print('Optimal Alpha: {}'.format(opt_param['alpha']))
print('Optimal Beta: {}'.format(opt_param['beta']))

np.savez('{}/NMF_CrossValidation.Optimal_Param.npz'.format(path_ExpData),
         opt_param=opt_param,
         all_param=all_param)

### Amass all parameters and compute optima based on discussion in (Ng (1997). ICML)
opt_dict = np.load('{}/NMF_CrossValidation.Optimal_Param.npz'.format(path_ExpData))
opt_param = opt_dict['opt_param'][()]
all_param = opt_dict['all_param'][()]

# Generate quality measure plots
for qmeas in ['test_error']:
    for param in ['rank', 'alpha', 'beta']:
        ax_jp = sns.jointplot(all_param[param], all_param[qmeas], kind='kde',
                              space=0, n_levels=60, shade_lowest=False)

        ax = ax_jp.ax_joint
        ax.plot([opt_param[param], opt_param[param]],
                [ax.get_ylim()[0], ax.get_ylim()[1]],
                lw=1.0, alpha=0.75, linestyle='--')

        ax.yaxis.set_ticks_position('left')
        ax.xaxis.set_ticks_position('bottom')
       ...


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Reduce Seed Subgraphs to Consensus Subgraphs
WARNING: Will Delete Existing Output
Can you write Python code for it?

# Reduce NMF consensus from all seed subgraphs
job_str = './NMF_consensus_reduce.py {} {} {}'.format(echobase_path, path_InpData, path_ExpData)
qsub_str = 'qsub {}'.format(job_str)

os.chdir('./e02-SGE_Scripts/')
!sh {qsub_str}
os.chdir('../')


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
SGE Helper Script for NMF Cross-Validation
Can you write Python code for it?

# Map NMF xval to all the parameter sets
job_str = './NMF_xval.py {} {} {}'.format(echobase_path, path_InpData, path_ExpData)
qsub_str = 'qsub -t 1-{} {}'.format(len(param_list), job_str)

os.chdir('./e02-SGE_Scripts/')
!sh {qsub_str}
os.chdir('../')

# Reduce NMF xval output to a qmeas_list
path_xval_out = glob.glob('{}/NMF_CrossValidation.Param.*.npz'.format(path_ExpData))
qmeas_list = [np.load(pth)['qmeas_dict'][()] for pth in path_xval_out]


I am working on the file "Analysis_Notebooks/e02-Detect_Dynamic_Subgraphs.ipynb".
The first task is:
Visualize Subgraphs
Surface render brain systems
Can you write Python code for it?

pixmap_path = '{}/brain_system_pixmap.npz'.format(path_Figures)
if os.path.exists(pixmap_path):
    brain_system_pixmap = np.load(pixmap_path)
else:
    from mayavi import mlab
    import nibabel as nib

    brain_system_pixmap = {}
    brain_system_rgbcol = {}

    sys_scalar = [15, 18, 6, 0, 3, 4, 2, 8, 10]
    view_angle = {'Sag_PA': [0.0, 90.0],
                  'Sag_AP': [180.0, 90.0]}

    # Get the pial surface recons
    pial_hemi = {'LH': {},
                 'RH': {}}
    pial_hemi['LH']['vert'], pial_hemi['LH']['tria'] = nib.freesurfer.io.read_geometry('{}/BrainRenderSubject15/surf/lh.pial'.format(path_CoreData))
    pial_hemi['RH']['vert'], pial_hemi['RH']['tria'] = nib.freesurfer.io.read_geometry('{}/BrainRenderSubject15/surf/rh.pial'.format(path_CoreData))

    # Get the Lausanne label files for each ROI
    label_files = []
    for roi in df_to_yeo['df_laus_yeo']:
        laus_lbl = roi[1].lower()
        hemi = roi[2].lower()

        # Parse the atlas name and find the label file if it exists
        lbl_file = '{}.{}.label'.format(hemi, laus_lbl)
        lbl_file = lbl_file.replace(' ', '')

        label_files.append('{}/BrainRenderSubject15/label/regenerated_{}_125/{}'.format(path_CoreData, hemi, lbl_file))

    # Iterate over hemisphere of the pial surface
    for hemi in pial_hemi.keys():
        n_vert = len(pial_hemi[hemi]['vert'])

        # Iterate over brain system
        for sys_id, sys_lbl in enumerate(df_to_yeo['yeo_name']):
            print(sys_lbl)
            sys_ix = np.flatnonzero(df_to_yeo['yeo_lbl'] == sys_lbl)

            # Find the label file for each ROI and get vertices
            if sys_lbl == 'SUB':
                pial_scalars = sys_scalar[sys_id]*np.ones(n_vert)
            else:
                pial_scalars = 15*np.ones(n_vert)

            for roi_ix, (roi, lbl_file) in enumerate(zip(df_to_yeo['df_laus_yeo'], label_files)):
                if roi[2] != hemi:
                    continue

                if not...


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
!!! NMF TEST !!!
Can you write Python code for it?

cfg_matr = df_cfg['cfg_matr'].copy()
#cfg_matr = (df_cfg['cfg_matr'].T * df_cfg['cfg_L2']).T

rank = 13
alpha = 1e-7
beta = 1e-7
n_fac = rank

# Grab the task ID of the current job (and the associated parameter dictionary)
fac_subnet = np.random.uniform(low=0, high=1.0, size=(rank, cfg_matr.shape[1]))
fac_coef = np.random.uniform(low=0, high=1.0, size=(rank, cfg_matr.shape[0]))

# Run NMF Algorithm
fac_subnet, fac_coef, err = Echobase.Network.Partitioning.Subgraph.nmf.snmf_bcd(
    cfg_matr, alpha=alpha, beta=beta, fac_subnet_init=fac_subnet, fac_coef_init=fac_coef,
    max_iter=20, sparse_dim='conn', verbose=True)


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
Expression Constrasts
Stroop vs Navon
Expression Plot
Can you write Python code for it?

%matplotlib inline

coef_ix = np.array(df_cfg['cfg_obs_lut'], dtype=int)
alpha = 0.05

del_expr_mean_stroop = []
del_expr_stdv_stroop = []
del_expr_mean_navon = []
del_expr_stdv_navon = []
tv = []
pv = []
clr = []

for ii, fac_key in enumerate(np.sort(sort_fac_dict.keys())):
    fac_ii = sort_fac_dict[fac_key]

    sel_fac_coef = df_subgraph['system_subgraph'][fac_ii]['expr_coef'][coef_ix]
    pos_expr = sel_fac_coef[:, :, :, :, 0, :]
    neg_expr = sel_fac_coef[:, :, :, :, 1, :]

    del_expr = (pos_expr-neg_expr).mean(axis=-1)[:, :, :, 1].mean(axis=-1)
    del_expr_mean = del_expr.mean(axis=0)
    del_expr_stdv = del_expr.std(axis=0) / np.sqrt(n_subj)

    del_expr_mean_stroop.append(del_expr_mean[0])
    del_expr_stdv_stroop.append(del_expr_stdv[0])
    del_expr_mean_navon.append(del_expr_mean[1])
    del_expr_stdv_navon.append(del_expr_stdv[1])

    # Within-experiment Stats
    tv_expr, pv_expr = stats.ttest_rel(*del_expr.T)
    tv.append(tv_expr)
    pv.append(pv_expr)

#
is_sig = multicomp.multipletests(pv, alpha=alpha, method='fdr_bh')[0]
for sig_bool in is_sig:
    if sig_bool:
        clr.append('r')
    else:
        clr.append([0.2, 0.2, 0.2])

print(stats.spearmanr(del_expr_mean_stroop, del_expr_mean_navon))

plt.figure(figsize=(3,3), dpi=300)
ax = plt.subplot(111)
for ii in xrange(n_fac):
    xx = del_expr_mean_stroop[ii]
    yy = del_expr_mean_navon[ii]

    ax.scatter(xx, yy, lw=0, color=clr[ii], s=10.0)
    ax.plot([xx, 1/2*(xx+yy)],
            [yy, 1/2*(xx+yy)], color=clr[ii])

min_val, max_val = -3.5, 8

ax.plot([min_val, max_val],
        [min_val, max_val], 'k', alpha=0.1)

ax.vlines(0, min_val, max_val, 'r')
ax.hlines(0, min_val, max_val, 'r')

ax.set_xlim([min_val, max_val])
ax.set_ylim([min_val, max_val])

ax.yaxis.set_ticks_position('left')
ax.xaxis.set_ticks_position('bottom')

ax.set_xlabel('Stroop: Relative Expression')
ax.set_ylabel('Navon: Relative Expression')

plt.savefig('{}/Stroop_vs_Navon.svg'.format(path_Figures))
plt.show()
plt.close()


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
Generate list of data
Load Data
Can you write Python code for it?

%matplotlib inline

# Load Functional Data
df_cfg = np.load('{}/Population.Configuration_Matrix.npz'.format(path_InpData_Netw))
df_subgraph = np.load('{}/Yeo_Subgraph.All.npz'.format(path_InpData_Subg))
df_to_yeo = np.load('{}/Lausanne125_to_Yeo.npz'.format(path_InpData_Netw))
df_nmf = np.load("{}/NMF_Consensus.Param.All.npz".format(path_InpData_Subg),
                 mmap_mode='r')

n_subj, _, _, _, _, n_block = df_cfg['cfg_obs_lut'].shape
n_fac = len(df_subgraph['system_subgraph'])

surr_coef = np.array([np.load(pth, mmap_mode='r')['fac_coef'][...]
                      for pth in glob.glob('{}/NMF_Surrogate.Param.*.npz'.format(path_InpData_Subg))])
surr_coef = surr_coef.reshape(-1, surr_coef.shape[-1])

# Load Behavioral Data
df_blk = io.loadmat('{}/BlockwiseDataCorrectTrialsOnly.mat'.format(path_CoreData))
bad_subj_ix = [1, 6]
good_subj_ix = np.setdiff1d(np.arange(n_subj+2), bad_subj_ix)
"""
'high control accuracy', 'low control accuracy', 'high control mean RT',
'high control median RT', 'low control mean RT', 'low control median RT'
"""
df_perf = {'Stroop': {'lo': {'accuracy': df_blk['StroopData'][good_subj_ix, 1, :],
                             'meanRT': df_blk['StroopData'][good_subj_ix, 4, :],
                             'medianRT': df_blk['StroopData'][good_subj_ix, 5, :]},
                      'hi': {'accuracy': df_blk['StroopData'][good_subj_ix, 0, :],
                             'meanRT': df_blk['StroopData'][good_subj_ix, 2, :],
                             'medianRT': df_blk['StroopData'][good_subj_ix, 3, :]}
                     },
           'Navon' : {'lo': {'accuracy': df_blk['NavonData'][good_subj_ix, 1, :],
                             'meanRT': df_blk['NavonData'][good_subj_ix, 4, :],
                             'medianRT': df_blk['NavonData'][good_subj_ix, 5, :]},
                      'hi': {'accuracy': df_blk['NavonData'][good_subj_ix, 0, :],
                             'meanRT': df_blk['NavonData'][good_subj_ix, 2, :],
                             'medianRT':...


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
Lo vs Hi
Expression Plot
Can you write Python code for it?

%matplotlib inline

coef_ix = np.array(df_cfg['cfg_obs_lut'], dtype=int)

for expr_ii, expr_id in enumerate(df_cfg['cfg_key_label'][()]['Experiment_ID']):
    del_expr_mean_lo = []
    del_expr_stdv_lo = []
    del_expr_mean_hi = []
    del_expr_stdv_hi = []
    pv = []
    tv = []
    clr = []

    for ii, fac_key in enumerate(np.sort(sort_fac_dict.keys())):
        fac_ii = sort_fac_dict[fac_key]

        sel_fac_coef = df_subgraph['system_subgraph'][fac_ii]['expr_coef'][coef_ix]
        pos_expr = sel_fac_coef[:, expr_ii, :, :, :, :][:, :, :, 0, :][:, :, 1, :]
        neg_expr = sel_fac_coef[:, expr_ii, :, :, :, :][:, :, :, 1, :][:, :, 1, :]

        del_expr = (pos_expr-neg_expr).mean(axis=-1)
        del_expr_mean = del_expr.mean(axis=0)
        del_expr_stdv = del_expr.std(axis=0) / np.sqrt(n_subj)

        del_expr_mean_lo.append(del_expr_mean[0])
        del_expr_stdv_lo.append(del_expr_stdv[0])
        del_expr_mean_hi.append(del_expr_mean[1])
        del_expr_stdv_hi.append(del_expr_stdv[1])

        # Within-experiment Stats
        tv_expr, pv_expr = stats.ttest_rel(*del_expr.T)
        pv.append(pv_expr)
        print(tv_expr, pv_expr)

    for is_sig in Echobase.Statistics.FDR.fdr.bhp(pv, alpha=0.05, dependent=True):
        if is_sig:
            clr.append('r')
        else:
            clr.append([0.2, 0.2, 0.2])

    print(stats.spearmanr(del_expr_mean_lo, del_expr_mean_hi))

    plt.figure(figsize=(3,3), dpi=300)
    ax = plt.subplot(111)
    for ii in xrange(n_fac):
        xx = del_expr_mean_lo[ii]
        yy = del_expr_mean_hi[ii]

        ax.scatter(xx, yy, lw=0, color=clr[ii], s=10.0)
        ax.plot([xx, 1/2*(xx+yy)],
                [yy, 1/2*(xx+yy)], color=clr[ii])

    min_val, max_val = -3.5, 8

    ax.plot([min_val, max_val],
            [min_val, max_val], 'k', alpha=0.1)

    ax.vlines(0, min_val, max_val, 'r')
    ax.hlines(0, min_val, max_val, 'r')

    ax.set_xlim([min_val, max_val])
   ...


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
Regional Contribution to Modulation in Performance
Can you write Python code for it?

from mayavi import mlab
import nibabel as nib

cmap = 'RdBu'
view_angle = {'Sag_PA': [0.0, 90.0],
              'Sag_AP': [180.0, 90.0]}

# Get the pial surface recons
pial_hemi = {'LH': {},
             'RH': {}}
pial_hemi['LH']['vert'], pial_hemi['LH']['tria'] = nib.freesurfer.io.read_geometry('{}/BrainRenderSubject15/surf/lh.pial'.format(path_CoreData))
pial_hemi['RH']['vert'], pial_hemi['RH']['tria'] = nib.freesurfer.io.read_geometry('{}/BrainRenderSubject15/surf/rh.pial'.format(path_CoreData))

# Get the Lausanne label files for each ROI
label_files = []
for roi in df_to_yeo['df_laus_yeo'][df_to_yeo['sort_laus_to_yeo']]:
    laus_lbl = roi[1].lower()
    hemi = roi[2].lower()

    # Parse the atlas name and find the label file if it exists
    lbl_file = '{}.{}.label'.format(hemi, laus_lbl)
    lbl_file = lbl_file.replace(' ', '')

    label_files.append('{}/BrainRenderSubject15/label/regenerated_{}_125/{}'.format(path_CoreData, hemi, lbl_file))

subgraph_partc_pixmap = {}
for expr_ii, expr_id in enumerate(subgraph_partc.keys()):
    xmax = [0.005, 0.01]

    for cnd_id in subgraph_partc[expr_id].keys():

        real_partc = subgraph_partc[expr_id][cnd_id]['real'].copy()
        null_partc = subgraph_partc[expr_id][cnd_id]['null'].copy()

        pv = np.mean(np.abs(null_partc) > np.abs(real_partc), axis=0)
        is_sig = Echobase.Statistics.FDR.fdr.bhp(pv, alpha=0.05, dependent=True)

        real_partc[~is_sig] = 0

        # Iterate over hemisphere of the pial surface
        for hemi in pial_hemi.keys():
            n_vert = len(pial_hemi[hemi]['vert'])
            pial_scalars = np.zeros(n_vert)

            # Assign subcortical
            nonparc_lbl = []
            for roi_ii, roi in enumerate(df_to_yeo['df_laus_yeo'][df_to_yeo['sort_laus_to_yeo']]):
                lbl_file = label_files[roi_ii]
                sys_lbl = roi[3]

                if roi[2] != hemi:
                    continue

                if (sys_lbl == 'CRB') or (sys_lbl ==...


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
Subgraph Topology
Can you write Python code for it?

%matplotlib inline

coef_ix = np.array(df_cfg['cfg_obs_lut'], dtype=int)

mean_rel_expr = {}
corep_index = {}

n_laus = len(df_to_yeo['yeo_lbl'])
n_yeo = len(df_to_yeo['yeo_name'])
alpha = 0.05 / (2*n_yeo)

fsize = 5.5
fig = plt.figure(figsize=(6, 6), dpi=300)
for ii, fac_key in enumerate(np.sort(sort_fac_dict.keys())):
    fac_ii = sort_fac_dict[fac_key]

    sel_fac_coef = df_subgraph['system_subgraph'][fac_ii]['expr_coef'][coef_ix]
    pos_expr = sel_fac_coef[:, :, :, :, 0, :]
    neg_expr = sel_fac_coef[:, :, :, :, 1, :]
    del_expr = (pos_expr-neg_expr).mean(axis=-1).mean(axis=-1).mean(axis=-1).mean(axis=-1).mean(axis=-1)
    mean_rel_expr[fac_key] = del_expr

    intra_sys = df_subgraph['system_subgraph'][fac_ii]['intra_sys']
    inter_sys = df_subgraph['system_subgraph'][fac_ii]['inter_sys']
    null_intra_sys = df_subgraph['system_subgraph'][fac_ii]['null_intra_sys']
    null_inter_sys = df_subgraph['system_subgraph'][fac_ii]['null_inter_sys']

    sig_intra_sys = np.mean(null_intra_sys > intra_sys, axis=0) < alpha
    sig_inter_sys = np.mean(null_inter_sys > inter_sys, axis=0) < alpha

    intra_sys_thresh = intra_sys.copy()
    inter_sys_thresh = inter_sys.copy()
    intra_sys_thresh[~sig_intra_sys] = 0
    inter_sys_thresh[~sig_inter_sys] = 0

    corep_index[fac_key] = np.nanmean((intra_sys_thresh - inter_sys_thresh) / \
                                      (intra_sys_thresh + inter_sys_thresh))

    intra_colors = np.array(['k' for xx in xrange(n_yeo)])
    inter_colors = np.array(['k' for xx in xrange(n_yeo)])

    intra_colors[sig_intra_sys] = 'r'
    inter_colors[sig_inter_sys] = 'r'

    ax = fig.add_subplot(3, 4, ii+1)
    ax.barh(np.arange(n_yeo), intra_sys, color=intra_colors, lw=0)
    ax.barh(np.arange(n_yeo), -1*inter_sys, color=inter_colors, lw=0)
    ax.plot([0.0, 0.0], [0.0, n_yeo], color='r', lw=1.0)

    for x_ii, (low_x, high_x) in enumerate(zip(np.percentile(null_intra_sys, 100-100*alpha, axis=0),
                                     ...


I am working on the file "Analysis_Notebooks/e03-Subgraph_Cognitive_Control-Dynamics.ipynb".
The first task is:
Task Performance
Behavioral Correlation
Can you write Python code for it?

%matplotlib inline

coef_ix = np.array(df_cfg['cfg_obs_lut'], dtype=int)
perf_id = 'meanRT'

subgraph_perf = {}
for expr_ii, expr_id in enumerate(df_cfg['cfg_key_label'][()]['Experiment_ID']):
    perf_lo = df_perf[expr_id]['lo'][perf_id].mean(axis=-1)
    perf_hi = df_perf[expr_id]['hi'][perf_id].mean(axis=-1)

    expr_perf = {}
    for cnd_ii, cnd_id in enumerate(df_cfg['cfg_key_label'][()]['Condition_ID']):

        cnd_perf = {}
        for ii, fac_key in enumerate(np.sort(sort_fac_dict.keys())):
            fac_ii = sort_fac_dict[fac_key]
            sel_fac_coef = df_subgraph['system_subgraph'][fac_ii]['expr_coef'][coef_ix]

            pos_expr = sel_fac_coef[:, expr_ii, :, :, :, :][:, cnd_ii, 1, 0, :]
            neg_expr = sel_fac_coef[:, expr_ii, :, :, :, :][:, cnd_ii, 1, 1, :]
            rel_expr = pos_expr-neg_expr

            cnd_perf[fac_key] = stats.spearmanr(rel_expr.mean(axis=-1),
                                                perf_hi-perf_lo)
        expr_perf[cnd_id] = cnd_perf
    subgraph_perf[expr_id] = expr_perf


I am working on the file "Analysis_Results.ipynb".
The first task is:

Figure 2g: Bar graphs of death & survival rates for age groups within each class/sex grouping.  The death and survival rates for each class/sex population are also included.
Note: The data shows children had the highest rates of survival in each sex/class population, with the exception of 1st class, female passengers. Figure 2f revealed that on average, female adults had higher survival rates than female children. However, we now see that this only occurred within the 1st class population.
The numbers also show that children and adults traveling in 3rd class had the lowest rates of survival, with the exception of adult males. Adult male passengers traveling in 2nd class had lower survival rates than those in 3rd class, which expands on the findings concerning male averages in Figure 1e.
Can you write Python code for it?

data = freq3_store(titanic['Pclass'],
            titanic['Sex'],titanic['age_group'])

counts = data[0]['female'].ix[['1st', '2nd', '3rd', 'All'], ['Child', 'Adult']]
perc = data[1]['female'].ix[['1st', '2nd', '3rd', 'All'], ['Child', 'Adult']]

perc.columns = ['% of Children', '% of Adults']

table = pd.concat((counts['Child'], perc['% of Children'], counts['Adult'], perc['% of Adults']), axis=1)
table.index.name = None

table

fem_data = freq4_store(titanic['Pclass'],
                       titanic['Sex'], titanic['Survived'],
                       titanic['age_group'])[0]['female']

child = fem_data['Died']['Child']
child_death = child / table['Child']

adult = fem_data['Died']['Adult']
adult_death = adult / table['Adult']

ftable = pd.concat((child, child_death, adult, adult_death), axis=1)
ftable.columns = ['Child Deaths', 'Death Rate', 'Adult Deaths', 'Death Rate']

ftable


I am working on the file "Analysis_Results.ipynb".
The first task is:

Figure 3d: Bar graphs depicting death and survival rates across classes within each sex and age group. The population rates for each sex and age group were also added for comparison.
_Note: Again we see that 3rd class passengers experienced the lowest survival rates across each sex and age groups.  The largest imblance is found within children, who survived at rates of 92% (1st) and 91%(2nd) in the upper classes and just 37% in 3rd class.  A similar trend is seen in the female population, with 1st and 2nd class survival rates of 97% & 92%, respectively, compared to only 50% in 3rd class. Adult data followed suit, with survival rates of 64% in 1st class, 41% in 2nd, and 20% in 3rd.  Interestingly, the male passenger data slighty deviated from this pattern. The highest survival rates still belonged to 1st class passengers (37%), however the survival rates for 2nd class (16%) were nearly as low as 3rd class (14%).__
Can you write Python code for it?

ez_graph2(titanic['Pclass'],
          titanic['age_group'], titanic['Survived'],
         'Adult', 'Adult Class Distribution')

ez_graph2(titanic['Pclass'],
          titanic['age_group'], titanic['Survived'],
         'Child', 'Child Class Distribution')

ez_graph2(titanic['Pclass'],
          titanic['Sex'], titanic['Survived'],
         'male', 'Male Class Distribution')

ez_graph2(titanic['Pclass'],
          titanic['Sex'], titanic['Survived'],
         'female', 'Female Class Distribution')


I am working on the file "Analysis_Results.ipynb".
The first task is:

Appendix B
Exploration of Data Relationships
This appendix contains the exploration of relationships between different variables.  The majority of the analysis was centered on the search for correlations to death and/or survival.
The dataworkflow.visualize module contains the majority of the functions used in this section for manipulating and displaying the data.
The majority of functions used for manipulating and displaying data in this analysis were created in the dataworkflow.visualize module.
Can you write Python code for it?

titanic = get_data()

titanic.head(10) #sanity


I am working on the file "Analysis_Results.ipynb".
The first task is:
Age

Missing data (177, 714/891)
Contains estimates for ages under 1
interesting var, can also be used to create alternate interesting vars

Decision: Transform keep current vals, keep missing data empyt for now, Create new variable for Child, Adults(>18), and Unknown
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Cabin

Missing large number of values (687, 204/891)
Mix of unique(147) and duplicated(57) data
Appears to be too little data for analysis

Decision: Delete
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Class & Age Group
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Create New Variables
Can you write Python code for it?

#Family Count
titanic['FamilyTot'] = titanic['SibSp'] + titanic['Parch']

#Single or Family Variable
bins = [-1,0,np.inf]
labels = ['Single','Family']
fam_status = pd.cut(titanic['FamilyTot'], bins, labels=labels)

fam_status.describe() #sanity check

#Add FamStatus
titanic['FamStatus'] = fam_status

#Adult (>18) or Child (<18) Var
bins = [0, 17, 1000, np.inf]
labels = ['Child', 'Adult', 'Unknown']
age_groups = pd.cut(titanic['Age'], bins, labels=labels)

age_groups.describe() #sanity check

#Add age_groups
titanic['age_group'] = age_groups

titanic.head() #sanity check


I am working on the file "Analysis_Results.ipynb".
The first task is:
Discussion
1. Did Passenger Sex Impact Survival?
1.1 Population Statistics
Can you write Python code for it?

sex_a = freq2_store(titanic['Sex'], titanic['age_group'])

sex_tab1 = sex_a[2].ix[['female', 'male'],['Child','Adult','RowTotals']]

sex_tab1.index = ['Female', 'Male']
sex_tab1.columns = ['Child','Adult','Population']

sex_tab1

sex_tab2 = sex_a[1].ix[['female', 'male', 'Totals'], ['Child', 'Adult', 'Unknown']]

sex_tab2.index = ['Female', 'Male', 'Population']
sex_tab2.columns = ['Child', 'Adult', 'Age Unknown']

sex_tab2.transpose()


I am working on the file "Analysis_Results.ipynb".
The first task is:
Embarked

Missing 2 values
Categorical stored as Object
Useful for analysis

Decision: Modify keep values, update dtype to category
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Evaluate Variables
PassangerID

Full set of data (891)
Unique numerical values for every passenger
Does not appear relevant for analysis

Decision: Delete
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Explore Data
Dimension/Type
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 1a: (1) Frequency table depicting the percentages of males and females within each age group (Child < 18, Adult => 18), as well as the general population. (2) Frequency table depicting the percentage of children and adults within each of the passenger sexes.  The overall percentage of adults and children in the population has also been included for comparison.
Note: The ship's population consisted of 65% male and 35% female passengers. Male passengers also made up the majority of both adults (66%) and children (51%). Children made up a larger percentage of female passengers (18%) than male passengers (10%).  Each sex had roughly the same percentage of adults – female: 66%, male: 68%.
Can you write Python code for it?

sex_c = freq2_store(titanic['Sex'],
                    titanic['Pclass'])[1][['1st','2nd', '3rd']]*100
sex_c.index = ['Female','Male', 'Population']

ez_bar(sex_c, 'Class Distribution', '% of Total')

sex_c.transpose()/100


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 1b: Frequency table and bar graph showing how each sex and the overall population were spread across the 3 classes on the Titanic.
_Notes: The majority of both sexes traveled in 3rd class, followed by 1st and then 2nd. The same pattern can be seen in the general population as well. Female passengers were fairly evenly spread across class, with the largest portion in 3rd class (45%). Male passengers were more skewed towards 3rd class. In fact, there were more male passengers in 3rd class than both the 1st and 2nd combined. _
1.2 Survival Statistics
Can you write Python code for it?

sex_surv = freq2_store(titanic['Survived'],
                       titanic['Sex'])[2].ix[['Died', 'Lived']]*100
sex_surv.columns = ['Female', 'Male', 'Population']

ez_bar(sex_surv.transpose(), 'Death & Survival Rates', '% of Total')

sex_surv/100


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 1c: Frequency table & bar graph depicting passenger survival & death rates for both sexes and the overall population.
Notes: On whole, female passengers (74%) had more than 3x higher survival rates than male passengers (19%). Roughly, 3/4 of female passengers survived aboard the titanic, compared to only 1/5 of male passengers.
Can you write Python code for it?

#frq = freq2_store(titanic['Sex'],titanic['Survived'])[2].ix[:-1]

sex_s = freq2_store(titanic['Survived'],
                    titanic['Sex'])[1][['female','male']]*100
sex_s.index = ['Died', 'Lived', 'Population']
sex_s.columns = ['Female','Male']

ez_bar(sex_s, 'Distribution of Sexes', '% of Total')

sex_s.transpose()/100


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 2b: Stacked bar graph depicting the percentage of each age group across the sexes and general population.
Notes: Adults (18+) made up 67% of the population, children (under 18) made up 13%, and 20% of the passenger's ages were unknown.  We know from the previous section that the majority of adults were male (66%) and that children were split more evenly between females (49%) and males (51%).
Can you write Python code for it?

age_cls = freq2_store(titanic['age_group'],
                      titanic['Pclass'])[1].ix[[0,1,3],
                                               ['1st', '2nd', '3rd']]*100
age_cls.index = ['Child', 'Adult', 'Population']

ez_bar(age_cls, 'Class Distribution', '% of Total')

print('Class Dsitribution')

age_cls.transpose()/100


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 2c: Frequency table and stacked bar graph depicting the class distribution of each age group and of the overall population.
Notes: The majority of adults and children traveled in 3rd class. There were more than twice as many children in 3rd class (69%) than in the 1st (11%) and 2nd (20%) classes combined. Adults were distributed across the classes in a pattern similar to the general population, with the majority in 3rd (46%) class followed by 1st (29%) and then 2nd (25%).
2.2 Survival Statistics
Can you write Python code for it?

age_surv = freq2_store(titanic['Survived'], titanic['age_group'])
pt = age_surv[2].ix[['Died','Lived'],['Child','Adult','RowTotals']]
pt.columns = ['Child', 'Adult', 'Population']
print('Death & Survival Rates')
pt

ag =age_surv[1][['Child', 'Adult', 'Unknown']].transpose()
ag.columns = ['Died','Lived','Population']
print('Age Group Distributions')
ag


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 2d: Frequency tables depicting (1) death and survival rates for each age group and (2) distribution of age groups across dead & surviving passengers. Population statistics were added to each table for comparison.
_Notes: On average, children had higher survival rates (54%) than adults (38%). Surviving passengers were made up of 18% children, 67% adults, and 15% of an unknown age. Adults were represented in both the dead and surviving passengers at approximately the same rate as they existed in the general population. (~67%) Children were overrepresented in the surviving group (~5%) and underrepresented in the dead. (~3%)
Can you write Python code for it?

#create tables
tab = freq3_store(titanic['Survived'],titanic['Pclass'], titanic['age_group'])[1]
tot_by_class = freq2_store(titanic['Survived'], titanic['Pclass'])[2].ix[['Died', 'Lived'],['1st', '2nd', '3rd']]
#1st class
tab_1st = tab['1st'].ix[['Died','Lived'],['Child','Adult']]*100
tab_1st.index.name = None
tab_1st.columns.name = None
tab_1st['Population'] = tot_by_class['1st']*100

#2nd class
tab_2nd = tab['2nd'].ix[['Died','Lived'],['Child','Adult']]*100
tab_2nd.index.name = None
tab_2nd.columns.name = None
tab_2nd['Population'] = tot_by_class['2nd']*100

#3rd class
tab_3rd = tab['3rd'].ix[['Died','Lived'],['Child','Adult']]*100
tab_3rd.index.name = None
tab_3rd.columns.name = None
tab_3rd['Population'] = tot_by_class['3rd']*100

#stacked bar graphs
ez_bar(tab_1st.transpose(), '1st Class Death & Survival Rates', '% of Total')
ez_bar(tab_2nd.transpose(), '2nd Class Death & Survival Rates', '% of Total')
ez_bar(tab_3rd.transpose(), '3rd Class Death & Survival Rates', '% of Total')


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 3E: Frequency tables & bar graphs depicting the class distribution of passengers who died, survived, and existed within each sex and age group.
Note: The numbers show the majority of dead passengers from each group were traveling in 3rd class - adults (59%), children (94%), males (64%), females (88%). When comparing these numbers to the population statistics, we see that 3rd class passengers were overrepresented in each case. The most lopsided numbers are found with children, whose 3rd class travelers accounting for 94% of the dead and only 68% of the population.
_The data also shows that 1st class passengers made up the largest portion of surviving adults (48%) and females (39%). In both cases, first class passengers appeared at higher percentages than they did in their respective populations. Men on the other hand had nearly as many 1st (41%) and 3rd (43%) class survivors. However, this is misleading as there were nearly 3 times as many male passengers traveling in 3rd class (60%) as there were in 1st (21%). The majority of surviving children were traveling in 2nd (34%) and 3rd (48%) class, but this too is misleading. In comparison to the rate they appeared in the population, both 1st (+7%) and 2nd (+14%) class survivors were overrepresented, while 3rd class (-21%) survivors were underrepresented. _
Can you write Python code for it?

#Death & Survival rates by class for each Sex/Age Group

data = freq4_store(titanic['Survived'],titanic['age_group'],titanic['Sex'],titanic['Pclass'])[1]

data_totals = freq3_store(titanic['Survived'], titanic['age_group'], titanic['Sex'])[1]

for sex in ['female', 'male']:
    for val in ['Child', 'Adult']:
        #create data table
        table = data[val][sex].ix[['Died', 'Lived'],['1st', '2nd', '3rd']]
        table['Population'] = data_totals[val][sex]
        table.index.name = None
        table.columns.name = None
        table*100

        print ("\n\n {} ({}) Death & Survival Rates \n\n".format(sex.capitalize(),val), table)


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 3G: Frequency tables depicting class distribution of the dead and surviving passengers for each sex/age grouping.  The class distribution for each sex/age groups population has also been included for comparison.
Note:  Passengers from 3rd class represent a larger portion of the dead and smaller portion of the survivors than they do in their respective populations for each of the sex/age groupings. The most dramatic swing is seen in the adult female population, whose 3rd class passengers accounted for 83% of the dead and only 33% of the population.  The data for 1st class passengers in each sex/age group was reversed. They accounted for a smaller percentage of the dead and larger percentage of the survivors than existed in their entire populations.  The largest discrepancy was found in adult males, with 51% of the survivors traveling in 1st class compared to only 25% of the overall population.
3.3 Summary
A clear pattern emerged along class lines when analyzing the data set. In nearly every instance, we saw the highest chances of survival existed in 1st class, followed by 2nd, and then 3rd.  We also saw that 1st and 2nd class passengers represented a larger percentage of survivors and a lower percentage of the dead than they did in the population. While the 3rd class passengers experienced the opposite, they were underrepresented in the survivors and overrepresented in the dead compared to their population rates. This data indicates that class played a major role in survival aboard the titanic. The 1st and 2nd classes yielded much higher chances for survival, while traveling in 3rd class resulted in a higher likelihood of death.
Limitations
While the findings in this study were interesting, additional analysis is required to validate the assumption included in this report. The results will need to be tested for statistical significance and expirementation will be required to confirm which (if any) of the variables identified in this analysis are causal.
There were also limitations in the dataset...
Can you write Python code for it?

path = '/Users/jco/Desktop/data_science/Udacity/project_2/Titanic/train.csv'

titanic = pd.read_csv(path)


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 3b: Frequency table & bar graph showing class distribution within the dead & surviving populations. The class breakdown for the entire population has been included for comparison.
Note: The majority of passengers who died aboard the titanic were traveling in 3rd class (68%). When comparing the dead & surviving passenger distributions to the population we see that 3rd class passengers accounted for a larger portion of the dead (+13%) and a smaller portion of the living (-20%). 1st and 2nd class followed the exact opposite pattern, accounting for less of the dead (-9%, -3%) and more of the survivors. (+16%, +4%)
Can you write Python code for it?

tb = freq2_store(titanic['Pclass'],
                 titanic['Survived'])[1][['Died','Lived']]*100
tb.index = ['1st', '2nd', '3rd', 'Population']

ez_bar(tb,'Death & Survival Rates', '% of total')

tb.transpose()/100


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 3c: Frequency table & bar graph depicting death & survival rates for each class and the overall population.
Note: Passengers traveling in 1st class were most likely to survive (63%) and passengers traveling 3rd class were least likely to survive (24%). Death rates for 3rd class passengers (76%) were slightly more than double that of 1st class passengers (37%). A familiar trend appears when comparing class rates against the population, 3rd class passengers have lower survival and higher death rates, while 1st and 2nd class passengers experience the opposite.
Can you write Python code for it?

ez_graph3(titanic['Survived'],
         titanic['age_group'],titanic['Pclass'],
         'Child', 'Child Survival & Death Rates')

ez_graph3(titanic['Survived'],
          titanic['Sex'],titanic['Pclass'],
          'female', 'Female Survival & Death Rates')

ez_graph3(titanic['Survived'],
         titanic['age_group'],titanic['Pclass'],
         'Adult', 'Adult Survival & Death Rates')

ez_graph3(titanic['Survived'],
         titanic['Sex'],titanic['Pclass'],
         'male', 'Male Survival & Death Rates')


I am working on the file "Analysis_Results.ipynb".
The first task is:
Figure 3f: Frequency table of death & survival rates by class for each sex/age group pairing.  The death and survival rates for each population is also included for comparison.
Note: The 3rd class passengers from each sex/age grouping experienced higher than average death rates and lower than average survival rates in comparison to their populations.  The 1st class passengers from each of these groups experienced the opposite, with below average death rates and above average survival rates.
Upper class (1st & 2nd) passengers experienced the highest survival rates across each sex/age grouping, with the exception of adult males. This is line with our findings from Figure 3d. However, we now see the reason male passengers didn't fit the pattern was due to the high mortality rate of 2nd class, adult males.  In fact, this is the only instance where 3rd class passengers had higher survival rates than 2nd class. The effect of this on the overall male population was partially offset by the children, whose survival rates along class lines matched the trend seen in the other population sub-groups.
Can you write Python code for it?

#Death/Survival Rates Per Sex/Age Group

data = freq4_store(titanic['Pclass'], titanic['Sex'], titanic['age_group'],titanic['Survived'])[1]

totals = freq3_store(titanic['Pclass'], titanic['Sex'], titanic['age_group'])[1]

for sex in ['female', 'male']:
    for val in ['Child', 'Adult']:
        #create data table
        table = data[sex][val].ix[['1st', '2nd', '3rd']]
        table['Population'] = totals[sex][val]
        table.index.name = None
        table.columns.name = None
        table*100

        print ("\n Class Distribution of {} ({}) Passengers  \n\n".format(sex.capitalize(),val), table)


I am working on the file "Analysis_Results.ipynb".
The first task is:
Name

No missing values
All unique
Seperate variables exist for identifying passangers w/ family members
Does not appear relevant for analysis

Decision: Delete
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Parch

variable for # of parents or childeren aboard
no missing values
Would be interesting to use for identification of passengers w/ fam

Decision: Merge combine with SibSp to create new var identifying passengers w/ fam & count
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Pclass

Full set of data
Categorical Data stored as numerical 1=1st Class, 2=2nd Class, 3=3rd Class
Interesting variable for analysis

Decision: Modify convert to ordered category, replace nums with text (1st, 2nd, 3rd)
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Sample
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Sex

No missing values
Stored as object vs. category
Interesting variable for analysis

Decision: Modify keep values, convert dtype to category
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Sex & Age Group
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
SibSp

Variable for # of sibling or spouses aboard
No missing values
Would be interesting to use for identification of passengers with family

Decision: Merge combine with Parch to create new variables identifying passengers traveling with family and how many
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survival by AgeGroup & Sex
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survival by Class & Sex
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survival by Class, Sex, AgeGroup
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survival by FamilyStatus
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survival by Fare
Can you write Python code for it?

lived_fares = titanic['Fare'][titanic['Survived']=='Lived'].reset_index(drop=True)
died_fares = titanic['Fare'][titanic['Survived']=='Died'].reset_index(drop=True)

print('\n--------------Lived-----------\n', lived_fares.describe())
print('\n--------------Died-------------\n', died_fares.describe())

sns.boxplot(data=[lived_fares, died_fares])

sns.distplot(lived_fares, rug=True)

sns.distplot(died_fares, rug=True)


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survived by Age
Can you write Python code for it?

lived_age = titanic['Age'][titanic['Survived'] == 'Lived'].reset_index(drop=True)
died_age = titanic['Age'][titanic['Survived'] == 'Died'].reset_index(drop=True)

print('--------Lived------\n', lived_age.describe(), end='\n\n')
print('--------Died--------\n',died_age.describe())

sns.boxplot(data=[lived_age, died_age])


I am working on the file "Analysis_Results.ipynb".
The first task is:
Survived by AgeGroup
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Ticket

No missing values
Mix of unique and duplicates (681 uni, 310 dupes)
Mix of alphanumeric and numeric values
Does not appear relevant for analysis

Decision: Delete
Can you write Python code for it?


I am working on the file "Analysis_Results.ipynb".
The first task is:
Update/Add Variables
Numerical Category Data
Can you write Python code for it?

#Survived
new_survived = pd.Categorical(titanic['Survived'])
new_survived = new_survived.rename_categories(['Died', 'Lived'])

new_survived.describe()

#Pclass
new_pclass = pd.Categorical(titanic['Pclass'], ordered=True)
new_pclass = new_pclass.rename_categories(['1st','2nd','3rd'])

new_pclass.describe()

#Replace with categorical data
titanic['Survived'] = new_survived
titanic['Pclass'] = new_pclass

#sanity check
print (titanic['Survived'].dtype, titanic['Pclass'].dtype)


I am working on the file "Analysis_Results.ipynb".
The first task is:
__Survival by Class_
Can you write Python code for it?


I am working on the file "Analysis_Variance.ipynb".
The first task is:
Data analysis
The distribution of the three variables have been analyzed previously.
Variance analysis
Now that the univariate distribution as be plotted and described, the bivariate graphics will be plotted in order to test our research hypothesis.
Let's first focus on the primary research question;

The explanatory variable is the internet use rate (quantitative variable)
The response variable is the suicide per 100,000 people (quantitative variable)

From the scatter plot, a slope slightly positive has been seen. And as most of the countries have no or very low internet use rate, an effect is maybe seen only on the countries having the higher internet use rate.
Can you write Python code for it?

subdata2 = subdata.assign(internet_grp4 = pd.qcut(subdata.internetuserate, 4,
                                       labels=["1=25th%tile", "2=50th%tile",
                                               "3=75th%tile", "4=100th%tile"]))
sns.factorplot(x='internet_grp4', y='suicideper100th', data=subdata2,
               kind="bar", ci=None)
plt.xlabel('Internet use rate (%)')
plt.ylabel('Suicide per 100 000 people (-)')
_ = plt.title('Average suicide per 100,000 people per internet use rate quartile')


I am working on the file "Analysis_Variance.ipynb".
The first task is:
General information on the Gapminder data
Can you write Python code for it?

display(Markdown("Number of countries: {}".format(len(data))))
display(Markdown("Number of variables: {}".format(len(data.columns))))

# Convert interesting variables in numeric format
for variable in ('internetuserate', 'suicideper100th', 'employrate'):
    data[variable] = pd.to_numeric(data[variable], errors='coerce')


I am working on the file "Analysis_Variance.ipynb".
The first task is:
The first records of the data restricted to the three analyzed variables are:
Can you write Python code for it?

subdata = data[['internetuserate', 'suicideper100th', 'unemployrate']]
subdata.head(10)


I am working on the file "Analysis_Variance.ipynb".
The first task is:
The p-value found is 0.143 > 0.05. Therefore the null hypothesis cannot be rejected. There is no relationship between the internet use rate and the suicide rate.
Test case to fulfill the assignment
In order to fulfill the assignment, I will switch to the NESARC database.
Can you write Python code for it?

nesarc = pd.read_csv('nesarc_pds.csv', low_memory=False)

races = {1 : 'White',
         2 : 'Black',
         3 : 'American India/Alaska',
         4 : 'Asian/Native Hawaiian/Pacific',
         5 : 'Hispanic or Latino'}
subnesarc = (nesarc[['S3BQ4', 'ETHRACE2A']]
             .assign(ethnicity=lambda x: pd.Categorical(x['ETHRACE2A'].map(races)),
                     nb_joints_day=lambda x: (pd.to_numeric(x['S3BQ4'], errors='coerce')
                                                .replace(99, np.nan)))
             .dropna())

g = sns.factorplot(x='ethnicity', y='nb_joints_day', data=subnesarc,
                   kind="bar", ci=None)
g.set_xticklabels(rotation=90)
plt.ylabel('Number of cannabis joints per day')
_ = plt.title('Average number of cannabis joints smoked per day depending on the ethnicity')