I am working on the file "PT3S.ipynb".
The first task is:
TimeCurve Data
verfügbare Szenariumzeit
Can you write Python code for it?

tStartSz=mx.df.index[0]
tEndeSz=mx.df.index[-1]
tStartSz

tEndeSz

timeSpanSz=tEndeSz-tStartSz
timeSpanSz


I am working on the file "PT3S.ipynb".
The first task is:
Vector: Aggregate

TIME: MXS: SIR 3S t=0-Ergebnis als instat. AB (kein Aggregat) - implizit von _readMxsFile
TMIN,TMAX: MXS: SIR 3S Min./Max. - implizit von _readMxsFile
or Aggregates from 2 Times: MIN,MAX,... - implizit von getVecAggs
Can you write Python code for it?


I am working on the file "PT3S.ipynb".
The first task is:
about from PT3S ... import ... and pip install -e .
Can you write Python code for it?

#%pwd

# works only if pip install -e .is NOT active:
###from PT3S import Mx,Xm,Rm
# or if subdir is PT3S
#%cd -q ..
# ...
#%cd -q PT3S

# if pip install -e .IS active
# the local modules can be imported with:
#from PT3S
import Mx, Xm, Rm
# if %pwd is local devdir

import doctest
"""
>>> import pandas as pd
>>> pd.__version__
'1.2.4'
"""
doctest.testmod()


I am working on the file "PT3S.ipynb".
The first task is:
ausgewählter Zeitbereich
Can you write Python code for it?

tStart=tStartSz
tEnde=tEndeSz

tcData=mx.df.loc[tStart:tEnde,:]

tcData.index[0]

tcData.index[-1]


I am working on the file "PT3S.ipynb".
The first task is:
für Darstellungen, die mit 1 Zeit bzw. mit Aggregaten 1 Zeitraums arbeiten:
Start, Ende, Min, Max
Can you write Python code for it?

def mxAdd(xm,mx,timeReq=None,aggReq=None,timeReq2nd=None):
    xm.MxAdd(mx=mx,timeReq=timeReq,aggReq=aggReq,timeReq2nd=timeReq2nd)
    vAGSN=xm.dataFrames['vAGSN']
    vKNOT=xm.dataFrames['vKNOT']
    vROHR=xm.dataFrames['vROHR']
    vFWVB=xm.dataFrames['vFWVB']
    vVBEL=xm.dataFrames['vVBEL']

    vAGSN=xm.dataFrames['vAGSN']
    vAGSN['PH']=vAGSN.apply(lambda row: row.P*math.pow(10.,5.)/(row.RHO*9.81),axis=1)
    vAGSN['PH']=vAGSN['PH']+vAGSN['Z'].astype('float64')
    zBzg=30.
    vAGSN['bBzg']=vAGSN.apply(lambda row: row.RHO*9.81/math.pow(10.,5.),axis=1)
    vAGSN['zBzg']=           (vAGSN['Z'].astype('float64')-zBzg)*vAGSN['bBzg']
    vAGSN['zBzg0']=          (vAGSN['Z'].astype('float64')-0   )*vAGSN['bBzg']
    vAGSN['zBzgMin']=        (vAGSN['Z'].astype('float64')-vAGSN['Z'].astype('float64').min())*vAGSN['bBzg']
    vAGSN['bBzg']=vAGSN['P']+vAGSN['zBzg']

    hpSL=vAGSN[(vAGSN['LFDNR']=='1') & (vAGSN['Layer']==1)]
    hpRL=vAGSN[(vAGSN['LFDNR']=='1') & (vAGSN['Layer']==2)]

    vROHR=vROHR[(vROHR.apply(lambda x: True if x.CONT_ID == '1001' else False,axis=1))]

    return vAGSN,vKNOT,vROHR,vFWVB,vVBEL,hpSL,hpRL

vAGSN_TStart,vKNOT_TStart,vROHR_TStart,vFWVB_TStart,vVBEL_TStart,hpSL_Start,hpRL_Start=mxAdd(xm,mx,timeReq=mx.df.index[0])
vAGSN_TEnde,vKNOT_TEnde,vROHR_TEnde,vFWVB_TEnde,vVBEL_TEnde,hpSL_Ende,hpRL_Ende=mxAdd(xm,mx,timeReq=mx.df.index[-1])
vAGSN_SMin,vKNOT_SMin,vROHR_SMin,vFWVB_SMin,vVBEL_SMin,hpSL_SMin,hpRL_SMin=mxAdd(xm,mx,aggReq='TMIN')
vAGSN_SMax,vKNOT_SMax,vROHR_SMax,vFWVB_SMax,vVBEL_SMax,hpSL_SMax,hpRL_SMax=mxAdd(xm,mx,aggReq='TMAX')

# besserer Weg mehrere Zeiten auf 1x zu erhalten:
xm.MxAdd(mx=mx
,aggReq=['TIME','TMIN','TMAX','TIME'] # Start, Min, Max, Ende # z.B. P	P_1	P_2	P_3
,timeReq=3*[mx.df.index[0]]+[mx.df.index[-1]]
,timeReq2nd=4*[mx.df.index[-1]]
,viewList=['vAGSN','vKNOT','vROHR','vFWVB','vVBEL']
,ForceNoH5Update=True)

vAGSN=xm.dataFrames['vAGSN']
vAGSN.filter(items=['P','P_1','P_2','P_3'],axis=1).head()


I am working on the file "PT3S.ipynb".
The first task is:
ggf. Tests
Modultests

die Modultests (-m 1) gehen teilweise von einem definierten "leeren" Ausgangszustand aus: Relikte aus alten Testläufen sollten daher gelöscht werden (-t before), wenn die Modultests vollständig ohne Fehler durchlaufen sollen
die Modultests hinterlassen teilweise "Überbleibsel", die explizit gelöscht werden sollten (-t after ==> -t both)
es müssen nur die Modelle (-w ...) behandelt werden (vorher bzw. nachher aufräumen), die in den Modultests auch verwendet werden
es werden alle Modultests durchgeführt

nachfolgende Tests erzeugen (trotz -q) auch bei fehlerfreiem Durchlauf eine Ausgabe im Notebook
Can you write Python code for it?

%run test.py -l -q -m 1 -t both \
-w OneLPipe -w LocalHeatingNetwork -w GPipe -w GPipes -w TinyWDN

# mit SIR 3S Daten (Modelle + Ergebnisse) aus Version 11
%run test.py -l -q -m 1 -t both \
-w OneLPipe -w LocalHeatingNetwork -w GPipe -w GPipes -w TinyWDN \
--testDir testdata11

# mit SIR 3S Daten (Modelle + Ergebnisse) aus Version 10
%run test.py -l -q -m 1 -t both \
-w OneLPipe -w LocalHeatingNetwork -w GPipe -w GPipes -w TinyWDN \
--testDir testdata10

# mit SIR 3S Daten (Modelle + Ergebnisse) aus Version 09
%run test.py -l -q -m 1 -t both \
-w OneLPipe -w LocalHeatingNetwork -w GPipe -w GPipes -w TinyWDN \
--testDir testdata09 --dotResolution NONE


I am working on the file "PT3S.ipynb".
The first task is:
regelungstechnisches Signalmodell
Can you write Python code for it?

xmlFile=os.path.join(path,'testdata\DHNetwork.XML')
xm=Xm.Xm(xmlFile=xmlFile)
mx=xm.MxAdd()

vRSTN=xm.dataFrames['vRSTN']

rstnDiePGRPStellen=vRSTN[[
         'CONT'
        #,'CONT_PARENT'
        ,'KA'
        ,'BESCHREIBUNG'
        ,'ITYP_OBJTYPE'
        ,'ITYP_OBJATTR'
        ,'Chk'
        ,'ik_Chk'
      #  ,'OBJTYPE'
        ,'NAME_i'
        ,'NAME_k'
        ,'CONT_i'
      #  ,'TABL_Chk'
      #  ,'TABL'
      #  ,'KNOT'
      #  ,'RART'
      #  ,'RART_TYP'
        ,'RARTPG'
      #  ,'RCPL'
      #  ,'RCPL_KNOT1'
      #  ,'RCPL_KNOT2'
      #  ,'NAME_i_PUMP'
      #  ,'NAME_k_PUMP'
        ]].sort_values(by=['ITYP_OBJTYPE','ITYP_OBJATTR','CONT','KA'])[vRSTN['ITYP_OBJTYPE']=='PGRP']
rstnDiePGRPStellen

rstnDiePGRPStellen[rstnDiePGRPStellen['NAME_i']=='R-A-SS']

# Verbindungslinien
vREdges=xm.dataFrames['vREdges']

# Signalweg (mit Knotentyp_INFO)
G=nx.from_pandas_edgelist(vREdges, source='KnExt_Ki', target='KnExt_Kk', edge_attr=True,create_using=nx.DiGraph())
nx.shortest_path(G,'Leck_1_Ein_RSLW','KA-0008_RSTN')

# Signalweg (ohne Knotentyp)
G=nx.from_pandas_edgelist(vREdges, source='Kn_Ki', target='Kn_Kk', edge_attr=True,create_using=nx.DiGraph())
nx.shortest_path(G,'Leck_1_Ein','KA-0008')

G=nx.from_pandas_edgelist(vREdges, source='Kn_Ki', target='Kn_Kk', edge_attr=True,create_using=nx.Graph())
nl=[n for n in G.nodes()]
A=nx.to_scipy_sparse_matrix(G)
nlo=scipy.sparse.csgraph.reverse_cuthill_mckee(A)
optnl=[nl[idx] for idx in nlo]
spmtx=nx.laplacian_matrix(G,nodelist=optnl)
plt.spy(spmtx)


I am working on the file "PVPY Demo.ipynb".
The first task is:
Recreate the famous efficiency versus bandgap curves for the 4 default illumination spectra. You can always define your own custom spectrum by passing a (2,N) numpy array to the spectra keyword
Can you write Python code for it?

bandgaps = np.linspace(.35,4,125)
spectras = ["AM1.5G", "AM1.5D", "AM0Etr",  "BlackBody",]
for spectra in spectras:
    def effofbandgap(bandgap):
        cell = pvpy.SolarCell(bandgap=bandgap)
        spectrum = pvpy.PhotocurrentSpectrum(spectra=spectra)
        cell.set_illumination(spectrum)
        return cell.get_efficiency() *100
    effciencies = [effofbandgap(bandgap) for bandgap in bandgaps]
    plt.plot(bandgaps, effciencies)
plt.ylabel(r"Single Junction Efficiency")
plt.xlabel("Bandgap Energy (eV)")
plt.legend(spectras)


I am working on the file "PVPY Demo.ipynb".
The first task is:
pvpy allows you to easily plot photon or photocurrent spectra too. Either create the spectrum directly, or convert an existing spectrum to the desired.  Spectrum objects are: PowerSpectrum (W/m^2), PhotonSpectrum (photons / (m^2 s)), PhotocurrentSpectrum (A/m^2)
Can you write Python code for it?

# create a photon spectrum directly
am15g = pvpy.PhotonSpectrum(spectra="AM1.5G")
# or convert the previous power spectrum
bb5800.to_PhotonSpectrum()
# The spectrum objects hold their spectrum as an Nx2 numpy array that you can call with the dot operator, or
# more formally via the get_spectrum() method
plt.plot(am15g[0], am15g[1], bb5800[0], bb5800[1])
plt.ylabel(r"photons $m^{-2} s^{-1}$")
plt.xlabel("Wavelength (nm)")


I am working on the file "PVQ/ApproximatingPulses.ipynb".
The first task is:
Although the left bias is present in every sequence of the dataset, the negative vs positive bias appears to be content specific.
Estimating $K_{\text{Left}}$
Now on to the main event, predicting the value of K for the left half.
Can you write Python code for it?

akiyoLeftError = np.abs(akiyoPulses[:,0:7]).sum(axis=1) - np.abs(akiyoApproxPulses[:,0:7]).sum(axis=1)
ducksLeftError = np.abs(ducksPulses[:,0:7]).sum(axis=1) - np.abs(ducksApproxPulses[:,0:7]).sum(axis=1)
grandmaLeftError = np.abs(grandmaPulses[:,0:7]).sum(axis=1) - np.abs(grandmaApproxPulses[:,0:7]).sum(axis=1)
parkLeftError = np.abs(parkPulses[:,0:7]).sum(axis=1) - np.abs(parkApproxPulses[:,0:7]).sum(axis=1)
signLeftError = np.abs(signPulses[:,0:7]).sum(axis=1) - np.abs(signApproxPulses[:,0:7]).sum(axis=1)
sintelLeftError = np.abs(sintelPulses[:,0:7]).sum(axis=1) - np.abs(sintelApproxPulses[:,0:7]).sum(axis=1)
soccerLeftError = np.abs(soccerPulses[:,0:7]).sum(axis=1) - np.abs(soccerApproxPulses[:,0:7]).sum(axis=1)

print("Average Absolute Approximation Error Left (in pulses)")
print("Akiyo: %0.2f" % np.mean(np.abs(akiyoLeftError)))
print("Ducks: %0.2f" % np.mean(np.abs(ducksLeftError)))
print("Grandma: %0.2f" % np.mean(np.abs(grandmaLeftError)))
print("Park: %0.2f" % np.mean(np.abs(parkLeftError)))
print("Sign: %0.2f" % np.mean(np.abs(signLeftError)))
print("Sintel: %0.2f" % np.mean(np.abs(sintelLeftError)))
print("Soccer: %0.2f" % np.mean(np.abs(soccerLeftError)))


I am working on the file "PVQ/ApproximatingPulses.ipynb".
The first task is:
It appears we are much better at predicting $K_{\text{Right}}$ than we are for $K_{\text{Left}}$.
The Glass is not half empty
Turns out that predicting the left half is tricky, because the codewords vary from our estimates based on elements specific to the content. However, the right half is much more stable, and since we know K, we can use the complement to estimate $K_{\text{left}}$
Can you write Python code for it?

akiyoLeftErrorC = np.abs(akiyoPulses[:,0:7]).sum(axis=1) - (akiyoKs - np.abs(akiyoApproxPulses[:,7:15]).sum(axis=1))
ducksLeftErrorC = np.abs(ducksPulses[:,0:7]).sum(axis=1) - (ducksKs - np.abs(ducksApproxPulses[:,7:15]).sum(axis=1))
grandmaLeftErrorC = np.abs(grandmaPulses[:,0:7]).sum(axis=1) - (grandmaKs - np.abs(grandmaApproxPulses[:,7:15]).sum(axis=1))
parkLeftErrorC = np.abs(parkPulses[:,0:7]).sum(axis=1) - (parkKs - np.abs(parkApproxPulses[:,7:15]).sum(axis=1))
signLeftErrorC = np.abs(signPulses[:,0:7]).sum(axis=1) - (signKs - np.abs(signApproxPulses[:,7:15]).sum(axis=1))
sintelLeftErrorC = np.abs(sintelPulses[:,0:7]).sum(axis=1) - (sintelKs - np.abs(sintelApproxPulses[:,7:15]).sum(axis=1))
soccerLeftErrorC = np.abs(soccerPulses[:,0:7]).sum(axis=1) - (soccerKs - np.abs(soccerApproxPulses[:,7:15]).sum(axis=1))

print("Average Absolute Approximation Error Left (in pulses)")
print("Akiyo: %0.2f" % np.mean(np.abs(akiyoLeftErrorC)))
print("Ducks: %0.2f" % np.mean(np.abs(ducksLeftErrorC)))
print("Grandma: %0.2f" % np.mean(np.abs(grandmaLeftErrorC)))
print("Park: %0.2f" % np.mean(np.abs(parkLeftErrorC)))
print("Sign: %0.2f" % np.mean(np.abs(signLeftErrorC)))
print("Sintel: %0.2f" % np.mean(np.abs(sintelLeftErrorC)))
print("Soccer: %0.2f" % np.mean(np.abs(soccerLeftErrorC)))


I am working on the file "PVQ/ApproximatingPulses.ipynb".
The first task is:
Note that with this approach we have fractional pulses. The absolute sum of these fractional pulses is K. However, if we round, floor or ceil the pulses, the absolute sum may no longer be K.
Based on this original estimate, the encoder will try to play with the pulses to obtain integer pulses and to improve the rate distortion ratio of the codeword.
Let's measure the mean absolute error between our approximated pulses and the pulses of the Daala encoder.
Can you write Python code for it?

akiyoError = np.abs(akiyoPulses - akiyoApproxPulses).sum(axis=1)
ducksError = np.abs(ducksPulses - ducksApproxPulses).sum(axis=1)
grandmaError = np.abs(grandmaPulses - grandmaApproxPulses).sum(axis=1)
parkError = np.abs(parkPulses - parkApproxPulses).sum(axis=1)
signError = np.abs(signPulses - signApproxPulses).sum(axis=1)
sintelError = np.abs(sintelPulses - sintelApproxPulses).sum(axis=1)
soccerError = np.abs(soccerPulses - soccerApproxPulses).sum(axis=1)

print("Average Absolute Approximation Error (in pulses)")
print("Akiyo: %0.2f" % np.mean(akiyoError))
print("Ducks: %0.2f" % np.mean(ducksError))
print("Grandma: %0.2f" % np.mean(grandmaError))
print("Park: %0.2f" % np.mean(parkError))
print("Sign: %0.2f" % np.mean(signError))
print("Sintel: %0.2f" % np.mean(sintelError))
print("Soccer: %0.2f" % np.mean(soccerError))


I am working on the file "PVQ/ApproximatingPulses.ipynb".
The first task is:
On average, our estimate is off by one or two pulses.
Error distribution
Now, let's look at how the error distributes over the codewords.
Can you write Python code for it?

plt.figure(figsize=(16,20));
plt.subplot(4,2,1);
p = plt.stem(np.abs(akiyoPulses - akiyoApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)");
plt.xlabel("Codeword index");
plt.title("Akiyo");
plt.gca().set_xlim([-0.5, 14.5])

plt.subplot(4,2,2);
plt.stem(np.abs(ducksPulses - ducksApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)");
plt.xlabel("Codeword index");
plt.title("Ducks");
plt.gca().set_xlim([-0.5, 14.5]);

plt.subplot(4,2,3);
plt.stem(np.abs(grandmaPulses - grandmaApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)");
plt.xlabel("Codeword index");
plt.title("Grandma");
plt.gca().set_xlim([-0.5, 14.5]);

plt.subplot(4,2,4);
plt.stem(np.abs(parkPulses - parkApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)")
plt.xlabel("Codeword index");
plt.title("Park");
plt.gca().set_xlim([-0.5, 14.5]);

plt.subplot(4,2,5)
plt.stem(np.abs(signPulses - signApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)");
plt.xlabel("Codeword index");
plt.title("Sign");
plt.gca().set_xlim([-0.5, 14.5]);

plt.subplot(4,2,6);
plt.stem(np.abs(sintelPulses - sintelApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)");
plt.xlabel("Codeword index");
plt.title("Sintel");
plt.gca().set_xlim([-0.5, 14.5]);

plt.subplot(4,2,7);
plt.stem(np.abs(soccerPulses - soccerApproxPulses).mean(axis=0));
plt.ylabel("Mean Absolute Error(in pulses)");
plt.xlabel("Codeword index");
plt.title("Soccer");
plt.gca().set_xlim([-0.5, 14.5]);


I am working on the file "PVQ/ApproximatingPulses.ipynb".
The first task is:
Questions:

"Given a starting pulse allocation based on L1 energy, what's the range of K values I need to consider on each side to ensure I get the optimal some (large) percentage of the time?"
Can you write Python code for it?

print("Percentage of time KLeft estimate +-x is not optimal for x=0, x=1, x=2")

def checkCorrect(absErr):
    numElements = absErr.shape
    return (np.array([(absErr != 0).sum(), (absErr > 1).sum(), (absErr > 2).sum()]) / numElements * 100)

print("AKiyo: %s" % (checkCorrect(np.abs(akiyoLeftErrorC))))
print("Ducks: %s" % (checkCorrect(np.abs(ducksLeftErrorC))))
print("Grandma: %s" % (checkCorrect(np.abs(grandmaLeftErrorC))))
print("Park: %s" % (checkCorrect(np.abs(parkLeftErrorC))))
print("Sign: %s" % (checkCorrect(np.abs(signLeftErrorC))))
print("Sintel: %s" % (checkCorrect(np.abs(sintelLeftErrorC))))
print("Soccer: %s" % (checkCorrect(np.abs(soccerLeftErrorC))))

print("Percentage of time KRight estimate +-x is not optimal for x=0, x=1, x=2")

def checkCorrect(absErr):
    numElements = absErr.shape
    return (np.array([(absErr != 0).sum(), (absErr > 1).sum(), (absErr > 2).sum()]) / numElements * 100)

print("AKiyo: %s" % (checkCorrect(np.abs(akiyoRightError))))
print("Ducks: %s" % (checkCorrect(np.abs(ducksRightError))))
print("Grandma: %s" % (checkCorrect(np.abs(grandmaRightError))))
print("Park: %s" % (checkCorrect(np.abs(parkRightError))))
print("Sign: %s" % (checkCorrect(np.abs(signRightError))))
print("Sintel: %s" % (checkCorrect(np.abs(sintelRightError))))
print("Soccer: %s" % (checkCorrect(np.abs(soccerRightError))))


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
As you noticed, we refer to the norm of v as the gain of v, which represents the amount of energy in the v (I know there's a norm function in numpy, but doing it this way shows you why the gain measures energy).
The next thing we need is the unit vector of v:
Can you write Python code for it?

def shape(v):
    return np.true_divide(v, gain(v))
print("Unit vector (aka shape): %s" % (shape(v)))


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
By adding more codes to our codebook, we decrease the error. Let's draw a plot to better understand what happens to the error when we increase k.
Can you write Python code for it?

plt.figure(figsize=(15,5))
sad = []
sse = []
numKs = 20

for k in range(2,numKs):
    recon = shape(findCode(v,k)) * gain(v)
    print("k = %d Reconstructed Codeword = %s" % (k, recon))
    sad.append(sum(abs(recon - v)))
    sse.append(sum((recon - v)**2))

plt.plot(range(2,numKs), sad, label='SAD', c="orange")
plt.plot(range(2,numKs), sse, label='SSE', c="red")
plt.xlabel("k")
legend = plt.legend(loc='upper center', shadow=True)


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
By using the shape of the v instead of the v itself, all vectors with the same shape will have the same codeword. Vectors with the same shape, are vectors that point in the same direction. In other words all different scales of the same vector. There's more to it than that, but for now let's focus on building the codebook.
Building the codebook
You might have imagined the codebook as a big book of codes, each code cherry picked by an engineer for optimal performance. This is not that type of codebook, basically a simple equation is used to generate all possible values. The equation is: The sum of absolute values of the codeword must sum to k.
Just for fun (because we won't need it later) let's build the codebook. This is probably not the fastest way to build the codebook, but it should be easy to understand. We have 2 nested loops because we have 2 elements in v. Since the absolute value operator is used, valid values range from -k to k.
Can you write Python code for it?

k = 2

def build2DCodeBook(k):
    codebook = []
    for x0 in range(-k,k+1):
        for x1 in range(-k,k+1):
            if abs(x0) + abs(x1) == k:
                codebook.append([x0, x1])
    return np.array(codebook)

def showCodeBook(codebook, k):
    plt.scatter(codebook[:][:,0], codebook[:][:,1], c="blue", label="Codewords", alpha=0.5, s=50)
    plt.scatter(v[0], v[1], c="red", label="v", alpha=0.5, s=50)
    plt.title("Codebook for k=%d" % (k))
    plt.legend(loc='lower right', shadow=True)
    plt.axis('equal')

codebook = build2DCodeBook(k)

print("Codebook for k=%d: \n%s" %(k,codebook))

showCodeBook(codebook, 2)


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
Geometrically, we notice that the codewords form a diamond shape, where the edges are at values of k in one axis. When we perform vector quantization, we chose a the code word closest to v as the codeword representing v. The distance between the codeword and v is the error.
Let's examine some other codeboooks
Can you write Python code for it?

plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
showCodeBook(build2DCodeBook(3), 3)

plt.subplot(2,2,2)
showCodeBook(build2DCodeBook(6), 6)

plt.subplot(2,2,3)
showCodeBook(build2DCodeBook(9), 9)

plt.subplot(2,2,4)
showCodeBook(build2DCodeBook(12), 12)


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
I guess if you look hard enough you can see the difference, but the truth is your eye is not really good a differentiating even and odd shades of gray. We can remove more shades of gray and see what quantization does
Can you write Python code for it?

plt.figure(figsize=(15,10))
plt.subplot(2,2,1)
showImage(quantize(im, 16))
plt.subplot(2,2,2)
showImage(quantize(im, 31))
plt.subplot(2,2,3)
showImage(quantize(im, 62))
plt.subplot(2,2,4)
showImage(quantize(im, 125))


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
Instead of "perceptually vector quantizing" v, we will "perceptually vector quantize" shape and "scalar quantize" gain.

Wait a second, this requires "perceptually vector quantizing" 4 values, plus "scalar quantizing" the gain. How is this better than just "perceptually vector quantizing" the 4 values of v?
Can you write Python code for it?

v2 = [2,1]
print("Gain of v2 = %s" % (gain(v2)))
print("Shape of v2 = %s" % (shape(v2)))

assert(shape(v).all() == shape(v2).all())


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
It's Quantizing time!
We will reduce the number of colors by a factor of 2. Each odd valued color will be replaced by the value of the even number right before it. For example, 43 becomes 42, we call this scalar quantization (scalar is just a fancy word for "one number").
Can you write Python code for it?

def quantize(im, step):
    return np.floor(im / step) * step

showImage(quantize(im, 2))


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
Now I get Quantization! So let's "perceptually vector quantize" that tiger image!

Hold on there, turns out it's a lot more complicated, so we will keep that for part 2 in this series. For now, let's "perceptually vector quantize" something a little simpler, like this vector v:
Can you write Python code for it?


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
Vector quantization is a fancy way of saying that we want to convert the whole vector into a single code. This code is called a codeword. The idea is that all the codewords are defined in a codebook. Before we present the codebook, let's start with the concepts of gain and shape, which will be needed to better understand how codewords work.
Gain and Shape
To "perceptually vector quantize" v, we must first compute the norm of v:
Can you write Python code for it?

def gain(v):
    return np.sqrt(np.dot(v,v))
print("Norm (aka gain) of v = %f" % (gain(v)))


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
We did it!
We "perceptually vector quantized" v! You can check the codeword is in the codebook.
The burning question now is: "how good is our quantization of v?". Let's find out:
Can you write Python code for it?

def computeError(codeword, v):
    recon = shape(codeword) * gain(v)
    print("Reconstructed v = %s" % (recon))
    print("Sum of Absolute Difference: %f" % (sum(abs(recon - v))))
    print("Sum of Squared Error: %f" % (sum((recon - v)**2)))

computeError(findCode(v,2), v)


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
We refer to the unit vector of v as the shape of v. As its name suggest, the values of this vector show the shape of the energy distribution of v. The shape vector is in the same direction as v but is squaled to unit length.
We can get back v from shape like so:
Can you write Python code for it?


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
Woah! normalizing changed the shape looks like a circle.

Yes, and it's no ordinary circle it's a unit circle (I admit that's just a fancy word for a circle of radius 1)
Let's look at what happens when we increase k:
Can you write Python code for it?

plt.figure(figsize=(15,15))
plt.subplot(3,2,1)
showShapedCodeBook(shapeCodeBook(build2DCodeBook(3)), 3)

plt.subplot(3,2,2)
showShapedCodeBook(shapeCodeBook(build2DCodeBook(6)), 6)

plt.subplot(3,2,3)
showShapedCodeBook(shapeCodeBook(build2DCodeBook(7)), 7)

plt.subplot(3,2,4)
showShapedCodeBook(shapeCodeBook(build2DCodeBook(12)), 12)

plt.subplot(3,2,5)
showShapedCodeBook(shapeCodeBook(build2DCodeBook(15)), 15)

plt.subplot(3,2,6)
showShapedCodeBook(shapeCodeBook(build2DCodeBook(20)), 20)


I am working on the file "PVQ/PVQ.ipynb".
The first task is:
Yikes! That's far from v.
Increasing k
We already know what happens when we increase k. If k = 3 the codeword should be a perfect match, let's try it out:
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
Board Verification
Project 2: 2 Switchs 4 LEDS
https://timetoexplore.net/blog/arty-fpga-verilog-01
myHDL Code
Can you write Python code for it?

@block
def S2L4(sw, clk, led):
    """
    FPGA Hello world of two switchs controlling four LED based on
    https://timetoexplore.net/blog/arty-fpga-verilog-01

    Target:
        ZYNQ 7000 Board (Arty, PYNQ-Z1, PYNQ-Z2) with at least 2
        switchs and 4 leds

    Input:
        sw(2bitVec):switch input
        clk(bool): clock input
    Ouput:
        led(4bitVec): led output

    """

    @always(clk.posedge)
    def logic():
        if sw[0]==0:
            led.next[2:]=0
        else:
            led.next[2:]=3

        if sw[1]==0:
            led.next[4:2]=0
        else:
            led.next[4:2]=3

    return instances()


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
Need to figure out how to write/run these long simulations better in python
Verilog Code
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
PYNQ-Z1 Constraints File
Below is what is found in file constrs_S0L0.xdc
Notice that the orgianl port names found in the PYNQ-Z1 Constraints file have been changed to the port names of the module S0L0
Verilog Testbench
Can you write Python code for it?

@block
def BDCLed_TBV():

    clk=Signal(bool(0))
    led=Signal(intbv(0)[4:])

    @always_comb
    def print_data():
        print(sw, clk, led)

    DUT=BDCLed(clk, led)

    @instance
    def clk_signal():
        while True:
            clk.next = not clk
            yield delay(1)

    @instance
    def stimules():
        i=0
        while True:
            if i==1000:
                raise StopSimulation()
            i+=1
            yield clk.posedge

    return instances()

TB=BDCLed_TBV()
TB.convert(hdl="Verilog", initial_values=True)
VerilogTextReader('BDCLed_TBV');


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
Verilog Testbench
PYNQ-Z1 Constraints File
Below is what is found in file constrs_S0L0.xdc
Notice that the orgianl port names found in the PYNQ-Z1 Constraints file have been changed to the port names of the module S0L0
Board Verification
Project 4: Basic Duty Cycle
https://timetoexplore.net/blog/arty-fpga-verilog-02
myHDL Code
Can you write Python code for it?

@block
def BDCLed(clk, led):
    counter=Signal(modbv(0)[8:])
    duty_led=Signal(modbv(8)[8:])

    @always(clk.posedge)
    def logic():
        counter.next=counter+1
        if counter<duty_led:
            led.next=15
        else:
            led.next=0

    return instances()


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
\begin{figure}
\centerline{\includegraphics[width=10cm]{S0L0_RTL.png}}
\caption{\label{fig:S0L0RTL} S0L0 RTL schematic; Xilinx Vivado 2017.4}
\end{figure}
\begin{figure}
\centerline{\includegraphics[width=10cm]{S0L0_SYN.png}}
\caption{\label{fig:S0L0SYN} S0L0 Synthesized Schematic; Xilinx Vivado 2017.4}
\end{figure}
\begin{figure}
\centerline{\includegraphics[width=10cm]{S0L0_SYN.png}}
\caption{\label{fig:S0L0SYN} S0L0 Implementated Schematic; Xilinx Vivado 2017.4}
\end{figure}
PYNQ-Z1 Constraints File
Below is what is found in file constrs_S0L0.xdc
Notice that the orgianl port names found in the PYNQ-Z1 Constraints file have been changed to the port names of the module S0L0
Verilog Testbench
Can you write Python code for it?

swTVal=intbv(int(''.join([str(i) for i in swTVals]), 2))[len(swTVals):]
print(f'swTest: {swTVals}, {swTVal}, {[int(i) for i in swTVal]}')

@block
def S0L0_TBV():
    clk=Signal(bool(0))
    sw=Signal(intbv(0)[2:])
    led=Signal(intbv(0)[4:])

    #test stimuli
    swTVals=Signal(swTVal)

    @always_comb
    def print_data():
        print(sw, clk, led)

    DUT=S0L0(sw, clk, led)

    @instance
    def clk_signal():
        while True:
            clk.next = not clk
            yield delay(1)

    @instance
    def stimules():
        for i in range(10):
            sw.next[0]=swTVals[i]
            yield clk.posedge
        raise StopSimulation()

    return instances()

TB=S0L0_TBV()
TB.convert(hdl="Verilog", initial_values=True)
VerilogTextReader('S0L0_TBV');


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
\begin{figure}
\centerline{\includegraphics[width=10cm]{S2L4_RTL.png}}
\caption{\label{fig:S2L4RTL} S2L4 RTL schematic; Xilinx Vivado 2017.4}
\end{figure}
\begin{figure}
\centerline{\includegraphics[width=10cm]{S2L4_SYN.png}}
\caption{\label{fig:S2L4SYN} S2L4 Synthesized Schematic; Xilinx Vivado 2017.4}
\end{figure}
\begin{figure}
\centerline{\includegraphics[width=10cm]{S2L4_IMP.png}}
\caption{\label{fig:S2L4SYN} S2L4 Implementated Schematic; Xilinx Vivado 2017.4}
\end{figure}
Verilog Testbench (ToDo)
will write later when testbench conversion is improved
PYNQ-Z1 Constraints File
using same one as in 1 Switch 1 LED: constrs_S0L0.xdc
Board Verification
Project 3: Countdown
myHDL Code
Can you write Python code for it?

@block
def countLED(clk, led):
    counter=Signal(modbv(0)[33:])

    @always(clk.posedge)
    def logic():
        counter.next=counter+1
        led.next[0]=counter[26]
        led.next[1]=counter[24]
        led.next[3]=counter[22]
        led.next[4]=counter[20]

    return instances()


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
myHDL Testing
Can you write Python code for it?

Peeker.clear()
clk=Signal(bool(0)); Peeker(clk, 'clk')
sw=Signal(intbv(0)[2:]); Peeker(sw, 'sw')
led=Signal(intbv(0)[4:]); Peeker(led, 'led')

np.random.seed(18)
swTVals=[int(i) for i in np.random.randint(0,4, 10)]

DUT=S2L4(sw, clk, led)

def S2L4_TB():

    @always(delay(1))
    def ClkGen():
        clk.next=not clk

    @instance
    def stimules():
        for i in range(10):
            sw.next=swTVals[i]
            yield clk.posedge
        raise StopSimulation()

    return instances()

sim=Simulation(DUT, S2L4_TB(), *Peeker.instances()).run()

Peeker.to_wavedrom()

Peeker.to_dataframe()


Peeker.clear()
clk=Signal(bool(0)); Peeker(clk, 'clk')
sw=Signal(intbv(0)[2:]); Peeker(sw, 'sw')
led=Signal(intbv(0)[4:]); Peeker(led, 'led')

np.random.seed(18)
swTVals=[int(i) for i in np.random.randint(0,2, 10)]

DUT=S0L0(sw, clk, led)

def S0L0_TB():

    @always(delay(1))
    def ClkGen():
        clk.next=not clk

    @instance
    def stimules():
        for i in range(10):
            sw.next[0]=swTVals[i]
            yield clk.posedge
        raise StopSimulation()

    return instances()

sim=Simulation(DUT, S0L0_TB(), *Peeker.instances()).run()

Peeker.to_wavedrom()

Peeker.to_dataframe()


Peeker.clear()
clk=Signal(bool(0)); Peeker(clk, 'clk')
led=Signal(intbv(0)[4:]); Peeker(led, 'led')

DUT=BDCLed(clk, led)

def BDCLed_TB():

    @always(delay(1))
    def ClkGen():
        clk.next=not clk

    @instance
    def stimules():
        i=0
        while True:
            if i==1000:
                raise StopSimulation()
            i+=1
            yield clk.posedge

    return instances()

sim=Simulation(DUT, BDCLed_TB(), *Peeker.instances()).run()

Peeker.to_wavedrom()

BDCLedData=Peeker.to_dataframe()
BDCLedData=BDCLedData[BDCLedData['clk']==1]
BDCLedData.plot(y='led');


I am working on the file "PYNQLearn/FabricOnly/.ipynb_checkpoints/myHDL_PYNQZ12_FabricOnly-checkpoint.ipynb".
The first task is:
pwm myHDL Testing
Can you write Python code for it?

Peeker.clear()
clk=Signal(bool(0)); Peeker(clk, 'clk')
dutyCount=Signal(intbv(4)[8:]); Peeker(dutyCount, 'dutyCount')
o_state=Signal(bool(0)); Peeker(o_state, 'o_state')

DUT=pwm(clk, dutyCount, o_state)

def pwm_TB():
    pass


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
Board Verification
Project 2: 2 Switchs 4 LEDS
https://timetoexplore.net/blog/arty-fpga-verilog-01
myHDL Code
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
Need to figure out how to write/run these long simulations better in python
Verilog Code
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
Project 1: 1 Switch 1 LED
https://timetoexplore.net/blog/arty-fpga-verilog-01
Constraints File
myHDL Code
Can you write Python code for it?

@block
def S0L0(sw, clk, led):
    """
    FPGA Hello world of one switch controlling one LED based on
    https://timetoexplore.net/blog/arty-fpga-verilog-01

    Target:
        ZYNQ 7000 Board (Arty, PYNQ-Z1, PYNQ-Z2) with at least 2
        switchs and 4 leds

    Input:
        sw(2bitVec):switch input
        clk(bool): clock input
    Ouput:
        led(4bitVec): led output

    """

    @always(clk.posedge)
    def logic():
        if sw[0]==0:
            led.next[0]=True
        else:
            led.next[0]=False

    return instances()


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
Verilog Code
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
Verilog Testbench
PYNQ-Z1 Constraints File
Below is what is found in file constrs_S0L0.xdc
Notice that the orgianl port names found in the PYNQ-Z1 Constraints file have been changed to the port names of the module S0L0
Board Verification
Project 4: Basic Duty Cycle
https://timetoexplore.net/blog/arty-fpga-verilog-02
myHDL Code
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
\begin{figure}
\centerline{\includegraphics[width=10cm]{S0L0_RTL.png}}
\caption{\label{fig:S0L0RTL} S0L0 RTL schematic; Xilinx Vivado 2017.4}
\end{figure}
\begin{figure}
\centerline{\includegraphics[width=10cm]{S0L0_SYN.png}}
\caption{\label{fig:S0L0SYN} S0L0 Synthesized Schematic; Xilinx Vivado 2017.4}
\end{figure}
\begin{figure}
\centerline{\includegraphics[width=10cm]{S0L0_SYN.png}}
\caption{\label{fig:S0L0SYN} S0L0 Implementated Schematic; Xilinx Vivado 2017.4}
\end{figure}
PYNQ-Z1 Constraints File
Below is what is found in file constrs_S0L0.xdc
Notice that the orgianl port names found in the PYNQ-Z1 Constraints file have been changed to the port names of the module S0L0
Verilog Testbench
Can you write Python code for it?


I am working on the file "PYNQLearn/FabricOnly/myHDL_PYNQZ12_FabricOnly.ipynb".
The first task is:
myHDL Testing
Can you write Python code for it?


I am working on the file "PYT_DS_Clustering.ipynb".
The first task is:
What are we doing there?  Along each row, each digit, we add some noise and create an "almost the same" digit, which tne gets added to X, making it twice as long.  Lets make sure we look at n_samples before and after.
Can you write Python code for it?

np.random.normal(size=2)

print(X[1796])

print(X[-1])

#----------------------------------------------------------------------
# Visualize the clustering
def plot_clustering(X_red, X, labels, title=None):
    x_min, x_max = np.min(X_red, axis=0), np.max(X_red, axis=0)
    X_red = (X_red - x_min) / (x_max - x_min)

    plt.figure(figsize=(6, 4))
    for i in range(X_red.shape[0]):
        plt.text(X_red[i, 0], X_red[i, 1], str(y[i]),
                 color=plt.cm.spectral(labels[i] / 10.),
                 fontdict={'weight': 'bold', 'size': 9})

    plt.xticks([])
    plt.yticks([])
    if title is not None:
        plt.title(title, size=17)
    plt.axis('off')
    plt.tight_layout()

#----------------------------------------------------------------------
# 2D embedding of the digits dataset
print("Computing embedding")
X_red = manifold.SpectralEmbedding(n_components=2).fit_transform(X)
print("Done.")

from sklearn.cluster import AgglomerativeClustering

for linkage in ('ward', 'average', 'complete'):
    clustering = AgglomerativeClustering(linkage=linkage, n_clusters=10)
    t0 = time()
    clustering.fit(X_red)
    print("%s : %.2fs" % (linkage, time() - t0))

    plot_clustering(X_red, X, clustering.labels_, "%s linkage" % linkage)

plt.show()

from IPython.display import YouTubeVideo
YouTubeVideo("EUQY3hL38cw")

from IPython.display import YouTubeVideo
YouTubeVideo("-Afa1WI3iug")


I am working on the file "PYT_DS_Clustering.ipynb".
The first task is:
manifold.SpectralEmbedding looks intriguing.  That gets used further on, after we've doubled the dataset with some clever "shadowing".
Can you write Python code for it?

np.random.seed(0)

def nudge_images(X, y):
    # Having a larger dataset shows more clearly the behavior of the
    # methods, but we multiply the size of the dataset only by 2, as the
    # cost of the hierarchical clustering methods are strongly
    # super-linear in n_samples
    shift = lambda x: ndimage.shift(x.reshape((8, 8)),
                                  .3 * np.random.normal(size=2),
                                  mode='constant',
                                  ).ravel()
    X = np.concatenate([X, np.apply_along_axis(shift, 1, X)])
    Y = np.concatenate([y, y], axis=0)
    return X, Y

X, y = nudge_images(X, y)

n_samples, n_features = X.shape
print("n_samples after:", n_samples)


I am working on the file "PY_Basics/TMWP_DictionaryBasics.ipynb".
The first task is:

keyDict object
Can you write Python code for it?

class KeyDict(dict):
    def __missing__(self, key):
        #self[key] = key  # uncomment if desired behavior is to add keys when they are not found (w/ key as value)
        #this version returns the key that was not found
        return key

kdTst = KeyDict(super_heroes)
print(kdTst['The Incredible Hulk'])
print(kdTst['Ant Man'])   # value not found so it returns itself as per __missing__ over-ride

help(SortedDict)


I am working on the file "PY_Basics/TMWP_DictionaryBasics.ipynb".
The first task is:

Working With Dictionaries and Nested Dictionaries - Helpful Code
This section has additional resources for working with dictionaries and nested dictionaries:

FileDataObj code - the FileDataObject stores contents from a file in a nested dictionary and explores sorting and summarising the nested dict
dictionary and nested dictionary functions in a PY module - merging dictionaries, adding to a nested dictionary, summarizing a nested dictionary, etc. (some of this code was created from the previous example)

Sorting
Can you write Python code for it?

# Help on Collections Objects including Counter, OrderedDict, dequeu, etc:
#  https://docs.python.org/2/library/collections.html

# regular dictionary does not necessarily preserve order (things added in randomly?)
# original order of how you add elements is prserved in OrderedDict

from collections import OrderedDict

myOrdDict = OrderedDict({'banana': 3, 'apple': 4, 'pear': 1, 'orange': 2})
print(myOrdDict)
myOrdDict['pork belly'] = 7
print(myOrdDict)
myOrdDict['sandwich'] = 5
print(myOrdDict)
myOrdDict['hero'] = 5
print(myOrdDict)

# sorting the ordered dictionary ...

# dictionary sorted by key
# replacing original OrderedDict w/ results
myOrdDict = OrderedDict(sorted(myOrdDict.items(), key=lambda t: t[0]))
print("myOrdDict (sorted by key):\n %s" %myOrdDict)

# dictionary sorted by value
myOrdDict2 = OrderedDict(sorted(myOrdDict.items(), key=lambda t: t[1]))
print("myOrdDict2 (sorted by value):\n %s" %myOrdDict2)

# dictionary sorted by length of the key string
myOrdDict3 = OrderedDict(sorted(myOrdDict.items(), key=lambda t: len(t[0])))
print("myOrdDict3 (sorted by length of key):\n %s" %myOrdDict3)

# collections.OrderedDict(sorted(dictionary.items(), reverse=True))
# pd.Series(OrderedDict(sorted(browser.items(), key=lambda v: v[1])))

# changing sort order to reverse key sort:
myOrdDict3 = OrderedDict(sorted(myOrdDict.items(), reverse=True))
print("myOrdDict3 (reverse key sort):\n %s" %myOrdDict3)

# testing of above strategy ... usually works but encountered  cases where it failed for no known reason
# lambda approach may be more reliable:

import pandas as pd

# value sort as pandas series:
myOrdDict4 = pd.Series(OrderedDict(sorted(myOrdDict.items(), key=lambda v: v[1])))
print("myOrdDict4 (value sort / alternate method):\n %s" %myOrdDict4)

# value sort in reverse order:
myOrdDict5 = OrderedDict(sorted(myOrdDict.items(), key=lambda t: (-t[1],t[0])))
print("myOrdDict5 (sorted by value in reverse order):\n %s" %myOrdDict5)

# Help on Collections Objects including Counter, OrderedDict, dequeu, etc:
# ...


I am working on the file "PY_Basics/TMWP_DictionaryBasics.ipynb".
The first task is:

nested dictionaries
Can you write Python code for it?

# list of dictionaries:

FoodList = [foods, {'meats':'beef', 'fruit':'banana', 'vegetables':'broccoli'}]
print(FoodList[0])
print(FoodList[1])

# dictionary of dictionaries (sometimes called "nested dictionary"):
# note: this is an example only.  In real world, sinde FoodList is inclusive of foods, you probably would not include both
#       uniform structures (same number of levels across all elements) is also advisable if possible

nestedDict = { 'heroes':super_heroes, 'foods': foods, 'complex_foods':FoodList }
print(nestedDict['heroes'])
print('-'*72)
print(nestedDict['complex_foods'])


I am working on the file "PY_Basics/TMWP_List_Comprehension_Examples.ipynb".
The first task is:
Misc. Complex Examples
Can you write Python code for it?

# working with nested lists:

tstLst = [['1','2.0','3.5'],['0.0','-5.789', '666'],['13.7']]

out1 = [[float(y) for y in x] for x in tstLst]   # convert to float, output in original nested structure
out2 = [float(y) for x in tstLst for y in x]     # convert to float, output as flat single list
out3 = [float(y) for x in tstLst for y in x if float(y) > 13]                     # combine with if test
out4 = [[float(y) for y in x if float(y) > 13 or float(y) < 1] for x in tstLst]   # combine w/ if test

print(out1)
print(out2)
print(out3)
print(out4)

# finding second biggest number in a list
arr = [14, 400, 35, 17, 34, 399, 0, 77]
print(max([a for a in arr if a != max(arr)]))

# example of cross-tab query like list comprehension (from two lists) found on Stack Overflow

list1 = [1, 2, 3]
list2 = [40, 50]
vals=[(x,y) for x in list1 for y in list2]
print(list1)
print(list2)
print(vals)

# Not best practice, but printing from within a list comprehension and using enumerate

listofints = [1,2,3,9,10,11,12,19,99,100,101,102, 999, 1000, 1001, 50102030]
# note:  printing from within a list comprehension is criticized online as being "not pythonic"
#        concepts illustrated:
#          * ind, x  in enumerate(list) ==> how to get index and value
#          * (x, print())[0] ==> x is output using [0] to grab just first element of output, print then happens from [1]
#          * tempLst is capturing element [0] only
#          * print is probably returning None but element [1] (the return value) is not captured in tempLst

tempLst = [((x, listofints[ind+1]), print(x, listofints[ind+1]))[0]
           for ind, x in enumerate(listofints) if ind < len(listofints)-1]

print("\ntempLst:\n%s" %tempLst)

# this example was used in a function to test that all values in an array
# are between an allowed range (sample arrays and just the relavent code excerpt provided here)

# because of any(), if any of the values in the list comprehension are True, the condition is True,
#                   if the list comprehension finds no...


I am working on the file "PY_Basics/TMWP_List_Comprehension_Examples.ipynb".
The first task is:
More Complex or Specific Examples
Demonstration Accessing Index and Value of List & if/else
Can you write Python code for it?

# this function solves converting a number to a string with commas for thousands separators
# using a single list comprehension to do it.  The following sample code helped in writing this:
#      * Example of enumerate to get value,index: [(value, ind) for ind, value in enumerate(Lst)]
#      * Example of if/else: [i for i in Lst[:-1] if (((i+1) - (i))/(i)) > 0.08 else ... (condition here)]

def thousands_with_commas(integer):
    strng = str(integer)
    return ''.join([',' + str(value) if (len(strng) - ind) % 3 == 0 and ind > 0
            else value for ind, value in enumerate(strng)])

## tests:
print(thousands_with_commas(1234))
print(thousands_with_commas(12))
print(thousands_with_commas(123456789))
print(thousands_with_commas(1234567890))


I am working on the file "PY_Basics/TMWP_List_Comprehension_Examples.ipynb".
The first task is:
Simple Email Address Validation Example
This code shows how a multi-line task (without the list comprehension) condenses down to a single line list comprehension that gets user input from within its nesting conditions.  Readability is definitely sacrificed for what the list comprehension may gain us in terms of speed and succinctness of code.
Can you write Python code for it?

import sys

lst = []
N = int(input().strip())
for eachOne in range(N):
    firstName, emailID = input().strip().split(' ')
    firstName, emailID = [str(firstName),str(emailID)]

    if emailID[-10:] == "@gmail.com":
        lst.append(firstName)

print("----------------------") # this line will be lost in our list comprehension example that follows
print("\n".join(sorted(lst)))

import sys
print("\n".join(sorted([l.split()[0] for l in [input() for _ in range(int(input()))] if '@gmail.com' in l])))
             # more accurate:
             #     end the code like this:                                           if l[-10:] == '@gmail.com'


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
... so inside crazyList ?  The answer is that the list is one level deep and the elements are yet another level in:
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
For a simple list ... we can fix that by simply using list() during our attempt to create the copy:
Can you write Python code for it?

bList = list(aList)
bList[0] = 999
aList[1] = 998
print(aList)
print(bList)

bList.append(19)
print(aList)
print(bList)


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
In the tests that follow ... anything that does not work is wrapped in exception handling (that displays the error) so this notebook can be run from start to finish ... Note that it is not good practice to use a catch all for all errors. In real coding errors should be handled individually by type.
How do we access the first index (element 2) of the first array object in our complex list (which resides at index 0)?
Can you write Python code for it?

try:                  # not this way ...
    crazyList[0][1]
except Exception as ex:
    print("%s%s %s" %(type(ex), ":", ex))

# let's look at what we built:  all the objects are here but are no longer named so we need to get indices right
crazyList

# note that both of these get the same data, but also note the difference in the format: "[[]]" and array([])".
# look at the source and you will see we are drilling in at different levels of "[]"
# there can be situations in real coding where extra layers are created by accident so this example is good to know

print(crazyList[0])
crazyList[0][0]


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Just analyzing how the numbers are arranged, we see that in shape(a,b,c,d), it just added the new extra dimensional layer to the front of the list so that now:
- a = larger hyper grouping (2 of them)
- b = first subgroup within (3 of them)
- c = rows within these groupings (4 of them)
- d = columns within these groupings (5 of them)
It appears that rows always come before columns, and then it looks like groupings of rows and columns and groupings or groupings, etc. . . are added to the front of the index chain.
Building something complex just to drill in more on how to access sub-elements:
Can you write Python code for it?

# some simple arrays:
simp1=np.array([[1,2,3,4,5]])
simp2=np.array([[10,9,8,7,6]])
simp3=[11,12,13]

# a dictionary
dfrm1 = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'],
        'year': [2000, 2001, 2002, 2001, 2002],
        'population': [1.5, 1.7, 3.6, 2.4, 2.9]}
# convert dictionary to DataFrame
dfrm1 = pd.DataFrame(dfrm1)
dfrm1

# pandas indexing works a little differently:
#   * column headers are keys
#   * as shown here, can ask for columns, rows, and a filter based on values in the columns
#     in any order and the indexing will still work

print(dfrm1["population"][dfrm1["population"] > 1.5][2:4])  # all of these return values from "population" column only
print("---")                                                # where "population" > 1.5
print(dfrm1["population"][2:4][dfrm1["population"] > 1.5])  # and row index is between 2 and 4
print("---")
print(dfrm1[dfrm1["population"] > 1.5]["population"][2:4])
print("---")
print(dfrm1[dfrm1["population"] > 1.5][2:4]["population"])
print("---")
print(dfrm1[2:4]["population"][dfrm1["population"] > 1.5])
print("---")
print(dfrm1[2:4][dfrm1["population"] > 1.5]["population"]) # this last one triggers a warning

# breaking the above apart:
print(dfrm1[dfrm1["population"] > 1.5])  # all rows and columns filtered by "population" values > 1.5
print("---")
print(dfrm1["population"])               # return whole "population" column
print("---")
print(dfrm1[2:4])                        # return whole rows 2 to 4

crazyList = [simp1, m3d, simp2, n3d, simp3, dfrm1, o3d]

# Accessing the dataframe inside the list now that it is a sub element:
crazyList[5]["population"][crazyList[5]["population"] > 1.5][2:4]


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Let's add the true simple array to our crazy object and then create working examples of accessing everything ...
Can you write Python code for it?

crazyList.append(trueSimp1)   # append mutates so this changes the original list
crazyList                     # Warning! if you re-run this cell, you will keep adding more copies of the last object
                              # to the end of this object.  To be consistent with content in this NB
                              # clear and re-run the whole notebook should that happen

# The elements at either end of crazyList:
print(crazyList[0])
print(crazyList[-1])  # ask for last item by counting backwards from the end

# get a specific value by index from within the subelements at either end:
print(crazyList[0][0][2]) # extra zero for the extra [] .. structurally this is really [0 [0 ], [1] ] but 1 does not exist
print(crazyList[-1][2])


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Looking at just that first element again:
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Mutation is avoided.  Now we can change our two objects independantly.  However, with complex objects like crazyList, this does not work.
The following will illustrate the problem and later, options to get around it are presented.
Can you write Python code for it?

crazyList2 = list(crazyList)

crazyList2


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Note how the second is really a reference to the first so changing one changes the other:
Can you write Python code for it?

aList[0] = 0
bList[1] = 1
bList.append(4)
print(aList)
print(bList)


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Note the [] versus the [[]] ... our "simple arrays" were copied from an example, but are actually nested objects of 1 list of 5 elements forming the first object inside the array.  A true simple array would like this:
Can you write Python code for it?

trueSimp1=np.array([10,9,8,7,6])
print(trueSimp1.shape)             # note:  output shows that Python thinks this is 5 rows, 1 column
trueSimp1


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Notice which numbers moved where.  This would seem to indicate that in shape(a,b,c):
- a is like the object's depth (how many groupings of rows/columns are there?)
- b is like the object's rows per grouping (how many rows in each subgroup)
- c is like the object's columns
What if the object had 4 dimensions?
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Now let's access other stuff in the list ...
Can you write Python code for it?

crazyList[1]  # this is the second object of the list (Python like many languages starts indicies at 0)
              # this is the full output of m3d

crazyList[0]  # after the above demo, no surprises here ... simp1 was the first object we added to the list


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Now we'll look at just the last object in both "crazyLists" showing what changed:
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
Should even deepcopy() not work, this topic online may prove helpful in these situations:  Stack Overflow: When Deep Copy is not Enough.

Finding The Index of a Value
Suppose we didn't know how to find the element but we knew the value we were looking for?  How to get its index?
Can you write Python code for it?

print(stupidList)
print(stupidList[1].index(5))  # this works on lists

# but for nested lists, you would need to loop through each sublist and handle the error that
# gets thrown each time it does not find the answer

for element in stupidList:
    try:
        test_i = element.index(5)
    except Exception as ex:
        print("%s%s %s" %(type(ex), ":", ex))

print(test_i)

# this strategy will not work on numpy arrays though
try:
    crazyList[0].index(2)
except Exception as anyE:
    print(type(anyE), anyE)

# because we have a list containing numpy arrays, we could look in each one like this:
print(crazyList[0])
np.where(crazyList[0]==2)

# the above indicates that 2 lives here:
crazyList[0][0][1]   # started with crazyList[0], then found it at [0][1] inside the data structure

# For floating point numbers, the level of precision matters
# details on how this works are presented in this notebook:  TMWP_np_where_and_floatingPoint_numbers.ipynb

# the simple test in the cells that follow should help illustrate the problem and what to do, but
# see aforementioned notebook for more detail

# to perform a where() test on a structure like this, it is important to note that print()
# rounds the result to 8 decimal places.  The real underlying numbers have more decimal places

print(crazyList2[1]); print("")
print(crazyList2[1][2][3][4])                       # get a number to test with
print("{0:.20}".format(crazyList2[1][2][3][4]))     # show more decimal places of the test number

# Warning! If you re-run this notebook, new random nubers are generated and the value used for the test in this
#          cell will probably then fail.  To fix this, re-run previous cell and copy in the final number shown
#          above up to at least 17 decimal places.

print(np.where(crazyList2[1]==0.95881217854380618)) # number copied from output of previous line up to 17 decimal places
                                                    # np.where() can find this, but will also return other values
...


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
So what about the array?  The array was originally built in "simp1" and then added to crazyList.  Its source looks like this:
Can you write Python code for it?

print(type(simp1))
print(simp1.shape)

print(simp1)
print(simp1[0])  # note that the first two give us the same thing (whole array)
simp1[0][1]


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
The "13" replaced the value at this location in both crazyList and crazyList2.  We are not dealing with true copies but rather references to the same data as further illustrated here:
Can you write Python code for it?

crazyList[7][1] = 9   # change on of them again and both change
print(crazyList[7])
print(crazyList2[7])


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
deepcopy() comes from the copy library and the commands are documented at Python.org.  For this situation, this solution seems to work for when mutation is undesirable:
Can you write Python code for it?

import copy
crazyList4 = copy.deepcopy(crazyList)

print("before:")
print(crazyList[4])
print(crazyList4[4])
crazyList4[4][0] = 15
print("")
print("after:")
print(crazyList[4])
print(crazyList4[4])


I am working on the file "PY_Basics/TMWP_PY_CrazyList_Indexing_and_Related_Experiments.ipynb".
The first task is:
remember that this object if it were not in a list would be accessed like so:
Can you write Python code for it?

simp1[0][1]     # second element inside it


I am working on the file "PY_Basics/TMWP_PY_Recursion_Factorial_Example.ipynb".
The first task is:
Performance testing
These tests show performance comparisons.  They also prove that we should use the math library instead of our own function.
Can you write Python code for it?

%%timeit   # %% magic only works in a iPython cells. Use import timeit and timeit() in .py code
factorial(800)

%%timeit
factorial_lambda(800)

%%timeit
factorial_loop(800)

import math

%%timeit
math.factorial(800)


I am working on the file "PY_Basics/TMWP_Pandas_Dataframe_Basics.ipynb".
The first task is:

Building a Simple DataFrame

Create DF from Dictionary
Can you write Python code for it?

# create a dictionary
stateData = {'state': ['Ohio', 'Ohio', 'Ohio', 'Nevada', 'Nevada'], \
        'year': [2000, 2001, 2002, 2001, 2002],
        'pop': [1.5, 1.7, 3.6, 2.4, 2.9]}
# convert to DataFrame
df_stdt = pd.DataFrame(stateData)
df_stdt

print("Index:  %s" %df_stdt.index)
print("Values:  \n%s" %df_stdt.values)
print("Columns: %s" %df_stdt.columns)
print("dataframe.describe():")
df_stdt.describe()


I am working on the file "PY_Basics/TMWP_Pandas_Dataframe_Basics.ipynb".
The first task is:

Renaming Dataframe Columns
Study these tests.  They show what works and what doesn't as counterintuitive as that is.  This code was written python 3.6.1
Can you write Python code for it?

testDF = pd.DataFrame({"color":["red", "green", "blue"],
              "thing":["house", "dog", "machine"]})

testDF

## get single column name
testDF.columns[0]

## rename one column
testDF.columns.values[0]='primaries'
testDF

testDF.primaries

testDF.color

testDF.color.name

testDF.columns = ['primary', 'something']
testDF

testDF.something

testDF.primary

## stackoverflow: https://stackoverflow.com/questions/11346283/renaming-columns-in-pandas/11346337
##   sample: df.rename(columns={'oldName1': 'newName1', 'oldName2': 'newName2'}, inplace=True)

testDF.rename(columns={'primary':'myColors'}, inplace=True)
testDF

testDF.rename(columns={'myColors':'TruColors', 'something':'anything'}, inplace=True)
testDF


I am working on the file "PY_Basics/TMWP_Pandas_Dataframe_Basics.ipynb".
The first task is:

Setting Data Types on Columns
Read comments as well as code in cells which follow:
Can you write Python code for it?

# Recommended:  To set dtype for all columns, build the Dataframe,
# and then use astype() to set the columns as shown here

# though online help topics indicate it should be possible to pass in a dictionary or list of tuples
# to set all the datatypes initially, under Python 2.7, this does not seem to work.
# came closest with this sample:
#   solutionPD2 = pd.DataFrame({ 'disk':[],'fromPeg':[], 'toPeg':[], 'text':[], 'notes':[]},
#                            dtype=[('disk', 'int64'), ('fromPeg', 'int64' ), ('toPeg', 'float64'),
#                                   ('text', 'str'), ('notes', 'object')] )
# Related links:
#    http://stackoverflow.com/questions/21197774/assign-pandas-dataframe-column-dtypes
#    http://pandas.pydata.org/pandas-docs/stable/generated/pandas.DataFrame.append.html

solutionPD2 = pd.DataFrame({ 'disk':[],'fromPeg':[], 'toPeg':[], 'notes':[], 'text':[]}, dtype=np.int64)
solutionPD2 = solutionPD2.astype({ 'disk':'int64','fromPeg':'float64', 'toPeg':'float64', 'notes':'object',
                                   'text':'str'})

solutionPD2 = solutionPD2.append(pd.DataFrame({ 'disk':[5],'fromPeg':[1], 'toPeg':[3],
                                                'notes':'hello', 'text':'more text' }), ignore_index=True)
solutionPD2 = solutionPD2.append(pd.DataFrame({ 'disk':[4],'fromPeg':[1], 'toPeg':[2],
                                                'notes':'good bye', 'text':'even more text'}), ignore_index=True)

# print("disk column type: %s" %type(solutionPD['disk'][0]))
print(solutionPD2.dtypes)
solutionPD2

# disk is currently an int
# we will convert this numeric column to text here
solutionPD2.disk = solutionPD2.disk.astype('str')
print(solutionPD2.dtypes)  # disk changes to 'object' which permits text


I am working on the file "PY_Basics/TMWP_Pandas_Dataframe_Basics.ipynb".
The first task is:
The Solution
A Stack Overflow post  indicates to use the import statement that is commented out in the cell that follows.
The one that is not commented out works with the Python 3.6 Anaconda implementation tested here.
By default: this Anaconda implementation installed ipython and ipython_genutils packages.  If these are missing, they would need to be installed using conda install <package>.  You can use the --dry-run switch to test that the install is compatible with your specific package environment first.
Can you write Python code for it?

## use this line if it works ... if not, try the one used at start of this code cell
## this cell tested with Anaconda / Python 3.6
# from Ipython.display import display

import ipython_genutils

def presentDF(df):
    display(df)

presentDF(df_stdt)
print("="*72)
display(solutionPD)
print("="*72)
display(solutionPD2)
print("="*72)
df_stdt


I am working on the file "PY_Basics/TMWP_PerfTesting_in_JupyterNBs.ipynb".
The first task is:
Another Simple Test
This test shows how two lines performs better than one since it avoids doing a list conversion twice
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PerfTesting_in_JupyterNBs.ipynb".
The first task is:
Performance Test One:
Can you write Python code for it?

timeit(get_subStrs_of_uniqueChars(longString, 6623))

# hackerrank, most voted solution from discussion board (for Python)
# S, N = input(), int(input())

def hackrRank_getSubsOfUniqueChars_v1(S, N):
                 # modified slightly to integrate for this timeit test:
    rtnVal = []  # changed print() to rtnVal
    for part in zip(*[iter(S)] * N):
        d = dict()
        rtnVal.append(''.join([ d.setdefault(c, c) for c in part if c not in d ]))
    return rtnVal

# tests to show it works the same as earlier examples
test_funcs_thisNB(hackrRank_getSubsOfUniqueChars_v1)
hackrRank_getSubsOfUniqueChars_v1(longString, 6623)  # should only ouput one string


I am working on the file "PY_Basics/TMWP_PerfTesting_in_JupyterNBs.ipynb".
The first task is:
Performance Test Two:
This one runs faster by a noticeable factor (6x faster).  But the next one is even more extreme.
Can you write Python code for it?

timeit(hackrRank_getSubsOfUniqueChars_v1(longString, 6623))

# a solution from the discussion board of hackerrank

def without_repetition(t):
    s = set()
    o = []
    for c in t:
        if c not in s:
            s.add(c)
            o.append(c)
    return ''.join(o)

def hackrRank_getSubsOfUniqueChars_v2(s, w):
    n = len(s)
    rtnVal = []
    for t in [s[i:i+w] for i in range(0, n, w)]:
        rtnVal.append(without_repetition(t))
    return rtnVal

# tests to show it works the same as earlier examples
test_funcs_thisNB(hackrRank_getSubsOfUniqueChars_v2)
hackrRank_getSubsOfUniqueChars_v2(longString, 6623)  # should only ouput one string


I am working on the file "PY_Basics/TMWP_PerfTesting_w_time_magics_JupyterNBs.ipynb".
The first task is:
For this example, timeit() needs to be the only function in the cell, and then your code is called in as a valid function call as in this demo:
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PerfTesting_w_time_magics_JupyterNBs.ipynb".
The first task is:
If you get the 'slowest run took ...' message, try re-running the code cell to over-write the caching
Can you write Python code for it?

%timeit 10*1000000
# this syntax allows comments ... note that if you leave off the numeric argument, %timeit seems to do nothing
myFun(12)

%%timeit
# this syntax allows comments ... if defaults the looping argument
myFun(12)

%time
# generates "wall time" instead of CPU time
myFun(12)

# getting more detail using %time on a script or code
%time {for i in range(10*1000000): x=1}

%timeit -n 1 10*1000000
# does it just once which may be inaccurate due to random events
myFun(12)


I am working on the file "PY_Basics/TMWP_PerfTesting_w_time_magics_JupyterNBs.ipynb".
The first task is:
These tests use the following inputs.  As per requirements in the challenge problem, what each line mean is also given here:
<pre>
10
999 10001 574 39 12345678900100111, 787878, 999999, 1000000000000000000008889934567, 8989, 1111111111111111111111110000009999999
5
999 10001 574 39 73277773377737373000000000000007777888
</pre>
Can you write Python code for it?


I am working on the file "PY_Basics/TMWP_PerfTesting_w_time_magics_JupyterNBs.ipynb".
The first task is:
Unlike timeit(), the other options provided here (using iPython cell magics) can test any snippet of code within a python cell.
Symmetric Difference Example
This code from hackerrank shows increasingly smaller snippets of code to find the symmentric difference between two sets.  Symmetric difference of sets A and B is the set of values from both sets that do not intersect (i.e., values in A not found in B plus the values in B not found in A).  This code was written to accept 4 lines of input as per a www.hackerrank.com specification.  The problem itself is also from www.hackerrank.com.
Performance tests are attempted but are hard to know what is really going on since variance in the time to input the values could also account for speed differences just as easily as the possibility of coding efficiencies.
Can you write Python code for it?

def find_symmetricDiff_inputSetsAB_v1():
    len_setA = int(input())
    set_A = set([int(i) for i in input().split()])
    len_setB = int(input())
    set_B = set([int(i) for i in input().split()])
    [print(val) for val in sorted(list(set_A.difference(set_B).union(set_B.difference(set_A))))]

def find_symmetricDiff_inputSetsAB_v2():
    setsLst = [0,0]
    for i in range(2):
        int(input()) # eat value ... don't need it
        setsLst[i] = set([int(i) for i in input().split()])
    [print(val) for val in sorted(list(setsLst[0].difference(setsLst[1]).union(setsLst[1].difference(setsLst[0]))))]

''' understanding next two versions:
    * key=int, applies int() to each value to be sorted so the values are sorted as 1,2,3 ... not: '1', '2', '3'
    * a^b is the same as a.symmetric_difference(b)

    these two come from discussion boards on hackerrank
'''
def find_symmetricDiff_inputSetsAB_v3():
    a,b = [set(input().split()) for _ in range(4)][1::2]
    return '\n'.join(sorted(a.symmetric_difference(b), key=int))

def find_symmetricDiff_inputSetsAB_v4():
    a,b = [set(input().split()) for _ in range(4)][1::2]
    return '\n'.join(sorted(a^b, key=int))


I am working on the file "PY_Basics/TMWP_Python_Getting_Names_of_Things.ipynb".
The first task is:

Using globals() and .items() to get Names
Can you write Python code for it?

### try these lines after some variables and functions are defined:
# globals()
# globals().items()
### result is long and rather messy ...

# this code is from the stack overflow post: http://stackoverflow.com/a/1538399/7525365

def variable_for_value(value):
    for n,v in globals().items():
        if v == value:
            return n
    return None

variable_for_value(peg1)

variable_for_value(move_from)

print(variable_for_value(peg1[0]))  # cannot pass in an index or pointer and get the name
print(variable_for_value(*peg2))    # None is returned

# this is probably because it is treating the above literally as if you were asking for
# thing named '*peg2' or thing named peg1[0]
# thing doesn't exist?  It does this:

x = variable_for_value('does_not_exist') # to even get nonexistent thing to pass in have to wrap in quotes and treat as string
                                         # otherwise, NameError is triggered before we can see how it handles it
print(x)

def move_from(source, target):
    print("Move %d from %s to %s next." %(source[-1], getname(*source), getname(*target)))

def test_another_way_fun(fun, x, y):
    fun(x, y)
    lst = [getname(fun), variable_for_value(x), variable_for_value(y)]
    return lst

test_another_way_fun(move_from, peg1, peg3)

# using just variable_for_value

def move_from(source, target):
    print("Move %d from %s to %s next." %(source[-1], getname(*source), getname(*target)))

def test_another_way_fun(fun, x, y):
    fun(x, y)
    lst = [variable_for_value(fun), variable_for_value(x), variable_for_value(y)]
    return lst

test_another_way_fun(move_from, peg1, peg3)

'''Odd symptom to be aware of ... in earlier tests, somehow when re-running one of the cells above,
   the last value returned as "_" instead of its true name.  Despite numerous edits to code in this cell
   to attempt to replicate the symtpom, it does not seem to recur.  It is not known if this is a random glitch
   or something that may occur again under the right coditions. '''

def...


I am working on the file "PY_Basics/TMWP_Python_Getting_Names_of_Things.ipynb".
The first task is:

Looping over variables in globals() to get Values
Can you write Python code for it?

# getting the value of variables by their name on globals()

x = peg1
y = peg2
z = peg3

my_list = ["x", "y", "z"] # x, y, z have been previously defined

for name in my_list:
    print("handling variable %s" %name)
    bla = globals()[name]  # accessing the value at the index for name
    print("bla: %s" %bla)


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:

Using map(), reduce() and lambda() To Solve It
This code addresses the underlying calculations.  It initially tests the code using a number sequence that is not the full 123... range progression of the original problem, but we can tells from the tests that these approaches work.  This code is also designed to quickly solve the problem in as few lines as possible (unlike the loop which was also designed to give options on how to show its inner workings).
Can you write Python code for it?

# source: eruciform on StackOverflow
# note:   powOfTens(x) is just int(math.log10(x))

import math   # to get math.log
listofints = [1,2,3,9,10,11,12,19,99,100,101,102, 999, 1000, 1001, 50102030]
n = reduce(lambda x,y:[x[0]*(10**(y[1]+1))+y[0],0],map(lambda x:[x,powOfTens(x)], listofints))[0]
            # to do this in one line with no external functions, replace powOfTens(x) w/:
            #       int(math.log10(x))
print(n)

# source: eruciform on StackOverflow
# we can also more simply crack this problem with just reduce()
n = reduce(lambda x,y:x*(10**(powOfTens(y)+1))+y,listofints)
            # to do this in one line with no external functions, replace powOfTens(x) w/:
            #       int(math.log10(y))
print(n)


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:

Simpler reduce() and lambda() Example
When the code is simplified to just use use reduce(), now only one lambda function is able to set up the power-of-tens calculation for each number and merge it all into terms needed for reduce().  Picking it apart from the inside, just the lambda logic is demonstrated here:
Can you write Python code for it?

# for comparison, here is the lambda logic split in two functions from the earlier example
testFun_r = lambda x,y:[x[0]*(10**(y[1]+1))+y[0],0]  # used in outer () to feed into reduce()
testFun_m = lambda x:[x,int(math.log10(x))]          # used in inner () to feed into map()

# for this idea, only one lambda does it all, feeding directly into reduce():
testFun2 = lambda x,y:x*(10**(int(math.log10(y))+1))+y

# test it:
testVal = [999, 1001]
testFun2(testVal[0], testVal[1])

# now reduce() applies it across the whole original list: function(n1, n2) = result1, function(result1, n3) = result2 ...
# until it produces a final answer to return ("reducing the list down to one number").

listofints = [1,2,3,10,12,19,99,100,101,50102030]
n = reduce(lambda x,y:x*(10**(int(math.log10(y))+1))+y,listofints)
print(n)

1*10**(int(math.log10(2))+1)+2


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:

Returning to the original Problem and Eliminating The Loops
After exploring different options, if we are to preserve all the functionality but lose the loops, a solution is chosen that combines reduce() with list comprehensions.  Some notes on this solution:
- this version supports the original 123...N where user passes in N
- it also supports the listofints used in the quick solutions above
- showWork now shows what reduce is fusing together to form the final output
- in accepting either N or listofints it tests the input argument for type.  This is not considered good practice, but is kept here to illustrate how to do it.  Coding sites generally recommend code be refactored to avoid using type testing within its operation.
Can you write Python code for it?

# this function presented and tested in earlier sections
# repeated here (unchanged) for conveninece:

import math                              # needed for log functions
def powOfTens(N):                        # find the powers of 10 inside a number
    if N < 1:
        N = 1                            # less than 1 would throw an error, this is a work-around based on
                                         # how this function is used in the code

    return int(math.log10(N))            # converts decimal to whole integer
                                         # int() rounds down no matter how big the decimal
                                         # note: math.log(x, 10) produces floating point rounding errors

from __future__ import print_function

def create_seqNumber(N, reverse=False, showWork=False, returnDescr=False, divLength=100):
    '''create_seqNumber() --> iput N, and get back a number built from the sequence of 1234...N
Arguments: reverse=True to get the sequence in revers, showWork=True to see numbers that add up to final,
returnDescr=True to print the answer in a sentence as well as returning it as a number.'''

    num = 0
    tensIncr = 0
    answer = 0

    if isinstance(N, list):
        if reverse == False:
            listofints = N
        else:
            listofints = N[::-1]
    elif isinstance(N, int):
        Ntens = powOfTens(N)
        if reverse == False:     # create range builder inputs
            rstart = 1
            rend = N+1
            rinc = 1
        else:
            rstart = N
            rend = 0
            rinc = -1
        listofints = range(rstart, rend, rinc)
    else:
        print(type(N))
        raise TypeError("Error: for create_seqNumber(N), N must be a list or an integer.")

    answer = reduce(lambda x,y:x*(10**(powOfTens(y)+1))+y,listofints)

    if showWork == True:
        print("Show Work:")
        print("#"*divLength)

        worklist = [reduce(lambda...


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:
As a point of curiosity though ... how would we do it using math instead of relying on print tricks which probably convert the numbers to strings under the covers anyway?  The basics of the solution is that the code needs to add zeroes to each number in the sequence in the right amount so that added together, they become the "string" of numbers:
<pre>
    3              1001
   20          10000000
+ 100       99900000000
======    ===============
  123       99910001001
</pre>

List Comprehensions (Incomplete Solution)
Initially, the problem is approached using a list comprehension to avoid looping.  This simple experiment is only a partial solution though:
Can you write Python code for it?

'''Initial experiment:  use powers of 10, but start from the end so that we get: 1 * 10**N + 2 * 10**N-1 + ...
   This idea fails however as soon as numbers get bigger than 9
   This example outputs the source list generated by range() ahead of the answer just to show it.  '''

def num_to_seqStr(N, showList = True):
    lst = range(1,N+1)
    answer = sum([i*10**(N-i) for i in lst])
    if showList == True:
        print(lst)
    return answer

for i in range(11):
    print("Answer: %s" %num_to_seqStr(i))


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:
Deconstructing the code from the inside out, this is what it is doing.

map() example with reduce() and lambda()
Starting with the first more complex example using map(),  lambda creates an anonymous function that "maps" to all the elements in the list, in this case listofints.  The anonymous lambda function applies int(math.log10(x)) to a single x, and then map() "maps" each x in listofints to the lambda function.
Can you write Python code for it?


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:
Each sublist now contains [original_number, number_of_tens_in_number].  reduce() "reduces" the list to a single number by applying the lambda function fed into it.  It takes each term in the list as function(n1 , n2), then the result goes back into the function as function(result, n3), then function(result2, n4) ... and so on until the entire list is consumed and one answer is spit back.
Can you write Python code for it?

reduce(lambda x,y:[x[0]*(10**(y[1]+1))+y[0],0],map(lambda x:[x,int(math.log10(x))], listofints))


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:
Extracting just the first term is why the whole thing ended with [0] in the original code:
<pre>     n = reduce(lambda x,y:[x[0]*(10**(y[1]+1))+y[0],0],map(lambda x:[x,int(math.log(x,10))], listofints))[0]</pre>

The original code:
Can you write Python code for it?


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:
Testing with showWork Off
These test cases show functionality that supports forward and reverse lists generated from a single number, or feeding in a custom list and spitting it back in forward or reverse.
Can you write Python code for it?

create_seqNumber(15, showWork=False, returnDescr=True)

create_seqNumber(15, reverse=True, showWork=False, returnDescr=True)

create_seqNumber(102, reverse=False, showWork=False, returnDescr=True)

for i in [1, 5, 9, 10, 11, 13, 98, 99, 100, 101, 102, 107, 1012]:   # returnDescr = False by default
    print("F: %s" %(create_seqNumber(i, reverse=False)))
    print("-----"*25)
    print("R: %s" %(create_seqNumber(i, reverse=True)))
    print("#####"*25)

tstLst = [1,2,3,9,10,11,12,59,99,100,101,50102030]
print("F: %s" %(create_seqNumber(tstLst, reverse=False)))
print("-----"*25)
print("R: %s" %(create_seqNumber(tstLst, reverse=True)))
print("#####"*25)


I am working on the file "PY_Basics/Walkthroughs/TMWP_Num_Seq_As_Num_Experiment.ipynb".
The first task is:
log() To The Rescue for Part Of The Problem
As suggested on Stack Overflow by userID: eruciform, taking the base-10 logarithm of a number should replace the loop logic used in the earlier "findTens()" function.  This helps us calculate the powers of 10 in a number to use to help calculate how many zeroes will be needed on each term.  This allows the terms to add together into a final that displays them all.
Can you write Python code for it?

import math                              # needed for log functions
def powOfTens(N):                        # find the powers of 10 inside a number
    return int(math.log10(N))            # converts decimal to whole integer
                                         # int() rounds down no matter how big the decimal
                                         # note: math.log(x, 10) produces floating point rounding errors

# output is of form: (oritinalNumber, powersOfTens)
countTensTest = [(1,    powOfTens(1)),
                 (7,    powOfTens(7)),
                 (112,  powOfTens(112)),
                 (13,   powOfTens(13)),
                 (1009, powOfTens(1009))]
countTensTest

listofints = [1,2,3,9,10,11,12,19,99,100,101,102, 999, 1000, 1001, 50102030]

for i in listofints:
    print(i, powOfTens(i), math.log10(i))  # show what we are really calculating:
                                           # (original, function result, un-modified log)


I am working on the file "PY_Basics/kwrds/TMWP_Python_Keyword_Reference.ipynb".
The first task is:
Tests in Python 2.7
Same tests performed in Python 2.7
Can you write Python code for it?

import keyword
keyword.kwlist

kw1 = keyword.kwlist
print("Number of Keywords: %d" %len(kw1))

print(keyword.iskeyword("yield"))  # testing with a word that is a keyword
keyword.iskeyword("queue")  # testing with a word that is not a keyword

# False / True are listed as keywords in Python 3.6 but not in 2.7
# at least accodring to keyword.kwlist
# checking them:

True  # highlights like keyword
False # highlights like keyword
None  # highlights like keyword

print(keyword.iskeyword("True"))  # does not know it
print(keyword.iskeyword("False")) # does not know it
print(keyword.iskeyword("None"))  # does not know it
                                  # all were added to keywords list for Python 3.x

help("keywords") # note: this list is also 31 words and does not list True/False/None

# final observation:  Keyword counts different b/w Python 2.7 and Python 3.6
#                     This cell looks for PY27 words that are depricated in PY36

py3_list = ['False', 'None', 'True', 'and', 'as', 'assert', 'break',
            'class', 'continue', 'def', 'del', 'elif', 'else', 'except',
            'finally', 'for', 'from', 'global', 'if', 'import', 'in',
            'is', 'lambda', 'nonlocal', 'not', 'or', 'pass',
            'raise', 'return', 'try', 'while', 'with', 'yield']   # list copied from earlier run in PY36

common_kw = []
temp = []

for word in kw1:
    temp = [(common_kw.append(word2), word2) for word2 in py3_list if word==word2]
    if len(temp) == 0:
        print("word not found in Python 3.6: %s" %word)

print("Common Keywords between Both Versions of Python:")
print(common_kw)


I am working on the file "PY_Basics/kwrds/TMWP_Python_Keyword_Reference.ipynb".
The first task is:
This solves the discrepencies.  Python 3.6 had a keyword count of 33 while Python 2.7 had a count of 31:
- Python 2.7 has 31 keywords
- 2 were converted to functions in Python 3.6 so that gives 3.6 a base of 29
- 29 + 4 new keywords (added to the list for PY36) and we get 33
However, it should be noted that True, False, None are reserved words in Python 2.7 that simply don't show in its keywords list.  This is more of a correction than an addition in Python 3.6
Getting Help on a Keyword
In other languages, documentation often includes a convenient page with hyperlinks for each keyword.  Python has this for builtin functions but I could not find it for keywords.  Using above code, we can generate the list, and then we can ask for help on any given keyword like this:
Can you write Python code for it?


I am working on the file "PY_Basics/kwrds/TMWP_Python_Keyword_Reference.ipynb".
The first task is:
exec and print were not depricated.  They were converted from key words to functions in Python 3.6.  This explains the syntax change in print statements that breaks old Python 2.7 code.
Can you write Python code for it?

# now let's look for the discrepencies the other way.  We already know about True/False/None
# but taken together with exec, print, the counts don't add up yet for the differences

common_kw = []
temp = []

for word in py3_list:
    temp = [(common_kw.append(word2), word2) for word2 in kw1 if word==word2]
    if len(temp) == 0:
        print("word not found in Python 2.7: %s" %word)

print("Common Keywords between Both Versions of Python:")
print(common_kw)


I am working on the file "P_Data_analysis.ipynb".
The first task is:
more stocks
There is all kinds of neat info on the web. Here is the SPY exchange-traded fund, which tracks the S&P 500 index.
Can you write Python code for it?

px = web.get_data_yahoo('SPY')['Adj Close']*10
px

plot(px)


I am working on the file "PageRank.ipynb".
The first task is:
In principle, we could use a linear algebra library, as below, to calculate the eigenvalues and vectors.
And this would work for a small system. But this gets unmanagable for large systems.
And since we only care about the principal eigenvector (the one with the largest eigenvalue, which will be 1 in this case), we can use the power iteration method which will scale better, and is faster for large systems.
Use the code below to peek at the PageRank for this micro-internet.
Can you write Python code for it?

eVals, eVecs = la.eig(L) # Gets the eigenvalues and vectors
order = np.absolute(eVals).argsort()[::-1] # Orders them by their eigenvalues
eVals = eVals[order]
eVecs = eVecs[:,order]

r = eVecs[:, 0] # Sets r to be the principal eigenvector
100 * np.real(r / np.sum(r)) # Make this eigenvector sum to one, then multiply by 100 Procrastinating Pats


I am working on the file "PageRank.ipynb".
The first task is:
Metoden som brukes for å finne en egenvektor er potensmetoden.
Eksempelmatrisen til PageRank-eksempelet, se 13.3 i kompendiet
Can you write Python code for it?

A1 = array([
    [0, 1, 0, 0,   0,   0 ],
    [1, 0, 0, 0,   0,   1 ],
    [0, 0, 0, 1/3, 1/2, 0 ],
    [0, 0, 0, 0,   0,   0 ],
    [0, 0, 0, 1/3, 0,   0 ],
    [0, 0, 1, 1/3, 1/2, 0 ] ])
brus = 1/6*array([
    [1,1,1,1,1,1],
    [1,1,1,1,1,1],
    [1,1,1,1,1,1],
    [1,1,1,1,1,1],
    [1,1,1,1,1,1],
    [1,1,1,1,1,1]])
A = .85*A1 + .15*brus

import networkx
G = networkx.from_numpy_matrix(A1.T, create_using=networkx.MultiDiGraph())
networkx.draw_networkx(G)

feil = 0.0001
x = rand(6) # tilfeldig valgt vektor x
x = x/norm(x) # nå har x lengde 1

for i in range(200):
    Ax = dot(A,x) # fordi vi trenger å huske to vektorer
    Ax = Ax/norm(Ax) # lengde 1 igjen
    if norm(Ax-x) < feil: # hvis vi er nær nok en egenvektor
        break
    x = Ax


I am working on the file "PageRank.ipynb".
The first task is:
Nå er x en egenvektor (omtrent).
Can you write Python code for it?

x = x/sum(x)
# nå representerer verdiene i x sannsynligheter.
# Det er disse tallene pagerank bruker for å rangere nettsidene,
# fra størst til minst verdi.

bar(arange(len(x)),x)


I am working on the file "PageRank.ipynb".
The first task is:
Or even better, we can keep running until we get to the required tolerance.
Can you write Python code for it?

r = 100 * np.ones(6) / 6 # Sets up this vector (6 entries of 1/6 × 100 each)
lastR = r
r = L @ r
i = 0
while la.norm(lastR - r) > 0.01 :
    lastR = r
    r = L @ r
    i += 1
print(str(i) + " iterations to convergence.")
r


I am working on the file "PageRank.ipynb".
The first task is:
PageRank as a linear algebra problem
Let's imagine a micro-internet, with just 6 websites (Avocado, Bullseye, CatBabel, Dromeda, eTings, and FaceSpace).
Each website links to some of the others, and this forms a network as shown,

The design principle of PageRank is that important websites will be linked to by important websites.
This somewhat recursive principle will form the basis of our thinking.
Imagine we have 100 Procrastinating Pats on our micro-internet, each viewing a single website at a time.
Each minute the Pats follow a link on their website to another site on the micro-internet.
After a while, the websites that are most linked to will have more Pats visiting them, and in the long run, each minute for every Pat that leaves a website, another will enter keeping the total numbers of Pats on each website constant.
The PageRank is simply the ranking of websites by how many Pats they have on them at the end of this process.
We represent the number of Pats on each website with the vector,
$$\mathbf{r} = \begin{bmatrix} r_A \ r_B \ r_C \ r_D \ r_E \ r_F \end{bmatrix}$$
And say that the number of Pats on each website in minute $i+1$ is related to those at minute $i$ by the matrix transformation
$$ \mathbf{r}^{(i+1)} = L \,\mathbf{r}^{(i)}$$
with the matrix $L$ taking the form,
$$ L = \begin{bmatrix}
L_{A→A} & L_{B→A} & L_{C→A} & L_{D→A} & L_{E→A} & L_{F→A} \
L_{A→B} & L_{B→B} & L_{C→B} & L_{D→B} & L_{E→B} & L_{F→B} \
L_{A→C} & L_{B→C} & L_{C→C} & L_{D→C} & L_{E→C} & L_{F→C} \
L_{A→D} & L_{B→D} & L_{C→D} & L_{D→D} & L_{E→D} & L_{F→D} \
L_{A→E} & L_{B→E} & L_{C→E} & L_{D→E} & L_{E→E} & L_{F→E} \
L_{A→F} & L_{B→F} & L_{C→F} & L_{D→F} & L_{E→F} & L_{F→F} \
\end{bmatrix}
$$
where the columns represent the probability of leaving a website for any other website, and sum to one.
The rows determine how likely you are to enter a website from any other, though these need not add to one.
The long time behaviour of this system is when $ \mathbf{r}^{(i+1)} = \mathbf{r}^{(i)}$, so we'll drop the superscripts here, and...
Can you write Python code for it?


I am working on the file "PageRank.ipynb".
The first task is:
Test your code before submission
To test the code you've written above, run the cell (select the cell above, then press the play button [ ▶| ] or press shift-enter).
You can then use the code below to test out your function.
You don't need to submit this cell; you can edit and run it as much as you like.
Can you write Python code for it?

# Use the following function to generate internets of different sizes.
generate_internet(5)

# Test your PageRank method against the built in "eig" method.
# You should see yours is a lot faster for large internets
L = generate_internet(10)

pageRank(L, 1)

# Do note, this is calculating the eigenvalues of the link matrix, L,
# without any damping. It may give different results that your pageRank function.
# If you wish, you could modify this cell to include damping.
# (There is no credit for this though)
eVals, eVecs = la.eig(L) # Gets the eigenvalues and vectors
order = np.absolute(eVals).argsort()[::-1] # Orders them by their eigenvalues
eVals = eVals[order]
eVecs = eVecs[:,order]

r = eVecs[:, 0]
100 * np.real(r / np.sum(r))

# You may wish to view the PageRank graphically.
# This code will draw a bar chart, for each (numbered) website on the generated internet,
# The height of each bar will be the score in the PageRank.
# Run this code to see the PageRank for each internet you generate.
# Hopefully you should see what you might expect
# - there are a few clusters of important websites, but most on the internet are rubbish!
%pylab notebook
r = pageRank(generate_internet(100), 0.9)
plt.bar(arange(r.shape[0]), r);


I am working on the file "PageRank.ipynb".
The first task is:
This is certainly better, the PageRank gives sensible numbers for the Procrastinating Pats that end up on each webpage.
This method still predicts Geoff has a high ranking webpage however.
This could be seen as a consequence of using a small network. We could also get around the problem by not counting self-links when producing the L matrix (an if a website has no outgoing links, make it link to all websites equally).
We won't look further down this route, as this is in the realm of improvements to PageRank, rather than eigenproblems.
You are now in a good position, having gained an understanding of PageRank, to produce your own code to calculate the PageRank of a website with thousands of entries.
Good Luck!
Part 2 - Assessment
In this assessment, you will be asked to produce a function that can calculate the PageRank for an arbitrarily large probability matrix.
This, the final assignment of the course, will give less guidance than previous assessments.
You will be expected to utilise code from earlier in the worksheet and re-purpose it to your needs.
How to submit
Edit the code in the cell below to complete the assignment.
Once you are finished and happy with it, press the Submit Assignment button at the top of this notebook.
Please don't change any of the function names, as these will be checked by the grading script.
If you have further questions about submissions or programming assignments, here is a list of Q&A. You can also raise an issue on the discussion forum. Good luck!
Can you write Python code for it?

# PACKAGE
# Here are the imports again, just in case you need them.
# There is no need to edit or submit this cell.
import numpy as np
import numpy.linalg as la
from readonly.PageRankFunctions import *
np.set_printoptions(suppress=True)

# GRADED FUNCTION
# Complete this function to provide the PageRank for an arbitrarily sized internet.
# I.e. the principal eigenvector of the damped system, using the power iteration method.
# (Normalisation doesn't matter here)
# The functions inputs are the linkMatrix, and d the damping parameter - as defined in this worksheet.
def pageRank(linkMatrix, d) :
    n = linkMatrix.shape[0]
    M = d * linkMatrix + (1-d)/n * np.ones([n, n])
    r = 100 * np.ones(n) / n
    lastR = r
    r = M @ r
    i = 0
    while la.norm(lastR - r) > 0.01 :
        lastR = r
        r = M @ r
        i += 1
    return r


I am working on the file "PageRank.ipynb".
The first task is:
We can automate applying this matrix multiple times as follows,
Can you write Python code for it?

r = 100 * np.ones(6) / 6 # Sets up this vector (6 entries of 1/6 × 100 each)
for i in np.arange(100) : # Repeat 100 times
    r = L @ r
r


I am working on the file "PageRank.ipynb".
The first task is:
We can see from this list, the number of Procrastinating Pats that we expect to find on each website after long times.
Putting them in order of popularity (based on this metric), the PageRank of this micro-internet is:
CatBabel, Dromeda, Avocado, FaceSpace, Bullseye, eTings
Referring back to the micro-internet diagram, is this what you would have expected?
Convince yourself that based on which pages seem important given which others link to them, that this is a sensible ranking.
Let's now try to get the same result using the Power-Iteration method that was covered in the video.
This method will be much better at dealing with large systems.
First let's set up our initial vector, $\mathbf{r}^{(0)}$, so that we have our 100 Procrastinating Pats equally distributed on each of our 6 websites.
Can you write Python code for it?

r = 100 * np.ones(6) / 6 # Sets up this vector (6 entries of 1/6 × 100 each)
r # Shows it's value


I am working on the file "PageviewAnalysis.ipynb".
The first task is:
Combine mobile app and mobile web views and create a new dataframe df_pageviews_mobile. Since both dataframes are of the same dimension and sorted the on timestamp, we can just add the values of views to get the total mobile views.
Can you write Python code for it?

df_pageviews_mobile = copy.copy(df_pageviews_mobile_web)
df_pageviews_mobile.views = df_pageviews_mobile.views + df_pageviews_mobile_app.views
df_pageviews_mobile.head()


I am working on the file "PageviewAnalysis.ipynb".
The first task is:
Create a column 'date' that combines year, month, and day. Use it as the x axis later when plotting
Can you write Python code for it?

df['day'] = '01'
df['date']= df.apply(lambda x:datetime.strptime("{0} {1} {2}".format(x['year'],x['month'], x['day']), "%Y %m %d"),axis=1)


I am working on the file "PageviewAnalysis.ipynb".
The first task is:
Get pagecounts of en.wikipedia.org through mobile.
Can you write Python code for it?

params = {'project' : 'en.wikipedia.org',
          'access' : 'mobile-site',
          'agent' : 'user',
          'granularity' : 'monthly',
          'start' : '2008010100',
          'end' : '2016080100'}

api_call = requests.get(endpoint.format(**params))
response = api_call.json()
with open('pagecounts_mobile-site_200801-201607.json', 'w') as outfile:
    json.dump(response, outfile)


I am working on the file "PageviewAnalysis.ipynb".
The first task is:
Pagecounts
Get pagecounts of en.wikipedia.org through desktop.
Can you write Python code for it?

endpoint = 'https://wikimedia.org/api/rest_v1/metrics/legacy/pagecounts/aggregate/{project}/{access}/{granularity}/{start}/{end}'
params = {'project' : 'en.wikipedia.org',
          'access' : 'desktop-site',
          'agent' : 'user',
          'granularity' : 'monthly',
          'start' : '2008010100',
          'end' : '2016080100'}

api_call = requests.get(endpoint.format(**params))
response = api_call.json()
with open('pagecounts_desktop-site_200801-201607.json', 'w') as outfile:
    json.dump(response, outfile)


I am working on the file "PageviewAnalysis.ipynb".
The first task is:
Pageview
Get page viewcounts of en.wikipedia.org through desktop.
Can you write Python code for it?

endpoint = 'https://wikimedia.org/api/rest_v1/metrics/pageviews/aggregate/{project}/{access}/{agent}/{granularity}/{start}/{end}'
params = {'project' : 'en.wikipedia.org',
          'access' : 'desktop',
          'agent' : 'user',
          'granularity' : 'monthly',
          'start' : '2015070100',
          'end' : '2017100100'}

api_call = requests.get(endpoint.format(**params))
response = api_call.json()

with open('pageviews_desktop-site_201507-201709.json', 'w') as outfile:
    json.dump(response, outfile)