I am working on the file "PRNN/Assignment/Assignment3/word2Vec.ipynb".
The first task is:
For w2v
Can you write Python code for it?

def totalSentences(sentencesClass):
    size = 0
    for i in range(0, len(sentencesClass)):
        size += len(sentencesClass[i])
    return size;

def defineW2V(sentencesClass, w2v, dim = 300):
    n = totalSentences(sentencesClass)
    mat = np.zeros((n, dim))
    labels = np.zeros(n)
    k = 0
    for i in range(0, len(sentencesClass)):
        for j in range(0, len(sentencesClass[i])):
            words = sentencesClass[i][j].split()
            d = np.zeros(300)
            ind = 0.0
            for w in words:
                if w not in w2v:
                    w2v[w] = np.random.uniform(-0.25,0.25,dim)
                d += w2v[w]
                ind += 1.0
            d /= ind
            mat[k] = d
            labels[k] = i+1
            k+=1
    return mat, labels

def defineW2VPOS(originalSentencesClass, sentencesClass, w2v, dim = 300):
    n = totalSentences(sentencesClass)
    labels = np.zeros(n)
    sizePOS, POSList = getUniquePOS()
    mat = np.zeros((n, dim + sizePOS))
    matFromW2V, labels = defineW2V(sentencesClass, w2v)

    for i in range(0, n):
        mat[i,:dim] = matFromW2V[i]

    k = 0
    for i in range(0, len(originalSentencesClass)):
        for j in range(0,  len(originalSentencesClass[i])):
            pos = POSForSentence(originalSentencesClass[i][j])
            for p in pos:
                mat[k, dim + POSList.index(p)] = 1.0
            k+=1

    return mat, labels


I am working on the file "PRNN/Assignment/Assignment3/word2Vec.ipynb".
The first task is:
Loading from file
Can you write Python code for it?

def loadW2V():
    w2v = np.load('word2VecDict.npy').item()

    i = 0
    for key, value in w2v.iteritems():
        if i>10:
            break
        print key, value
        i = i + 1

def main():
    dirName = "Email-classification_dataset/"
    classes = [1,2,3,4,5]
    fileName = ["RD-positive-800.txt", "meetings-positive-800.txt", "negative-800.txt", "fyi-positive-800.txt", "tp-positive-500.txt",]
    originalsentencesClass = sentenceExtractionForTraining(dirName, fileName, classes)
    sentencesClass = removeFC(originalsentencesClass)
    w2v = initializeWordVecs(sentencesClass)

    savew2vToFile(w2v)

    XW2V, yW2V = defineW2V(sentencesClass, w2v)

    XW2VPOS, yW2VPOS = defineW2VPOS(originalsentencesClass, sentencesClass, w2v)

    finalMatW2V = finalFeaturesLabel(XW2V, yW2V)
    finalMatW2VPOS = finalFeaturesLabel(XW2VPOS, yW2VPOS)

    saveW2V("w2v.npy",finalMatW2V)
    saveW2V("w2vpos.npy",finalMatW2VPOS)

if __name__=="__main__":
    main()


I am working on the file "PRNN/Assignment/Assignment3/word2Vec.ipynb".
The first task is:
Removing fancy characters
Can you write Python code for it?

def removeFancyChars(sentences):
    lengthPhrase = len(sentences)
    for i in range(lengthPhrase):
        sentences[i] = re.sub(r'([^\s\w]|_)+', '', sentences[i])
    return sentences

def removeFC(sentencesClass):
    for i in range(0, len(sentencesClass)):
        sentencesClass[i] = removeFancyChars(sentencesClass[i])
    return sentencesClass


I am working on the file "PRNN/Assignment/Assignment3/word2Vec.ipynb".
The first task is:
To run this file download To run download https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit?usp=sharing File name : "GoogleNews-vectors-negative300.bin"
Can also be found here: https://code.google.com/archive/p/word2vec/
Reading Files Line by Line
Can you write Python code for it?

def sentenceExtractionForTraining(dirName, fileName, classes):
    sentencesClass = []
    for i in range(0,len(classes)):
        sentences = readFile(dirName+fileName[i])
        sentencesClass.append(sentences)
    return sentencesClass

def readFile(fileName):
    f = open(fileName,"r+")
    sentences = []
    for line in f.readlines():
        line = line.strip()
        if len(line)==0 :
            continue
        if "TIMEX3" in line:
            continue
        sentences.append(line.lower().rstrip())
    f.close()
    return sentences

def createVocab(sentencesClass):
    vocab = set()
    for i in range(0, len(sentencesClass)):
        for j in range(0,len(sentencesClass[i])):
            words = sentencesClass[i][j].split()
            for w in words:
                vocab.add(w)
    return vocab


I am working on the file "PRNN/Assignment/Assignment3/word2Vec.ipynb".
The first task is:
word2vec.
Can you write Python code for it?

def load_bin_vec(fname, vocab):
    """
    Loads 300x1 word vecs from Google (Mikolov) word2vec
    """
    word_vecs = {}
    with open(fname, "rb") as f:
        header = f.readline()
        vocab_size, layer1_size = map(int, header.split())
        binary_len = np.dtype('float32').itemsize * layer1_size
        for line in xrange(vocab_size):
            word = []
            while True:
                ch = f.read(1)
                if ch == ' ':
                    word = ''.join(word)
                    break
                if ch != '\n':
                    word.append(ch)
            if word in vocab:
               word_vecs[word] = np.fromstring(f.read(binary_len), dtype='float32')
            else:
                f.read(binary_len)
    return word_vecs

def add_unknown_words(word_vecs, vocab, min_df=1, k=300):
    for word in vocab:
        if word not in word_vecs:
            word_vecs[word] = np.random.uniform(-0.25,0.25,k)

def initializeWordVecs(sentencesClass):
    vocab = createVocab(sentencesClass)
    w2vFile = "GoogleNews-vectors-negative300.bin"

    w2v = load_bin_vec(w2vFile, vocab)
    add_unknown_words(w2v, vocab)

    return w2v


I am working on the file "PROJECT/0009-PorschePrices-DADV-Python.ipynb".
The first task is:
Andrews Curves
Can you write Python code for it?

from pandas.tools.plotting import andrews_curves
plt.figure()
andrews_curves(porsche, 'Age', colormap = 'autumn')
plt.show()


I am working on the file "PROJECT/0009-PorschePrices-DADV-Python.ipynb".
The first task is:
Explore the Dataset
Can you write Python code for it?

porsche.shape

porsche.head(5)

porsche = porsche.rename(columns = {'Unnamed: 0':'Number'})

porsche.head(5)

porsche.describe()


I am working on the file "PROJECT/0009-PorschePrices-DADV-Python.ipynb".
The first task is:
Histogram
Can you write Python code for it?

plt.figure();
porsche['Mileage'].diff().hist(bins = 7)
plt.show()


I am working on the file "PROJECT/0009-PorschePrices-DADV-Python.ipynb".
The first task is:
Plotting Using Seaborn and PyPlot
Can you write Python code for it?

import seaborn as sns
import matplotlib.pyplot as plt


I am working on the file "PROJECT/0009-PorschePrices-DADV-Python.ipynb".
The first task is:
Radial Visualization
Can you write Python code for it?

from pandas.tools.plotting import radviz

plt.figure()
radviz(porsche, 'Age')
plt.show()


I am working on the file "PROJECT/0009-PorschePrices-DADV-Python.ipynb".
The first task is:
Vertical Barchart
Can you write Python code for it?

plt.figure();
porsche.plot(kind = 'bar', stacked = True);
plt.show()


I am working on the file "PROJECT/0010-Exoplanets-DV-Python.ipynb".
The first task is:
Plotting with PyPlot and Seaborn
Hexbin Plot
Can you write Python code for it?

import matplotlib.pyplot as plt
import numpy as np
from scipy.stats import kendalltau
import seaborn as sns
sns.set(style="ticks")

sns.jointplot(planets.period, planets.eccen, kind="hex", stat_func=kendalltau, color="blue")
plt.show()


I am working on the file "PROSAIL_experiments.ipynb".
The first task is:
Exploring the MTCI (MERIS Terrestrial Chlorophyll Index)
The MTCI is a vegetation index that was developed for data from the MERIS sensor onboard the ENVISAT platform, and that relates surface reflectance to canopy chlorophyll content. The index is defined as the ratio of reflectance differences between MERIS bands 10 and 9 to reflectance differences between bands 9 and 8.
$$
MTCI = \frac{R_{753}-R_{705}}{R_{705}-R_{681}},
$$
where $R_{x}$ indicated reflectance for a waveband whose centre wavelength is located at $x$. In this experiment, we will use PROSAIL to look at the robustness of such an index. We note that the bandwidth of these three bands is 7.5, 10 and 7.5 $nm$.

A first experiment will be to determine the strength of the relationship just sweeping over the chlorophyll concentration while keeping all other parameters fixed. Try different regions of parameters space (e.g. set x0 to some other value).
As a second experiment, add some nuisance parameters: e.g. add a random variation variation to other parameters other than $C_{ab}$. The variation in this parameters is controlled by the minvals and maxvals dictionaries. Comment on the results, and in particular, in the robustness of the regression.
A third experiment will also add some noise using the noise_level parameters. We can assume that noise_level is the standard deviation of some additive Gaussian noise. Values of 0.01 are probably optimistic for the type of atmospheric correction we typically get.
Finally, observe the changes in the $MTCI-C_{ab}$ relationship for different acquisition geometries.
Can you write Python code for it?

plt.figure(figsize=(12,12))

x, y, p0, p1 = mtci_experiment(nuisance=["lai","cw"])

x, y, p0, p1 = mtci_experiment

x, y, p0, p1 = mtci_experiment

from IPython.core.display import HTML

def css_styling():
    styles = open("custom.css", "r").read()
    return HTML(styles)
css_styling()


I am working on the file "PS-01/.ipynb_checkpoints/problem_set_1-checkpoint.ipynb".
The first task is:
1. Load image from a file and display the image. Determine the size of the image. Finally, save a new copy of the image in a new file.

Para resolver este punto, primero guardamos en la variable img1 la imagen que vamos a utilizar, esto lo hacemos llamando la función 'rg' que lee una imagen en escala de grices. Esta función recibe el nombre de la foto como parámetro, en este caso el primer elemento de la lista files, la cual contiene todos los nombres de las imágenes utilizadas en el taller. Luego, con img1.shape, creamos una tupla que contiene a hg y wd, que son height y width respectivamente; mostramos entonces en consola el valor de cada una de estas variables para saber su tamaño. Seguidamente con mplt.figure creamos una nueva figura, la cual mostrará img1 que es la imagen original en escala de grises. Luego, en img2 hacemos una copia de img1 para poder guardarla en un nuevo archivo con cv2.imwrite. Finalmente, agregarmos esta nueva imagen con el nombre 'new_cameraman.png' en la lista files.
Can you write Python code for it?

img1 = rg(files[0])

hg, wd = img1.shape # function that returns a tuple with 2 parameters
print 'height: ' + str(hg)
print 'width: '+ str(wd)

mplt.figure()
mplt.imshow(img1, cmap='gray')
mplt.title(files[0])

img2 = img1.copy()
cv2.imwrite(pth + 'new_cameraman.png', img2) # create a new file called 'cameraman_new.png'
if files[-1] != 'new_cameraman.png':
    files.append('new_cameraman.png') # add the new element to the list
print files # checking


I am working on the file "PS-01/.ipynb_checkpoints/problem_set_1-checkpoint.ipynb".
The first task is:
2. Write a function flip_image, which flips an image either vertically or horizontally. The function should take two input parameters: the matrix storing the image data and a flag to indicate whether the image should be flipped vertically or horizontally. Use this function to flip an example image both vertically and horizontally. Tip: You can use numpy array indexing or OpenCV's flip function to solve this problem.

En este punto, primero guardamos en la variable img la imagen que utilizaremos. Luego de guardarla, procedemos a crear nuestra función flip_image que recibe los parámetros 'im_data' que es la matriz de la imagen (img) y el parámetro 'flag' que indica el eje en que queremos rotar la imagen. 1 significa rotar en el eje y (verticalmente), mientras 0 rota en el eje x (horizontalmente). Dentro de la función retornamos im_flip que es una variable que almacena el resultado de usar la función cv2.flip. Ahora, en la variable vert guardamos el resultado de llamar la función que creamos antes con los parámetros img y 1; lo mismo pasa con la variable hor, pero con flag = 0. Para mostrar las tres imágenes, creamos una lista con las tres variables que necesitamos, img, vert y hor, para luego mostrarlas con mplt.imshow.
Can you write Python code for it?

img = rg(files[4])

def flip_image(im_data, flag):
    im_flip = cv2.flip(im_data, flag) # flag=1 flip arround y-axis; flag=0 flip arround x-axis
    return im_flip

vert = flip_image(img, 1)
hor = flip_image(img, 0)
ls = [img, vert, hor]

for i in ls:
    mplt.figure()
    mplt.imshow(i, cmap='gray')
    mplt.title(files[4])


I am working on the file "PS-01/.ipynb_checkpoints/problem_set_1-checkpoint.ipynb".
The first task is:
3.  Write a function to generate the negative of an image. This means that  a new image is created in which the pixel values are all equal to 1.0 minus the pixel value in the original image.

Para este punto, primero guardamos la imagen que deseamos utilizar en la variable img. Seguidamente creamos la función neg_im() que no posee parámetros y se encarga de restar 255 a la imagen original. En la variable image, llamamos la función y almacenamos el resultado para luego mostrarlo en una figura con la imagen original para ver el contraste entre las dos.
Can you write Python code for it?

img = rg(files[1])

def neg_im():
    im = 255 - img
    return im

image = neg_im()

mplt.figure()
mplt.imshow(img, cmap='gray')
mplt.title(files[1])

mplt.figure()
mplt.imshow(image, cmap='gray')
mplt.title(files[1]+'_negative')


I am working on the file "PS-01/.ipynb_checkpoints/problem_set_1-checkpoint.ipynb".
The first task is:
4. Write a function average_intensity, which calculates the average intensity level of an image. Use this function on example images and discuss your results. You can use images from section 2 and 3

La función average_intensity recibe como parámetros la matriz en escala de grises de la imagen y retorna el valor medio de esta img.mean(). Para probar la función anterior se iteró sobre la lista files del siguiente modo: se creó una lista de intensidades [avg ... for i in ...], se crea una tupla para cada intensidad con su nombre de archivo respectivo; cada tupla es desempacada en un ciclo for donde se muestra el mensaje al usuario.
Can you write Python code for it?

def average_intensity(img):
    return img.mean()

for avg, name in zip([average_intensity(rg(i)) for i in files], files):
    print 'Average intensity of "', name, '":', avg


I am working on the file "PS-01/.ipynb_checkpoints/problem_set_1-checkpoint.ipynb".
The first task is:
5. Write a function threshold_image which thresholds an image based on a threshold level given as a parameter to the function. The function should take two parameters: the image to be thresholded and the threshold level. The result of the function should be a new thresholded image.

La función threshold_image toma como parámetros la matriz de la imagen y un entero que indica el punto para hacer la umbralización; el interior de esta función es simple: se llama la función threshold de OpenCV, con la opción de que la imagen resultante sea binaria invertida; esta función retorna una tupla, de la cual solo nos interesa el segundo valor (el primero sería el valor de umbral óptimo, si se llamase la función threshold con th=0). La imagen obtenida luego se muestra con Matplotlib.
Can you write Python code for it?

def threshold_image(img, th):
    # ignore retVal parameter, no Otsu binarization will be applied
    _, t = cv2.threshold(img, th, 255, cv2.THRESH_BINARY_INV)
    return t

img = threshold_image(rg(files[3]), 150)
mplt.figure()
mplt.imshow(img, cmap='gray')
mplt.title('Threshold of ' + files[3])


I am working on the file "PS-01/.ipynb_checkpoints/problem_set_1-checkpoint.ipynb".
The first task is:
6. Write a function avg_intensity_threshold_image which takes an image as its only parameter and thresholds this image based on the images average intensity value. Hint: Simply write a new function that uses the two functions just written.

La función avg_intensity_threshold_image es muy simple, es solo llamar la función del ejercicio anterior pero siendo el segundo parámetro el promedio de intensidad de la imagen (función realizada en el punto 4). Luego se aplica la operación a varias imágenes y se muestran en pantalla.
Can you write Python code for it?

def avg_intensity_threshold_image(img):
    return threshold_image(img, average_intensity(img))

ths = [avg_intensity_threshold_image(rg(i)) for i in files]
for t, name in zip(ths, files):
    mplt.figure()
    mplt.imshow(t, cmap='gray')
    mplt.title('Threshold of ' + name)


I am working on the file "PS-05/.ipynb_checkpoints/problem_set_5-checkpoint.ipynb".
The first task is:
Problem 1
Write a function that describes each object in a binary image using the Hu statistical moments. The Hu moments are invariant to rotation, scale and translation. These moments can be defined for each region in a binary image. The OpenCV function to compute these moments is <tt>cv2.HuMoments</tt>. Write down the equations that compute the seven Hu moments for a region.
Análisis
La siguiente función tiene como objetivo encontrar los Hu moments que describen a cada objeto de la imagen <tt>shapes.png</tt>. Para esto primero guardamos la matriz de la imagen en una variable. Luego definimos nuestra función que recibe una imagen como parámetro. Aplicamos threshold a la imagen para luego con la resultante hallar los objetos con la función <tt>cv2.findContours</tt>. Ya que tenemos los valores de los contours, podemos mostrar cuantos de estos encontró con su longitud y confirmamos que sean la misma cantidad de objetos que se encuentran en la imagen. Ahora, para cada uno de estos objetos aplicamos <tt>cv2.huMoments</tt> y mostramos en pantalla los resultados.
Seven Hu moments
$ I_1 = n_{20} + n_{02} $

$ I_2 = (n_{20} - n_{02})^2 + 4n_{11}^2 $

$ I_3 = (n_{30} - n_{12})^2 + (n_{21} - n_{03})^2  $

$ I_4 = (n_{30} + n_{12})^2 + (n_{21} + n_{03})^2  $

$ I_5 = (n_{30} - 3n_{12})(n_{30} + n_{12})[(n_{30} + n_{12})^2 - 3(n_{21} + n_{03})^2] + (3n_{21} - n_{03})(n_{21} + n_{03})(3(n_{30} + n_{12})^2 - (n_{21} + n_{03})^2)  $

$ I_6 = (n_{20} - n_{02})[(n_{30} + n_{12})^2 - (n_{21} + n_{03})^2] + 4n_{11}(n_{30} + n_{12})(n_{21} + n_{03}) $

$ I_7 = (3n_{21} - n_{03})(n_{30} + n_{12})[(n_{30} + n_{12})^2 - 3(n_{21} + n_{03})^2] - (n_{30} - 3n_{12})(n_{21} + n_{03})(3(n_{30} + n_{12})^2 - (n_{21} + n_{03})^2)  $

Can you write Python code for it?

image = rg(files[-11])

def huMoments(image):
    ret, threshold = cv2.threshold(image, 0, 255, cv2.THRESH_BINARY+cv2.THRESH_OTSU)

    _, contours, _ = cv2.findContours(threshold, cv2.RETR_LIST,cv2.CHAIN_APPROX_NONE)
    print "Number of contours detected = " + str(len(contours))

    for i in range(len(contours)):
        Hu = cv2.HuMoments(cv2.moments(contours[i])).flatten()
        print str(i) + ' = ' + str(Hu)

    mplt.figure()
    mplt.imshow(image, cmap='gray')
    mplt.title(files[-11])

huMoments(image)


I am working on the file "PS-05/.ipynb_checkpoints/problem_set_5-checkpoint.ipynb".
The first task is:
Problem 2
Write a function that detects corners on an image using the Harris corner detection method. You can use the OpenCV built-in functions. Your function should output the $N$ detected corner locations in a $2 \times N$ matrix. Visualize your results by plotting the corners on top of the input image.  Apply your function to the binary image <tt> shapes.png</tt> and to the grayscale image <tt>face.tif</tt>.
Análisis
Primeramente cargamos las dos imágenes de interés. Luego creamos la función cornerDetection que recive como argumento una imagen. Guardamos los corners que hayamos por la función <tt>cv2.goodFeaturesToTrack</tt> en una variable y luego iteramos sobre estos para hayar las posiciones x,y donde los mostraremos en forma de círculo en la imagen a través de <tt>.ravel</tt>. Finalmente mostramos la imagen resultante.
Can you write Python code for it?

img = rg(files[-11])
img_2 = rg(files[18])

def cornerDetection(img):
    corners = cv2.goodFeaturesToTrack(img,100,0.01,10)
    corners = np.int0(corners)

    for k in corners:
        x,y = k.ravel()
        cv2.circle(img, (x,y), 3, 255, -1)
        print x,y

    mplt.figure()
    mplt.imshow(cv2.cvtColor(img, cv2.COLOR_GRAY2BGR))

cornerDetection(img)
cornerDetection(img_2)


I am working on the file "PS-05/.ipynb_checkpoints/problem_set_5-checkpoint.ipynb".
The first task is:
Problem 3
A company that bottles a variety of industrial chemicals has heard
of your success solving imaging problems and hires you to design an approach
for detecting when bottles are not full. The bottles appear as shown below
as they move along a conveyor line past an automatic
filling and capping station. A bottle is considered imperfectly filled when the
level of the liquid is below the midway point between the bottom of the neck and
the shoulder of the bottle.The shoulder is defined as the region of the bottle
where the sides and slanted portion of the bottle intersect. The bottles are
moving, but the company has an imaging system equipped with an illumination
flash front end that effectively stops motion, so you will be given images that
look very close to the sample shown below.
<img src="../data/files/bottles.png" />
Propose a solution for detecting
bottles that are not filled properly. State clearly all assumptions that you
make and that are likely to impact the solution you propose. Implement your
solution and apply it to the images <tt>bottles.tif, new_bottles.jpg</tt> and <tt> three_bottles.jpg</tt>. Visualize the results
of your algorithm by highlighting with false colors
the regions that are detected as correctly
filled bottles and the regions that are detected as not properly filled bottles.
Comentario
La idea es encontrar las botellas que no están llenas apropiadamente. No se requirió de comandos demasiado complejos de OpenCV, de hecho ninguno además de threshold, filter2D y erode. La idea se puede resumir como sigue:

Encontrar el ancho promedio de las botellas, este ancho promedio es aproximadamente el mínimo valor posible de líquido en una botella correcta,
Encontrar la altura en la imagen en la que ocurre por primera vez el ancho promedio (este va a ser el nivel límite de líquido)
Encontrar la cantidad de botellas de la imagen,
Encontrar los centros de estas botellas,
Al nivel mínimo verificar qué botella tiene aire todavía e indexarla.

Para empezar se suaviza la imagen con un filtro2D y...
Can you write Python code for it?

# Images to test
MIN_DIST = 30
LEVEL_OFFSET_INV = 10
LEVEL_OFFSET = 90
imgs = [files[i] for i in [5, -3, -20]]

# Bad bottle detector
def badBottleDetector(img):
    h, w = img.shape

    # Smooth
    kernel = np.ones((3,3),np.float32)/9
    simg = cv2.filter2D(img, -1, kernel)
    simg_c = simg.copy()

    _, simg_ct = cv2.threshold(simg_c, np.mean(simg_c)+LEVEL_OFFSET, 1, cv2.THRESH_BINARY)
    _, simg_cn = cv2.threshold(simg_c, LEVEL_OFFSET_INV, 1, cv2.THRESH_BINARY)
    kernel = np.ones((5,5), np.uint8)
    simg_ct = cv2.erode(simg_ct, kernel, iterations=1)

    bottles = list()
    sizes = list()
    indices = list()
    centers = list()
    for i in range(h):
        ruler = np.zeros([1, w]) + simg_cn[i: i+1]
        ruler = ruler[0]

        seed = 0
        break_points = list()
        for j in range(len(ruler)):
            if (ruler[j] != seed) or (j == len(ruler)-1 and ruler[j] == 1):
                break_points.append(j)
                seed = int(not seed)

        dist = list()
        center = list()
        for j in range(0, len(break_points)-1, 2):
            if break_points[j+1] - break_points[j] > MIN_DIST:
                dist.append(break_points[j+1] - break_points[j])
                center.append(int((break_points[j+1] + break_points[j])/2))

        if dist:
            bottles.append(len(dist))
            sizes.append(np.max(dist))
            indices.append(i)
            centers.append(center)

    counts = np.bincount(bottles)
    num_bottles = np.argmax(counts)
    base_size = int(np.mean(sizes))
    centers = [c for c in centers if len(c) == num_bottles]

    center = list()
    for i in range(num_bottles):
        bt = [c[i] for c in centers]
        center.append(int(np.mean(bt)))

    index = 0
    for i in range(len(sizes)):
        if sizes[i] > base_size:
            index = i
            break
    liquid_min_limit = indices[index]

    # Check which bottle has air at index liquid_min_limit
    ruler =...


I am working on the file "PS-05/.ipynb_checkpoints/problem_set_5-checkpoint.ipynb".
The first task is:
Problem 5
Write a function that extracts local interest points and computes
their descriptors using the SIFT transform. You can find implementations of
the SIFT transform in OpenCV.
Your function should return two matrices: A first matrix of size $3 \times N$, where $N$ is the number of detected points in the image, and the 3 elements correspond to the $x$, $y$ locations and $s$ size of the detected points. A second matrix of size $128 \times N$ that contains the SIFT descriptor of each interest point.
Apply your function to all car images <tt>image_00XX.jpg</tt>.
Store the results of each image in a separate data file.
Comentarios
La Siguiente función hace uso de la implementación de la transformada de SIFT que tiene OpenCV. La función utilizada fue detectAndCompute. La función que se creó, recibe la array de una imagen en escala de grises y regresa los puntos (x,y) y la escala de la imagen en la que fueron encontrados como una matriz de numpy 3xN y también los respectivos descriptores de SIFT para cada punto en otra matriz. También se puede pasar como segundo parametro a la función un booleano que determina si se imprime o no la imagen con los puntos sobre ella, en el caso por defecto, que es False, no imprime nada. Los datos se guardan en archivos de texto en formato json.
Can you write Python code for it?

def getPointsAndDescriptors(img,show_img = False):

    # Getting Keypoint structure object and Descriptor Array
    sift = cv2.SIFT()
    kp, D = sift.detectAndCompute(img,None)

    # Getting the array of points (x,y,s)
    points = np.zeros((3,len(kp)))
    for i in range(len(kp)):
        points[0][i] = kp[i].pt[0]
        points[1][i] = kp[i].pt[1]
        points[2][i] = kp[i].size

    if show_img == True:
        img_s = cv2.drawKeypoints(img, kp, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)
        mplt.imshow(img_s), mplt.xticks([]), mplt.yticks([]), mplt.figure()

    return points, D

# 25 -45 ind of cars images
for i in range(25,45):
    img_name = files[i]
    img = rg(img_name)
    points, D = getPointsAndDescriptors(img)
    f = open("data_image_"+str(i-24)+".json","w")
    data = {"points":points.tolist(),"Descriptors": D.tolist()}
    json.dump(data,f, sort_keys=True, indent=4)
    f.close()
    print "Points and SIFT descriptors for image "+str(i-24)+" extracted"

print "Done!"


I am working on the file "PS-05/problem_set_5.ipynb".
The first task is:
Problem 1
Write a function that describes each object in a binary image using the Hu statistical moments. The Hu moments are invariant to rotation, scale and translation. These moments can be defined for each region in a binary image. The OpenCV function to compute these moments is <tt>cv2.HuMoments</tt>. Write down the equations that compute the seven Hu moments for a region.
Análisis
La siguiente función tiene como objetivo encontrar los Hu moments que describen a cada objeto de la imagen <tt>shapes.png</tt>. Para esto primero guardamos la matriz de la imagen en una variable. Luego definimos nuestra función que recibe una imagen como parámetro. Aplicamos threshold a la imagen para luego con la resultante hallar los objetos con la función <tt>cv2.findContours</tt>. Ya que tenemos los valores de los contours, podemos mostrar cuantos de estos encontró con su longitud y confirmamos que sean la misma cantidad de objetos que se encuentran en la imagen. Ahora, para cada uno de estos objetos aplicamos <tt>cv2.huMoments</tt> y mostramos en pantalla los resultados.
Seven Hu moments
$ I_1 = n_{20} + n_{02} $

$ I_2 = (n_{20} - n_{02})^2 + 4n_{11}^2 $

$ I_3 = (n_{30} - n_{12})^2 + (n_{21} - n_{03})^2  $

$ I_4 = (n_{30} + n_{12})^2 + (n_{21} + n_{03})^2  $

$ I_5 = (n_{30} - 3n_{12})(n_{30} + n_{12})[(n_{30} + n_{12})^2 - 3(n_{21} + n_{03})^2] + (3n_{21} - n_{03})(n_{21} + n_{03})(3(n_{30} + n_{12})^2 - (n_{21} + n_{03})^2)  $

$ I_6 = (n_{20} - n_{02})[(n_{30} + n_{12})^2 - (n_{21} + n_{03})^2] + 4n_{11}(n_{30} + n_{12})(n_{21} + n_{03}) $

$ I_7 = (3n_{21} - n_{03})(n_{30} + n_{12})[(n_{30} + n_{12})^2 - 3(n_{21} + n_{03})^2] - (n_{30} - 3n_{12})(n_{21} + n_{03})(3(n_{30} + n_{12})^2 - (n_{21} + n_{03})^2)  $

Can you write Python code for it?


I am working on the file "PS-05/problem_set_5.ipynb".
The first task is:
Problem 3
A company that bottles a variety of industrial chemicals has heard
of your success solving imaging problems and hires you to design an approach
for detecting when bottles are not full. The bottles appear as shown below
as they move along a conveyor line past an automatic
filling and capping station. A bottle is considered imperfectly filled when the
level of the liquid is below the midway point between the bottom of the neck and
the shoulder of the bottle.The shoulder is defined as the region of the bottle
where the sides and slanted portion of the bottle intersect. The bottles are
moving, but the company has an imaging system equipped with an illumination
flash front end that effectively stops motion, so you will be given images that
look very close to the sample shown below.
<img src="../data/files/bottles.png" />
Propose a solution for detecting
bottles that are not filled properly. State clearly all assumptions that you
make and that are likely to impact the solution you propose. Implement your
solution and apply it to the images <tt>bottles.tif, new_bottles.jpg</tt> and <tt> three_bottles.jpg</tt>. Visualize the results
of your algorithm by highlighting with false colors
the regions that are detected as correctly
filled bottles and the regions that are detected as not properly filled bottles.
Comentario
La idea es encontrar las botellas que no están llenas apropiadamente. No se requirió de comandos demasiado complejos de OpenCV, de hecho ninguno además de threshold, filter2D y erode. La idea se puede resumir como sigue:

Encontrar el ancho promedio de las botellas, este ancho promedio es aproximadamente el mínimo valor posible de líquido en una botella correcta,
Encontrar la altura en la imagen en la que ocurre por primera vez el ancho promedio (este va a ser el nivel límite de líquido)
Encontrar la cantidad de botellas de la imagen,
Encontrar los centros de estas botellas,
Al nivel mínimo verificar qué botella tiene aire todavía e indexarla.

Para empezar se suaviza la imagen con un filtro2D y...
Can you write Python code for it?


I am working on the file "PS-05/problem_set_5.ipynb".
The first task is:
Problem 4
Suppose that you are observing objects in the night sky. Suppose that only ‘big’ objects are important to your observation. In this scenario, ‘small’ objects are considered noise. Write a python function that processes the image as follows:

Use a 15x15 averaging filter to blur the image.

Apply a threshold of 0.25 to binarize the resulting blurred image.

Use the binary image to ‘mask’ the noise of the original image: simply perform an element-wise multiplication of the binary image and the original image.

Use connected component analysis on the binary image to count the number of ‘big’ objects found.

The function should take three inputs: an image matrix, the size of the averaging filter and threshold value. Make sure your function displays the intermediary results of each step outlined above.
Apply your function to the input image ‘hubble-original.tif’. Try different values of smoothing kernel size and threshold value. Analyze the relationship between number of objects found and smoothing kernel size and threshold value. In particular, you might want to observe the result when using an averaging filter of size n=1 (i.e. no smoothing).
Comentarios
La idea del programa siguiente es encontrar cuerpos grandes en las imágenes; para ello se realizan los procedimientos anteriormente pedidos, esto es el filtro 15x15, la aplicación de la umbralización para crear la máscara, la eliminación de ruido de la imagen original y el análisis de cuerpo conexo. Estos pasos son sencillos de realizar y ya se han implementado anteriormente, a excepción del análisis de cuerpo conexo. Para comprobar conexión en un cuerpo, se elige un punto de semilla (aquel punto con intensidad de 1), a partir de este punto se analizan los vecinos de forma recursiva, esto es, se vuelve a llamar la función sobre los vecinos si estos tienen intensidad de 1. Esto arroja la cantidad de cuerpos conexos en la imagen, sin embargo, no hay garantía que estos cuerpos sean grandes. Para asegurar lo anterior se cuentan también el número de...
Can you write Python code for it?

img_origin = rg('hubble-original.tif')

BIG_OBJECT_COUNT = 80

def printer(iss, des):
    # Printing
    f, ax = mplt.subplots(1, len(iss), figsize=(15,15))

    for i in range(len(iss)):
        ax[i].imshow(iss[i], cmap='gray')
        ax[i].set_title(des[i])

def connectedLabeling(mat, i, j, h, w):
    mat[i][j] = 0
    if i+1 < h and mat[i+1][j] == 1:
        return 1 + connectedLabeling(mat, i+1, j, h, w)
    elif i-1 > -1 and mat[i-1][j] == 1:
        return 1 + connectedLabeling(mat, i-1, j, h, w)
    elif j+1 < w and mat[i][j+1] == 1:
        return 1 + connectedLabeling(mat, i, j+1, h, w)
    elif j-1 > -1 and mat[i][j-1] == 1:
        return 1 + connectedLabeling(mat, i, j-1, h, w)
    elif i-1 > -1 and j-1 > -1 and mat[i-1][j-1] == 1:
        return 1 + connectedLabeling(mat, i-1, j-1, h, w)
    elif i-1 > -1 and j+1 < w and mat[i-1][j+1] == 1:
        return 1 + connectedLabeling(mat, i-1, j+1, h, w)
    elif i+1 < h and j-1 > -1 and mat[i+1][j-1] == 1:
        return 1 + connectedLabeling(mat, i+1, j-1, h, w)
    elif i+1 < h and j+1 < w and mat[i+1][j+1] == 1:
        return 1 + connectedLabeling(mat, i+1, j+1, h, w)
    else:
        return 1

def countObjects(img, avg_size, th):
    kernel = np.ones((avg_size,avg_size),np.float32)/(avg_size*avg_size)
    fimg = cv2.filter2D(img, -1, kernel)
    _, timg = cv2.threshold(fimg, 255*th, 1, cv2.THRESH_BINARY)

    masked = img * timg

    # Count the objects in the masked image
    timgc = timg.copy()

    h, w = maskc.shape
    for i in range(h):
        for j in range(w):
            if timgc[i][j] == 1:
                neighbors = connectedLabeling(timgc, i, j, h, w)

                if neighbors > BIG_OBJECT_COUNT:
                    # print(neighbors)
                    timgc[i][j] = 1

    count = np.sum(timgc)
    print('Objetos grandes encontrados: ' + str(count))
    print('Se definió objeto grande aquel cuyo conjunto conexo tiene más de ' + str(BIG_OBJECT_COUNT) + ' miembros')
    print('Este parámetro se puede variar')

   ...


I am working on the file "PS-05/problem_set_5.ipynb".
The first task is:
Problem 5
Write a function that extracts local interest points and computes
their descriptors using the SIFT transform. You can find implementations of
the SIFT transform in OpenCV.
Your function should return two matrices: A first matrix of size $3 \times N$, where $N$ is the number of detected points in the image, and the 3 elements correspond to the $x$, $y$ locations and $s$ size of the detected points. A second matrix of size $128 \times N$ that contains the SIFT descriptor of each interest point.
Apply your function to all car images <tt>image_00XX.jpg</tt>.
Store the results of each image in a separate data file.
Comentarios
La Siguiente función hace uso de la implementación de la transformada de SIFT que tiene OpenCV. La función utilizada fue detectAndCompute. La función que se creó, recibe la array de una imagen en escala de grises y regresa los puntos (x,y) y la escala de la imagen en la que fueron encontrados como una matriz de numpy 3xN y también los respectivos descriptores de SIFT para cada punto en otra matriz. También se puede pasar como segundo parametro a la función un booleano que determina si se imprime o no la imagen con los puntos sobre ella, en el caso por defecto, que es False, no imprime nada. Los datos se guardan en archivos de texto en formato json.
Can you write Python code for it?


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
Accuracy v.s. MAG
Can you write Python code for it?

dei_class = np.array(deimos_match_uniq_good["class"], dtype=int)
hst_class = np.array(hst_match_uniq_good["MU_CLASS"]-1, dtype=int)
kron_mag = np.array(hst_match_uniq_good["MAG_BEST"])

binwidth = 1
Nboot = 100
mag_array = np.arange(14 , 26+binwidth, binwidth)

ml_acc_arr = np.zeros_like(mag_array, dtype=float)
ml_boot_scatt = np.vstack((np.zeros_like(mag_array, dtype=float), np.zeros_like(mag_array, dtype=float)))

for bin_num, binedge in enumerate(mag_array):
    bin_sources = np.where((kron_mag >= binedge) & (kron_mag < binedge + binwidth))
    ml_acc_arr[bin_num] = accuracy_score(dei_class[bin_sources],
                                         hst_class[bin_sources])
    ml_boot_acc = np.empty(Nboot)
    for i in range(Nboot):
        boot_sources = np.random.choice(bin_sources[0], len(bin_sources[0]),
                                        replace=True)
        ml_boot_acc[i] = accuracy_score(dei_class[boot_sources],
                                         hst_class[boot_sources])

    ml_boot_scatt[:,bin_num] = np.percentile(ml_boot_acc, [16, 84])

from sklearn.neighbors import KernelDensity
kde_grid = np.linspace(10,29,200)
deimos_stars = np.where(dei_class == 1)
deimos_gal = np.where(dei_class == 0)
deimos_kde_gal_norm = len(deimos_gal[0])/len(dei_class)
deimos_kde_star_norm = 1 - deimos_kde_gal_norm

kde_deimos = KernelDensity(bandwidth=1.059*np.std(kron_mag, ddof=1)*len(kron_mag)**(-0.2),
                         rtol=1E-4)
kde_deimos.fit(kron_mag[:, np.newaxis])

kde_deimos_stars = KernelDensity(bandwidth=1.059*np.std(kron_mag[deimos_stars], ddof=1)*len(kron_mag[deimos_stars])**(-0.2),
                               rtol=1E-4)
kde_deimos_stars.fit(kron_mag[deimos_stars[0], np.newaxis])

kde_deimos_gal = KernelDensity(bandwidth=1.059*np.std(kron_mag[deimos_gal], ddof=1)*len(kron_mag[deimos_gal])**(-0.2),
                             rtol=1E-4)
kde_deimos_gal.fit(kron_mag[deimos_gal[0], np.newaxis])

pdf_deimos = np.exp(kde_deimos.score_samples(kde_grid[:, np.newaxis]))
pdf_deimos_stars =...


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
Accuracy v.s. MAG with DEIMOSxHST
Can you write Python code for it?

from astropy.table import Table

deimos = pd.read_csv('./DEIMOS/DEIMOS.csv')

hst = Table.read('./DEIMOS/HST_COSMOS.fit').to_pandas()

hstX = np.empty((len(hst), 2), dtype=np.float64)
hstX[:, 0] = hst['ALPHA_J2000']
hstX[:, 1] = hst['DELTA_J2000']
deiX = np.empty((len(deimos), 2), dtype=np.float64)
deiX[:, 0] = deimos['ra']
deiX[:, 1] = deimos['dec']


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
ROC curve and Accuracy
Can you write Python code for it?

ps1_dei = pd.read_csv('./DEIMOS/PS1_DEIMOS_features.csv').drop_duplicates(subset='objid')
print("PS1xDEIMOS catalog constains %i sources."%len(ps1_dei))

ps1_dei_det_mask = np.logical_and(ps1_dei['class'] != -999, (ps1_dei.nDetections>0)&(ps1_dei.wwKronFlux>0))
ps1_dei = ps1_dei[ps1_dei_det_mask]
print("%i sources are classified by both of the DEIMOS and the ML model."%len(ps1_dei))

ps1_df = pd.read_csv('./DEIMOS/HST_COSMOS_features.csv')

dupl_mask = np.empty(len(ps1_dei), dtype=bool)
for i in range(len(dupl_mask)):
    dupl_mask[i] = ps1_dei.objid.iloc[i] in np.array(ps1_df.objid)

print("Only %i sources are included both of the PS1xDEIMOS and the PS1xHST catalog..."%np.sum(dupl_mask))
ps1_dei = ps1_dei[~dupl_mask]
#print("%i sources are not contained in PS1xHST catalog."%len(ps1_dei))

kron_mag = -2.5*np.log10(ps1_dei.wwKronFlux/3631)
ps1_dei_features = ps1_dei[features]
ps1_dei_class = ps1_dei['class']

ps1_dei_score = rf_obj.rf_clf_.predict_proba(ps1_dei_features)
ps1_dei_pred = rf_obj.rf_clf_.predict(ps1_dei_features)

print("Overall accuracy of the classification by the ML model is %f"%accuracy_score(ps1_dei_class, ps1_dei_pred))

fpr, tpr, thre = roc_curve(ps1_dei_class, ps1_dei_score[:,1])
plt.grid(linestyle='dotted')
plt.plot(fpr, tpr, 'k-')
#plt.xscale('log'); plt.yscale('log')
plt.xlim(1e-3, 1e-1); plt.ylim(0.1, 1.01)
plt.xlabel('FPR'); plt.ylabel('TPR')
plt.show()

ps1_dei_class = np.array(ps1_dei_class)
ps1_dei_score = np.array(ps1_dei_score)
kron_mag = np.array(kron_mag)

binwidth = 1.5
Nboot = 100
mag_array = np.arange(14 , 23+binwidth, binwidth)
kron_mag = np.array(-2.5*np.log10(ps1_dei['wwKronFlux']/3631))

ml_acc_arr = np.zeros_like(mag_array, dtype=float)
ml_boot_scatt = np.vstack((np.zeros_like(mag_array, dtype=float), np.zeros_like(mag_array, dtype=float)))

for bin_num, binedge in enumerate(mag_array):
    bin_sources = np.where((kron_mag >= binedge) & (kron_mag < binedge + binwidth))
    ml_acc_arr[bin_num] = accuracy_score(ps1_dei_class[bin_sources],
                            ...


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
Remove the sources which are not able to be classified to star or galaxy by the DEIMOS catalog.
Can you write Python code for it?

good_mask = deimos_match_uniq["class"] != -999

deimos_match_uniq_good = deimos_match_uniq[good_mask.values]
hst_match_uniq_good = hst_match_uniq[good_mask.values]

print("The number of sources used to verify the classification accuracy is %i"%len(deimos_match_uniq))

xlims = [12, 29]
ylims = [12, 29]
plt.hexbin(hst_match_uniq["MAG_BEST"], deimos_match_uniq["imag"],
           extent=[xlims[0], xlims[1], ylims[0], ylims[1]],
          bins='log', cmap='viridis')
plt.xlim(xlims); plt.ylim(ylims)
plt.xlabel('MAG_BEST(HST)')
plt.ylabel('imag(DEIMOS)')

from sklearn.metrics import accuracy_score

print("The overall accuracy od the crassification of the HST catalog is %0.4f"\
      %accuracy_score(deimos_match_uniq_good["class"], hst_match_uniq_good["MU_CLASS"]-1))


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
The distribution of galaxies looks similar to that of the HST COSMOS catalog, but that of stars has a peak at i-mag$\sim$22, which is not shown in that of the HSTxPS1 catalog.
Can you write Python code for it?

df = pd.DataFrame()
df['ra'] = ra; df['dec'] = dec
df['sel'] = sel
df['imag'] = imag; df['kmag'] = kmag
df['zspec'] = zspec
df['Qflag'] = Qflag; df['Q'] = Q
df['class'] = sgFlag

df[0:10]

df.to_csv('./DEIMOS/DEIMOS.csv', index=None)

import star_galaxy_models
rf_obj = star_galaxy_models.RandomForestModel()
rf_obj.read_rf_from_pickle()

features = ['wwpsfChiSq', 'wwExtNSigma', 'wwpsfLikelihood',
            'wwPSFKronRatio', 'wwPSFKronDist',  'wwPSFApRatio',
            'wwmomentRH', 'wwmomentXX', 'wwmomentXY', 'wwmomentYY',
            'wwKronRad']

from sklearn.metrics import roc_curve, accuracy_score, auc, make_scorer


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
The distribution of the distance has a peak at 0.1 arcsec. Changes the cross-matching radius to 0.3 arcsec.
Can you write Python code for it?

from astroML.crossmatch import crossmatch_angular
max_radius = 0.3 / 3600  # 0.3 arcsec
dist, ind = crossmatch_angular(hstX, deiX, max_radius)
match = ~np.isinf(dist)

print("The number of sources cross-matched is %i"%np.sum(match))

plt.hist(dist[match]*3600, bins=np.arange(0, 0.5,0.01))
plt.xlabel('Distance')
plt.show()

hst_match = hst[match]
deimos_match = deimos.loc[ind[match]]


I am working on the file "PS1casjobs/DEIMOS_COSMOS.ipynb".
The first task is:
if "Remarks" contains "star", the source is classifyed star.
Can you write Python code for it?

plt.hist(imag[sgFlag!=-999], bins=np.arange(15, 28, 0.2), color='0.8', label='All')
plt.hist(imag[sgFlag==0], bins=np.arange(15, 28, 0.2), alpha=0.5, label='GALAXY')
plt.hist(imag[sgFlag==1], bins=np.arange(15, 28, 0.2), alpha=0.5, label='STAR')
plt.yscale('log')
plt.xlabel('i mag'); plt.ylabel('#')
plt.legend(loc='best')
plt.show()


I am working on the file "PS1casjobs/DefineTrainingSet.ipynb".
The first task is:
From here is seems clear that the criteria necessary to satisfy the "suspect" condition are relatively narrow, and the real issue is whether or not poor photometry significantly affects the model.
Nearly 24% of all sources in the HST COSMOS-PS1 cross match have poor photometry. This has the potential to seriously jeopardize the efficacy of the model.
Can you write Python code for it?

print("Of the {} mean det sources, {} have suspect, and {} have poor photometry".format(sum(mean_det), sum(mean_det & suspect_phot), sum(mean_det & poor_phot)))


I am working on the file "PS1casjobs/DefineTrainingSet.ipynb".
The first task is:
On the other hand, only 6% of the sources that have detections in the mean catalog have poor photometry.
As shown below, these sources begin to dominate at wwKronMag ~ 23.7 mag, meaning they will dominate classifications for sources this faint [there will be few sources this faint with nDetections > 0].
Can you write Python code for it?

whiteKronMag = -2.5*np.log10(ts["wwKronFlux"]/3631)

plt.hist(whiteKronMag, bins=np.arange(14,25.5,0.25),alpha=0.5)
plt.hist(whiteKronMag[mean_det], bins=np.arange(14,25.5,0.25), alpha=0.5)
plt.hist(whiteKronMag[poor_phot], bins=np.arange(14,25.5,0.25), alpha=0.5)
plt.hist(whiteKronMag[poor_phot & mean_det], bins=np.arange(14,25.5,0.25), alpha=0.8)


I am working on the file "PS1casjobs/WeightsComparison.ipynb".
The first task is:
The picture changes dramatically when weighting all the flux measurements by the PSF uncertainties. For PSFKron the statistics reamin similar, but the major change is for the PSFAp ratios, which show tighter distributions than the use of Kron [as traced by IQR]. The contrast in the mean is smaller, however, and the use of Aperture photometry leads to more heavily skewed distributions.
Can you write Python code for it?

psf_kron_ww = calc_wwFlux('PSFFlux', 'KronFluxErr', hst_data, det_arr)
kron_kron_ww = calc_wwFlux('KronFlux', 'KronFluxErr', hst_data, det_arr)
ap_kron_ww = calc_wwFlux('ApFlux', 'KronFluxErr', hst_data, det_arr)

psf_kron = psf_kron_ww/kron_kron_ww
psf_ap = psf_kron_ww/ap_kron_ww

star_det = np.logical_and(hst_data['MU_CLASS'] == 2, psf_kron_ww > 0)
gal_det = np.logical_and(hst_data['MU_CLASS'] == 1, psf_kron_ww > 0)
star_mean_det = np.logical_and.reduce((hst_data['MU_CLASS'] == 2,
                                      psf_psf_ww > 0,
                                      hst_data['nDetections'] > 0))
gal_mean_det = np.logical_and.reduce((hst_data['MU_CLASS'] == 1,
                                     psf_psf_ww > 0,
                                     hst_data['nDetections'] > 0))

with sns.axes_style('whitegrid'):
    fig, (ax1, ax2) = plt.subplots(1, 2, sharex=True, figsize=(10,5))

    for subset, col, ht in zip([gal_det, gal_mean_det, star_det, star_mean_det],
                               ["C0", "C0", "C1", "C1"],
                               ["step", "stepfilled", "step", "stepfilled"]):
        ax1.hist(psf_kron[subset], histtype=ht, lw=2, color=col,
                 bins=np.arange(0,2.1,0.025), alpha=0.7)
        ax2.hist(psf_ap[subset], histtype=ht, lw=2, color=col,
                 bins=np.arange(0,2.1,0.025), alpha=0.7)
    ax2.set_xticks(np.linspace(0,2,11))
    fig.tight_layout()

# calculate a mini-stats table
ww_properties = pd.DataFrame(columns=['name', 'IQR', 'Median', 'Mean', 'Std', 'Skew'])
ww_properties['name'] = ['PSFKron_star', 'PSFKron_gal', 'PSFAp_star', 'PSFAp_gal']
ww_properties.set_index('name', inplace=True)
for pd_idx, flux_ratio in zip(ww_properties.index,
                              [psf_kron[star_det], psf_kron[gal_det],
                               psf_ap[star_det], psf_ap[gal_det]]):
    ww_properties["IQR"][pd_idx] = iqr(flux_ratio)
    ww_properties["Median"][pd_idx] = np.median(flux_ratio)
    describe_ratio = describe(flux_ratio)
   ...


I am working on the file "PS1casjobs/WeightsComparison.ipynb".
The first task is:
The results when weighting using the Kron Flux uncertainties is similar to the PSF flux uncertainties. THe skew and IQR are slightly smaller with Kron weighting, but the differences are small.
Can you write Python code for it?

psf_ap_ww = calc_wwFlux('PSFFlux', 'ApFluxErr', hst_data, det_arr)
kron_ap_ww = calc_wwFlux('KronFlux', 'ApFluxErr', hst_data, det_arr)
ap_ap_ww = calc_wwFlux('ApFlux', 'ApFluxErr', hst_data, det_arr)

psf_kron = psf_ap_ww/kron_ap_ww
psf_ap = psf_ap_ww/ap_ap_ww

star_det = np.logical_and(hst_data['MU_CLASS'] == 2, psf_ap_ww > 0)
gal_det = np.logical_and(hst_data['MU_CLASS'] == 1, psf_ap_ww > 0)
star_mean_det = np.logical_and.reduce((hst_data['MU_CLASS'] == 2,
                                      psf_psf_ww > 0,
                                      hst_data['nDetections'] > 0))
gal_mean_det = np.logical_and.reduce((hst_data['MU_CLASS'] == 1,
                                     psf_psf_ww > 0,
                                     hst_data['nDetections'] > 0))

with sns.axes_style('whitegrid'):
    fig, (ax1, ax2) = plt.subplots(1, 2, sharex=True, figsize=(10,5))

    for subset, col, ht in zip([gal_det, gal_mean_det, star_det, star_mean_det],
                               ["C0", "C0", "C1", "C1"],
                               ["step", "stepfilled", "step", "stepfilled"]):
        ax1.hist(psf_kron[subset], histtype=ht, lw=2, color=col,
                 bins=np.arange(0,2.1,0.025), alpha=0.7)
        ax2.hist(psf_ap[subset], histtype=ht, lw=2, color=col,
                 bins=np.arange(0,2.1,0.025), alpha=0.7)
    ax2.set_xticks(np.linspace(0,2,11))
    fig.tight_layout()

# calculate a mini-stats table
ww_properties = pd.DataFrame(columns=['name', 'IQR', 'Median', 'Mean', 'Std', 'Skew'])
ww_properties['name'] = ['PSFKron_star', 'PSFKron_gal', 'PSFAp_star', 'PSFAp_gal']
ww_properties.set_index('name', inplace=True)
for pd_idx, flux_ratio in zip(ww_properties.index,
                              [psf_kron[star_det], psf_kron[gal_det],
                               psf_ap[star_det], psf_ap[gal_det]]):
    ww_properties["IQR"][pd_idx] = iqr(flux_ratio)
    ww_properties["Median"][pd_idx] = np.median(flux_ratio)
    describe_ratio = describe(flux_ratio)
    ww_properties["Mean"][pd_idx] =...


I am working on the file "PS1casjobs/WeightsComparison.ipynb".
The first task is:
To determine the weighted white flux for a source, an individual filter can only be included if the source is detected in that filter, where detection is defined as:

PSFFlux > 0
ApFlux > 0
KronFlux > 0

[As a brief side note - the same test for Mean and Forced photometry from PS1 needs to include FluxErr as well]
Can you write Python code for it?

for filt in ['g','r','i','z','y']:
    exec("""det_{0} = np.logical_and.reduce((hst_data['{0}PSFFlux'] > 0,
                                             hst_data['{0}ApFlux'] > 0,
                                             hst_data['{0}KronFlux'] > 0))""".format(filt))
det_arr = np.vstack((det_g, det_r, det_i, det_z, det_y))

def calc_wwFlux(flux_type, weight_type, flux_table, det_arr):
    flux_arr = np.zeros_like(det_arr).astype(float)
    flux_unc_arr = np.zeros_like(det_arr).astype(float)
    for filtnum, filt in enumerate(['g','r','i','z','y']):
        flux_col = filt + flux_type
        unc_col = filt + weight_type
        flux_arr[filtnum] = np.divide(flux_table[flux_col]**3,
                                      flux_table[unc_col]**2,
                                      where=det_arr[filtnum],
                                      out=np.zeros_like(flux_table[flux_col]))
        flux_unc_arr[filtnum] = np.divide(flux_table[flux_col]**2,
                                          flux_table[unc_col]**2,
                                          where=det_arr[filtnum],
                                          out=np.zeros_like(flux_table[flux_col]))
    return np.divide(np.sum(flux_arr, axis=0),
                     np.sum(flux_unc_arr, axis=0),
                     where=np.sum(flux_unc_arr, axis=0)>0,
                     out=-999*np.ones_like(flux_unc_arr[0]))

cmap_star = sns.cubehelix_palette(rot=0.5, light=0.7,dark=0.3,as_cmap=True)
cmap_gal = sns.cubehelix_palette(start=0.3,rot=-0.5,light=0.7,dark=0.3,as_cmap=True)


I am working on the file "PS1casjobs/WeightsComparison.ipynb".
The first task is:
When weighting each white flux measurement by its corresponding uncertainties, it is clear that Kron does a much better job of separating stars and galaxies. My best guess for the reason behind this is that the aperture uncertainties are significantly underestimated and therefore wrong.
Focusing on the PSFKron distribution, the means are separated by $\sim$0.25, and the IQR is relatively narrow.
Can you write Python code for it?

psf_psf_ww = calc_wwFlux('PSFFlux', 'PSFFluxErr', hst_data, det_arr)
kron_psf_ww = calc_wwFlux('KronFlux', 'PSFFluxErr', hst_data, det_arr)
ap_psf_ww = calc_wwFlux('ApFlux', 'PSFFluxErr', hst_data, det_arr)

psf_kron = psf_psf_ww/kron_psf_ww
psf_ap = psf_psf_ww/ap_psf_ww

star_det = np.logical_and(hst_data['MU_CLASS'] == 2, psf_psf_ww > 0)
gal_det = np.logical_and(hst_data['MU_CLASS'] == 1, psf_psf_ww > 0)
star_mean_det = np.logical_and.reduce((hst_data['MU_CLASS'] == 2,
                                      psf_psf_ww > 0,
                                      hst_data['nDetections'] > 0))
gal_mean_det = np.logical_and.reduce((hst_data['MU_CLASS'] == 1,
                                     psf_psf_ww > 0,
                                     hst_data['nDetections'] > 0))

with sns.axes_style('whitegrid'):
    fig, (ax1, ax2) = plt.subplots(1, 2, sharex=True, figsize=(10,5))

    for subset, col, ht in zip([gal_det, gal_mean_det, star_det, star_mean_det],
                               ["C0", "C0", "C1", "C1"],
                               ["step", "stepfilled", "step", "stepfilled"]):
        ax1.hist(psf_kron[subset], histtype=ht, lw=2, color=col,
                 bins=np.arange(0,2.1,0.025), alpha=0.7)
        ax2.hist(psf_ap[subset], histtype=ht, lw=2, color=col,
                 bins=np.arange(0,2.1,0.025), alpha=0.7)
    ax2.set_xticks(np.linspace(0,2,11))
    fig.tight_layout()

# calculate a mini-stats table
ww_properties = pd.DataFrame(columns=['name', 'IQR', 'Median', 'Mean', 'Std', 'Skew'])
ww_properties['name'] = ['PSFKron_star', 'PSFKron_gal', 'PSFAp_star', 'PSFAp_gal']
ww_properties.set_index('name', inplace=True)
for pd_idx, flux_ratio in zip(ww_properties.index,
                              [psf_kron[star_det], psf_kron[gal_det],
                               psf_ap[star_det], psf_ap[gal_det]]):
    ww_properties["IQR"][pd_idx] = iqr(flux_ratio)
    ww_properties["Median"][pd_idx] = np.median(flux_ratio)
    describe_ratio = describe(flux_ratio)
    ww_properties["Mean"][pd_idx] =...


I am working on the file "PSD.ipynb".
The first task is:
Calculate SNR using the frequency domain (first peak is the signal assumption).
Can you write Python code for it?

f_sig_idx = np.argmax(Pxx)
SNR_f = 10 * np.log10(Pxx[f_sig_idx] / np.sum(np.delete(Pxx, f_sig_idx)))

print('SNR= %.2f dB (time domain SNR= %.2f dB)' % (SNR_f, SNR_t))


I am working on the file "PSD.ipynb".
The first task is:
Calculate the average power of the clean signal and of the noise from the time domain samples. Compute SNR.
Note: the power of a sinusoid with unit amplitude is -3dB.
Can you write Python code for it?

P_sig_t = np.mean(sig ** 2) # same as np.sum((sig ** 2) * dT) / T
P_noise_t = np.mean(noise ** 2)
SNR_t = 10 * np.log10(P_sig_t / P_noise_t)

print('P(sig)= %.2f V^2, P(noise)= %.2f V^2, SNR= %.2f dB' % (P_sig_t, P_noise_t, SNR_t))
print('RMS(sig)= %.2f V, RMS(noise)= %.2f V' % (np.sqrt(P_sig_t), np.sqrt(P_noise_t)))


I am working on the file "PSD.ipynb".
The first task is:
Due to the real time-domain samples we have a symmetric spectrum (complex conjugate). Let's take and scale the positive half of it.
Can you write Python code for it?

Pxx = Pxx[:N // 2]
Pxx[1:-2] *= 2    # conserve energy
f = f[:N // 2]

plt.semilogy(f, Pxx)
plt.title('Power Spectrum')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Power in bin ($V^2$)')
plt.ylim(1e-6, 1)
plt.grid()


I am working on the file "PSD.ipynb".
The first task is:
First important observation: the squared magnitude of the FFT values represent the energy distribution across the frequency bins for the given signal length (N). Thus, the absolute bin values depend on N.
Can you write Python code for it?

Exx = np.abs(X) ** 2

plt.semilogy(np.fft.fftshift(f), np.fft.fftshift(Exx))
plt.title('Energy Spectrum')
plt.xlabel('Frequency (Hz)')
plt.ylabel('Energy in bin ($V^2s$)')


I am working on the file "PSD.ipynb".
The first task is:
Observation: the PSD figure is better for showing the noise level (it's height does not change with N), but is hard to interpret for the signal (it's height changes). The 'spectrum' scaling is better for the signal (does not change with N) but misleading for the noise level.
Can you write Python code for it?

f3, Pxx3  = signal.periodogram(samples, FS, nfft=512, scaling='density')
plt.semilogy(f3, Pxx3)
plt.title('PSD with N=1024')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD ($V^2/Hz$)')
plt.ylim(1e-7, 1)
plt.grid()
plt.show()

f3, Pxx3  = signal.periodogram(samples, FS, nfft=8192, scaling='density')
plt.semilogy(f3, Pxx3)
plt.title('PSD with N=1024')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD ($V^2/Hz$)')
plt.ylim(1e-7, 1)
plt.grid()
plt.show()

f3, Pxx3  = signal.periodogram(samples, FS, nfft=512, scaling='spectrum')
plt.semilogy(f3, Pxx3)
plt.title('Power Spectrum with N=1024')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD ($V^2/Hz$)')
plt.ylim(1e-7, 1)
plt.grid()
plt.show()

f3, Pxx3  = signal.periodogram(samples, FS, nfft=8192, scaling='spectrum')
plt.semilogy(f3, Pxx3)
plt.title('Power Spectrum with N=1024')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD ($V^2/Hz$)')
plt.ylim(1e-7, 1)
plt.grid()
plt.show()


I am working on the file "PSD.ipynb".
The first task is:
Power Spectrum
Compute the DFT of the time domain samples using a fixed length (N).
Note: the DFT results have to be scaled by 1/sqrt(N) to conserve energy (unitary operator). You can achieve the same results with np.fft.fft(samples, norm='ortho'). Also, see Parseval's Theorem.
Can you write Python code for it?

N = 1000      # must be even for these computations

X = np.fft.fft(samples, n=N) / np.sqrt(N)
f = np.fft.fftfreq(N, dT)
# Verify if time and frequency domain energies are the same
np.sum(np.abs(X) ** 2), np.sum(samples[:N] ** 2)


I am working on the file "PSD.ipynb".
The first task is:
Power Spectrum Density
Instead of ploting the (average) power in each frequency bin we can compute/plot the power density. This is a scaling of the power spectrum results by the width of the bin (in Hz). We also compare this to the built-in periodogram with density scaling.
Can you write Python code for it?

plt.semilogy(f, Pxx / (FS / N))
plt.title('PSD computed from DFT')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD ($V^2/Hz$)')
plt.ylim(1e-7, 1)
plt.grid()
plt.show()

f2, Pxx2  = signal.periodogram(samples, FS, nfft=N, scaling='density')
plt.semilogy(f2, Pxx2)
plt.title('PSD using scipy.signal.periodogram')
plt.xlabel('Frequency (Hz)')
plt.ylabel('PSD ($V^2/Hz$)')
plt.ylim(1e-7, 1)
plt.grid()


I am working on the file "PSO/1D/1D-Python-PSO-algorithm-viz.ipynb".
The first task is:
Note:
<div class=\"alert alert-success\">
As of ipywidgets 5.0, only static images of the widgets in this notebook will show on http://nbviewer.ipython.org. To view the live widgets and interact with them, you will need to download this notebook and run it with a Jupyter Notebook server.
</div>
Can you write Python code for it?

# More examples in https://github.com/ipython/ipywidgets/tree/master/docs/source/examples


I am working on the file "PSO/1D/1D-Python-PSO-algorithm-viz.ipynb".
The first task is:
PSO Algorithm
Can you write Python code for it?

n_iterations = 50

def run_PSO(n_particles=5, omega=0.5, phi_p=0.5, phi_g=0.7):
    """ PSO algorithm to a funcion already defined.
    Params:
        omega = 0.5  # Particle weight (intertial)
        phi_p = 0.1  # particle best weight
        phi_g = 0.1  # global global weight

    """
    global x_best_p_global, x_particles, y_particles, u_particles, v_particles

    # Note: we are using global variables to ease the use of interactive widgets
    # This code will work fine without the global (and actually it will be safer)

    ## Initialization:

    x_particles = np.zeros((n_particles, n_iterations))
    x_particles[:, 0] = np.random.uniform(x_lo, x_up, size=n_particles)

    x_best_particles = np.copy(x_particles[:, 0])

    y_particles = f(x_particles[:, 0])
    y_best_global = np.min(y_particles[:])
    index_best_global = np.argmin(y_particles[:])
    x_best_p_global = np.copy(x_particles[index_best_global,0])

    # velocity units are [Length/iteration]
    velocity_lo = x_lo-x_up
    velocity_up = x_up-x_lo

    u_particles = np.zeros((n_particles, n_iterations))
    u_particles[:, 0] = 0.1*np.random.uniform(velocity_lo, velocity_up, size=n_particles)

    v_particles = np.zeros((n_particles, n_iterations)) # Needed for plotting the velocity as vectors

    # PSO STARTS

    iteration = 1
    while iteration <= n_iterations-1:
        for i in range(n_particles):
                x_p = x_particles[i, iteration-1]
                u_p = u_particles[i, iteration-1]
                x_best_p = x_best_particles[i]

                r_p = np.random.uniform(0, 1)
                r_g = np.random.uniform(0, 1)

                u_p_new = omega*u_p+ \
                            phi_p*r_p*(x_best_p-x_p) + \
                            phi_g*r_g*(x_best_p_global-x_p)

                x_p_new = x_p + u_p_new

                if not x_lo <= x_p_new <= x_up:
                    x_p_new = x_p # ignore new position, it's out of the domain
                    u_p_new = 0

               ...


I am working on the file "PSO/1D/1D-Python-PSO-algorithm-viz.ipynb".
The first task is:
We can define and plot the function we want to optimize:
Can you write Python code for it?

x_lo = 0
x_up = 14
n_points = 1000

x = np.linspace(x_lo, x_up, n_points)

def f(x):
    return x*np.sin(x) + x*np.cos(x)

y = f(x)

plt.plot(x,y)
plt.ylabel('$f(x) = \sin(x)+x\cos(x)$')
plt.xlabel('$x$')
plt.title('Function to be optimized')


I am working on the file "PSO/2D/2D-Python-PSO-algorithm-viz.ipynb".
The first task is:
Animation
Can you write Python code for it?

from __future__ import print_function
import ipywidgets as widgets
from IPython.display import display, HTML

def plotPSO_iter(i=0): #iteration
    """Visualization of particles and obj. function"""

    fig, (ax1, ax2) = plotPSO_2D(f, limits,
               particles_xy=(x_particles[i, :],y_particles[i, :]),
               particles_uv=(u_particles[i, :],v_particles[i, :]))

w_arg_PSO = widgets.interact_manual(run_PSO,
                            n_particles=(2,50),
                            omega=(0,1,0.001),
                            phi_p=(0,1,0.001),
                            phi_g=(0,1,0.001),
                            continuous_update=False)

w_viz_PSO = widgets.interact_manual(plotPSO_iter, i=(0,n_iterations-1), continuous_update=False)


I am working on the file "PSO/2D/2D-Python-PSO-algorithm-viz.ipynb".
The first task is:
PSO Algorithm
Can you write Python code for it?

n_iterations = 50

def run_PSO(n_particles=10, omega=0.3, phi_p=0.7, phi_g=0.7):
    """ PSO algorithm to a funcion already defined.
    Params:
        omega = 0.3  # Particle weight (intertial)
        phi_p = 0.7  # particle best weight
        phi_g = 0.7  # global global weight

    """
    global x_best_p_global, y_best_p_global, z_p_best_global, \
           x_particles, y_particles, z_particles, \
           u_particles, v_particles

    # Note: we are using global variables to ease the use of interactive widgets
    # This code will work fine without the global (and actually it will be safer)

    # Initialazing x postion of particles

    x_particles = np.zeros((n_iterations, n_particles))
    x_particles[0,:] = np.random.uniform(lo_b, up_b, size=n_particles)

    # Initialazing y postion of particles
    y_particles = np.zeros((n_iterations, n_particles))
    y_particles[0,:] = np.random.uniform(lo_b, up_b, size=n_particles)

    # Initialazing best praticles
    x_best_particles = np.copy(x_particles[0,:])
    y_best_particles = np.copy(y_particles[0,:])

    # Calculate Objective function (aka fitness function)
    z_particles = np.zeros((n_iterations, n_particles))

    for i in range(n_particles):
        z_particles[0,i] = f((x_particles[0,i],y_particles[0,i]))

    z_best_global = np.min(z_particles[0,:])
    index_best_global = np.argmin(z_particles[0,:])

    x_best_p_global = x_particles[0,index_best_global]
    y_best_p_global = y_particles[0,index_best_global]

    # Initialazin velocity
    velocity_lo = lo_b-up_b  # [L/iteration]
    velocity_up = up_b-lo_b  # [L/iteration]

    v_max = 0.07 # [L/iteration]

    u_particles = np.zeros((n_iterations, n_particles))
    u_particles[0,:] = 0.1*np.random.uniform(velocity_lo, velocity_up, size=n_particles)

    v_particles = np.zeros((n_iterations, n_particles))
    v_particles[0,:] = 0.1*np.random.uniform(velocity_lo, velocity_up, size=n_particles)

    # PSO STARTS
    iteration = 1
    while iteration <=...


I am working on the file "PSPython_01-linear-PDEs.ipynb".
The first task is:
Content provided under a Creative Commons Attribution license, CC-BY 4.0; code under MIT License. (c)2015 David I. Ketcheson
Version 0.3 - May 2022
Pseudospectral methods for wave equations in Python
Welcome to PseudoSpectralPython, a short course that will teach you how to solve wave equations using pseudospectral collocation methods.  This notebook is the first lesson, on solving linear problems.  Pseudospectral methods are great for wave problems where:

The solution is smooth (no shocks)
The domain is simple; e.g., cartesian or spherical domains.  In the course, we'll focus on periodic Cartesian domains.

Table of contents:

Advection-diffusion
Approximate solution
General linear PDEs

Advection-diffusion
Let's get started!  Run the code cell below to import a bunch of Python libraries we'll use.
Can you write Python code for it?

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
from matplotlib import animation
from IPython.display import HTML


I am working on the file "PSPython_01-linear-PDEs.ipynb".
The first task is:
Next, we set a value for epsilon and  compute the solution:
Can you write Python code for it?

epsilon=0.01  # Diffusion coefficient
a = 1.0       # Advection coefficient

# Store solutions in a list for plotting later
frames = [u.copy()]

# Now we solve the problem
for n in range(1,N+1):
    t = n*k
    uhat = np.exp(-(1.j*xi*a + epsilon*xi**2)*t) * uhat0
    u = np.real(np.fft.ifft(uhat))
    frames.append(u.copy())


I am working on the file "PSPython_01-linear-PDEs.ipynb".
The first task is:
The functions $u, \hat{u}$ discussed above are replaced by finite-dimensional vectors.  These vectors are related through the discrete version of the Fourier transform, aptly called the discrete Fourier transform (DFT).  We'll look at the DFT in more detail in the next lesson.  For now, let's set the initial condition to
$$u_0(x) = \begin{cases} \sin^2(2x) & -\pi \le x < -\pi/2 \ 0 & x>-\pi/2 \end{cases}$$
and compute its DFT:
Can you write Python code for it?

# Initial data
u = np.sin(2*x)**2 * (x<-L/4)
uhat0 = np.fft.fft(u)


I am working on the file "PSPython_01-linear-PDEs.ipynb".
The first task is:
To begin, let's consider the partial differential equation (PDE)
$$u_t + a u_x = \epsilon u_{xx}$$
referred to as the advection-diffusion equation, for reasons we'll soon discover.  Here we wish to find $u(x,t)$, which might be the density or concentration of some substance.  The subscripts denote partial differentiation; e.g. $u_t$ is the partial derivative of $u$ with respect to $t$.  The coefficients $a$ and $\epsilon$ are constants that determine the strength of the advective and diffusive effects.
Exact solution by Fourier analysis
Let's solve this equation on a periodic domain $[-\pi,\pi]$, with some initial data
$$u(x,0) = u_0(x).$$
If we suppose for a moment that our solution is composed of a single Fourier mode with wavenumber $\xi$ and time-dependent amplitude $\hat{u}$:
$$u(x,t; \xi) = \hat{u}(t) e^{i\xi x},$$
Then we obtain a simple ordinary differential equation (ODE) for $\hat{u}$:
$$\hat{u}'(t; \xi) + i\xi a \hat{u} = -\xi^2 \epsilon \hat{u}$$
We can solve this scalar ODE exactly:
$$\hat{u}(t; \xi) = e^{(-i \xi a - \epsilon \xi^2)t} \hat{u}(0).$$
We've transformed the original PDE into a simple ODE, but you may wonder whether this is useful, since we assumed a very simple form for the solution.  The marvelous fact is that every solution of our advection-diffusion equation can be written as a linear combination (a superposition) of simple solutions of the form above, with different wavenumbers $\xi$.  We can construct the general solution as follows.
First, we take a Fourier transform of the initial data:
$$\hat{u}(t=0;\xi) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^\infty u_0(x) e^{-i\xi x}dx.$$
Then each mode evolves according to the solution of the ODE above:
$$\hat{u}'(t; \xi) = e^{(-i \xi a - \epsilon \xi^2)t} \hat{u}(0;\xi).$$
Finally, we construct the solution again by taking the inverse Fourier transform.  This just means summing up all the Fourier modes:
$$u(x,t) = \frac{1}{\sqrt{2\pi}} \int_{-\infty}^\infty \hat{u}(t; \xi) e^{i\xi x}d\xi.$$
If you haven't seen Fourier analysis at all...
Can you write Python code for it?

# Spatial grid
m=64                            # Number of grid points in space
L = 2 * np.pi                   # Width of spatial domain
x = np.arange(-m/2,m/2)*(L/m)   # Grid points
dx = x[1]-x[0]                  # Grid spacing

# Temporal grid
tmax = 4.0     # Final time
N = 25       # number grid points in time
k = tmax/N   # interval between output times

x


I am working on the file "PSPython_01-linear-PDEs.ipynb".
The first task is:
We have computed and stored the solution.  The code below plots it as an animation.
Can you write Python code for it?

# Set up plotting
fig = plt.figure(figsize=(9,4)); axes = fig.add_subplot(111)
line, = axes.plot([],[],lw=3)
axes.set_xlim((x[0],x[-1])); axes.set_ylim((0.,1.))
plt.close()

def plot_frame(i):
    #fig = plt.figure()
    #plt.plot(x,frames[i])
    line.set_data(x,frames[i])
    axes.set_title('t='+str(i*k))
    fig.canvas.draw()
    return fig

# Animate the solution
anim = matplotlib.animation.FuncAnimation(fig, plot_frame,
                                   frames=len(frames),
                                   interval=200,
                                   repeat=False)

HTML(anim.to_jshtml())


I am working on the file "PSPython_02-pseudospectral-collocation.ipynb".
The first task is:
Content provided under a Creative Commons Attribution license, CC-BY 4.0; code under MIT License. (c)2015 David I. Ketcheson
Version 0.3 - May 2022
Can you write Python code for it?

%matplotlib inline
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
from matplotlib import animation
from IPython.display import HTML
font = {'size'   : 15}
matplotlib.rc('font', **font)


I am working on the file "PSPython_02-pseudospectral-collocation.ipynb".
The first task is:
Exercise: absolute stability
The region of absolute stability for the midpoint method is the interval $S=[-i,i]$.  The numerical solution will be stable only if $\lambda \Delta t \in S$ for all eigenvalues $\lambda$ of $M$.

Determine the maximum stable step size.  What happens if you take a step size larger than this value?

Now, in addition to truncating the Fourier series of the initial condition, we are truncating the Taylor series of the evolution in time.  This leads to truncation error that accumulates, leading to a solution that is less accurate as time progresses.

Set $\Delta t$ equal to the largest stable step size and notice the oscillations that appear in front of the pulse as it propagates.  Try reducing the size of $\Delta t$ to decrease the amount of truncation error.  Try some different velocity functions $a(x)$.

A nonlinear wave equation
Finally, let us consider the case in which the speed of propagation is given by the solution value $u$ itself:
$$
    u_t + u u_x = 0.
$$
We will again use the pseudospectral approach to approximate the term $uu_x$, by differentiating in frequency space and then multiplying in physical space.
We can implement this easily, using the code above but replacing the entries in the matrix $A$ by the values of $u$ on the grid.  We need to construct this matrix at each time step now, since $u$ is changing in time.
This implementation is very inefficient but since we are working on a small grid in one dimension we can afford to ignore efficiency for now.  Run the code below.  Can you explain what happens?  Can you "fix" the problem that occurs by adjusting the parameters in the code?
Can you write Python code for it?

m = 64
F = F_matrix(m)
Finv = np.linalg.inv(F)
L = 2 * np.pi
xi = np.fft.fftfreq(m)*m*2*np.pi/L
D = np.diag(1.j*xi)
x = np.arange(-m/2,m/2)*(L/m)

# Initial data
u = np.sin(2*x)**2 * (x<-L/4) + 1.
dx = x[1]-x[0]
dt = 2.0/m/np.max(np.abs(a(x)))/2.
T = 5.
N = int(np.round(T/dt))

frames = [u.copy()]
skip = N//100

# Start with an explicit Euler step
A = np.diag(u)
M = -np.dot(A,np.dot(Finv,np.dot(D,F)))
u_new = u + dt*np.dot(M,u)

# Now we solve the problem
for n in range(1,N+1):
    t = n*dt
    u_old = u.copy()
    u = u_new.copy()

    A = np.diag(u)
    M = -np.dot(A,np.dot(Finv,np.dot(D,F)))
    u_new = u_old + 2*dt*np.dot(M,u)
    if ((n % skip) == 0):
        frames.append(u_new.copy())

# Set up plotting
fig, ax1 = plt.subplots(1, 1, figsize=(8,4))

line1, = ax1.plot([],[],lw=3)
ax1.set_xlim((x[0],x[-1])); ax1.set_ylim((0.,2.1))
plt.close()

def plot_frame(i):
    line1.set_data(x,frames[i])
    ax1.set_title('t='+str(i*skip*dt))

# Animate the solution
anim = matplotlib.animation.FuncAnimation(fig, plot_frame,
                                   frames=len(frames),
                                   interval=200)

HTML(anim.to_jshtml())


I am working on the file "PSPython_02-pseudospectral-collocation.ipynb".
The first task is:
Notice that $F$ is symmetric; this is true for any value of $m$.  Also, $F$ is essentially unitary -- that is, it is possible to choose a normalization such that $F$ is unitary, but the normalization in common use means that $FF^=mI$ (and $F^{-1}(F^{-1})^ = m^{-1} I$).
We won't work with this matrix in practice since applying it naively is much slower than the FFT.  But it is a useful representation.  Thus we can write our "algorithm" simply as
$$
u(t) = F^{-1} D\left[e^{(-i \xi - \epsilon \xi^2)(t-t_0)}\right] F u(t_0).
$$
Now it is completely clear that we are simply applying a similarity transformation that diagonalizes the dynamics of our system:
$$
F u(t) =  D\left[e^{(-i \xi - \epsilon \xi^2)(t-t_0)}\right] F u(t_0).
$$
We can solve any linear, scalar constant-coefficient 1D evolution equation (with periodic boundary conditions) in similar manner, by simply replacing the argument of $D$.  Note that this algorithm is:

Exact in time (the only error is due to our initial truncation of the Fourier series by sampling the data at a finite set of points); and
Unconditionally stable.

Here is the implementation as a reminder.
Can you write Python code for it?

# Spatial grid
m=64                            # Number of grid points in space
L = 2 * np.pi                   # Width of spatial domain
x = np.arange(-m/2,m/2)*(L/m)   # Grid points
dx = x[1]-x[0]                  # Grid spacing

# Temporal grid
tmax=4.0     # Final time
N = 25       # number grid points in time
k = tmax/N   # interval between output times

xi = np.fft.fftfreq(m)*m*2*np.pi/L  # Wavenumber "grid"
# (this is the order in which numpy's FFT gives the frequencies)

# Initial data
u = np.sin(2*x)**2 * (x<-L/4)
uhat0 = np.fft.fft(u)

epsilon=0.01  # Diffusion coefficient
a = 1.0       # Advection coefficient

# Store solutions in a list for plotting later
frames = [u.copy()]

# Now we solve the problem
for n in range(1,N+1):
    t = n*k
    uhat = np.exp(-(1.j*xi*a + epsilon*xi**2)*t) * uhat0
    u = np.real(np.fft.ifft(uhat))
    frames.append(u.copy())

# Set up plotting
fig = plt.figure(figsize=(9,4)); axes = fig.add_subplot(111)
line, = axes.plot([],[],lw=3)
axes.set_xlim((x[0],x[-1])); axes.set_ylim((0.,1.))
plt.close()

def plot_frame(i):
    line.set_data(x,frames[i])
    axes.set_title('t='+str(i*k))

# Animate the solution
anim = matplotlib.animation.FuncAnimation(fig, plot_frame,
                                   frames=len(frames),
                                   interval=200)

HTML(anim.to_jshtml())


I am working on the file "PSPython_02-pseudospectral-collocation.ipynb".
The first task is:
Pseudospectral collocation methods
Welcome to lesson 2 -- in which we'll learn about the main focus of this course: pseudospectral collocation methods for wave equations.  But first, a bit of review.
Advection-diffusion (again)
$$
\newcommand{\F}{\mathcal F}
\newcommand{\Finv}{{\mathcal F}^{-1}}
$$
In notebook 1 we solved the scalar, linear 1D evolution equation:
$$u_t + u_x = \epsilon u_{xx}$$
by taking the Fourier transform, which "diagonalizes" this infinite-dimensional dynamical system:
$$\hat{u}_t + i\xi\hat{u} = -\xi^2 \epsilon \hat{u}$$
with solution
$$\hat{u}(t) = e^{(-i \xi - \epsilon \xi^2)t} \hat{u}(0)$$
for each mode.
To construct the full solution, we simply take the inverse Fourier transform.  All together, this algorithm looks like:
$$
u(t) = \Finv \left(D\left[e^{(-i \xi - \epsilon \xi^2)t}\right]\F(u) \right),
$$
where we have written $D[f(\xi)]$ to denote the diagonal matrix whose $(j,j)$ entry is given by $f(\xi_j)$.
In the exact solution, the wavenumbers $\xi$ range from $-\infty$ to $+\infty$ (and $D$ is infinite), but in practice we compute on a finite interval of length $L$ with $m$ collocation points.
The wavenumbers are then given by the formula
\begin{align}
\xi_j & = \frac{2 \pi j}{L} & \text{for } -m/2 \le j < m/2,
\end{align}
although the FFT routine orders them differently.
We can make the algorithm even more explicit by recognizing that the FFT (and its inverse) is a linear map, so it can be represented as a matrix multiplication.  Thus
$$\F(u) = Fu$$
where $F$ is a certain $m \times m$ matrix.  This matrix has a number of interesting properties (see e.g. Trefethen, Ch. 3), but for the moment we are only interested in the fact that it is a linear operator.  We can reverse engineer it by applying $\F$ to the columns of the identity matrix:
Can you write Python code for it?

def F_matrix(m):
    F = np.zeros((m,m),dtype=complex)
    for j in range(m):
        v = np.zeros(m)
        v[j] = 1.
        F[:,j] = np.fft.fft(v)
    return F

print(F_matrix(4))


I am working on the file "PSPython_02-pseudospectral-collocation.ipynb".
The first task is:
We see that the eigenvalues of $M$ are purely imaginary, regardless of the choice of $a(x)$.  This is not surprising, since $M$ is the product of a real diagonal matrix $A$ and a matrix that is similar to $D$.
It is also straightforward to see that the largest eigenvalues of $M$ have magnitude equal to $\max_i |a(x_i)| m/2$.
Thus we should choose a time integrator whose absolute stability region includes part of the imaginary axis -- ideally, a large part of it.  A simple integrator of this type is the explicit midpoint method (also referred to as the leapfrog method).  For our problem it amounts to
$$
U^{n+1} = U^{n-1} + 2\Delta t M U^n.
$$
Since this is a 2-step method, we need some other way to take the first step.  For that, we use the explicit Euler method:
$$
U^{n+1} = U^n + \Delta t M U^n.
$$
Can you write Python code for it?

#a = lambda x : np.ones(len(x))
a = lambda x : 2 + np.sin(x)

m = 64
F = F_matrix(m)
Finv = np.linalg.inv(F)
L = 2 * np.pi
xi = np.fft.fftfreq(m)*m*2*np.pi/L
D = np.diag(1.j*xi)
x = np.arange(-m/2,m/2)*(L/m)
A = np.diag(a(x))
M = -np.dot(A,np.dot(Finv,np.dot(D,F)))

# Initial data
u = np.sin(2*x)**2 * (x<-L/4)
dx = x[1]-x[0]
dt = 2.0/m/np.max(np.abs(a(x)))/2.
T = 10.
N = int(np.round(T/dt))

frames = [u.copy()]
skip = N//100

# Start with an explicit Euler step
u_new = u + dt*np.dot(M,u)

# Now we solve the problem
for n in range(1,N+1):
    t = n*dt
    u_old = u.copy()
    u = u_new.copy()
    u_new = u_old + 2*dt*np.dot(M,u)
    if ((n % skip) == 0):
        frames.append(u_new.copy())

# Set up plotting
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(8,8), sharex=True)

line1, = ax1.plot([],[],lw=3)
ax1.set_xlim((x[0],x[-1])); ax1.set_ylim((-0.1,1.1))
ax2.plot(x,a(x),lw=3); ax2.set_ylim(0,3.1)
plt.close()

def plot_frame(i):
    line1.set_data(x,frames[i])
    ax1.set_title('t='+str(i*skip*dt))

# Animate the solution
anim = matplotlib.animation.FuncAnimation(fig, plot_frame,
                                   frames=len(frames),
                                   interval=200)

HTML(anim.to_jshtml())


I am working on the file "PSPython_03-FFT-aliasing-filtering.ipynb".
The first task is:
Let's check the size of the imaginary axis interval contained in this method's absolute stability region:
Can you write Python code for it?

from nodepy import rk
ssp33 = rk.loadRKM('SSP33')
print(ssp33.imaginary_stability_interval())


I am working on the file "PT3S.ipynb".
The first task is:
### weitere Beispiele für Singletests
Beispiel: durchführen eines einzelnen Tests in Modul Mx über Mx.py
das Testobjekt ist eine Klassen-freie Funktion, die kein Modell-MockUp benötigt
Can you write Python code for it?

%run Mx.py -l -q -m 0 -s getMicrosecondsFromRefTime


I am working on the file "PT3S.ipynb".
The first task is:
Abschlussarbeiten
Anleitung Doc-Gen
Version-String in conf.py and setup.py
sphinx-build -b html . _build
sphinx-build -b latex . _build
in _build:
pdflatex --extra-mem-bot=10000000 PT3S.tex
erzeugt PT3S.PDF; kopieren nach ..
Anmerkungen
z.B. PDF Doc-Gen derzeit nur um die grundsätzliche Generierbarkeit zu prüfen: d.h. ob die Kommentar- und Testbereiche redaktionell soll aufgebaut sind, dass ein Generator damit zurecht kommt; die Einstiegs-Doc ist dieses Notebook
Deploy
Can you write Python code for it?

!python setup.py clean sdist

!twine upload -u PT3S -p PythonTools3S  dist/*


I am working on the file "PT3S.ipynb".
The first task is:
Beispiel: durchführen eines einzelnen Tests in Modul Xm über Xm.py
das Testobjekt ist eine Klassen-Funktion, welche die angegebenen Modell-MockUps benötigt
Can you write Python code for it?

%run Xm.py -l -q -m 0 -s Xm\.constructShortestPathFromNodeList -t both -y yes -z no -w GPipes -w LocalHeatingNetwork


I am working on the file "PT3S.ipynb".
The first task is:
Beispiel: weitere Singletests
Can you write Python code for it?

%run Rm.py -l -q -m 0 \
-s pltMakeCategoricalColors \
-s pltMakeCategoricalCmap \
-s Rm\. \
-y no -z no \
-w DHNetwork


I am working on the file "PT3S.ipynb".
The first task is:
Bsp1: Prozentdarstellung - keine Klassen
Can you write Python code for it?

plt.close('all')
fig=plt.figure(dpi=2*72,linewidth=1.)
# 3Classes und FixedLimits sind standardmaessig Falsch; RefPerc ist standardmaessig Wahr
pFWVB=rm.pltNetDHUS(
                 pltTitle='pltNetDHUS: Bsp1: Prozentdarstellung - keine Klassen'
                ,timeDeltaToT= pd.to_timedelta('30 seconds')
                ,pFIGNrcv=['WBLZ~WärmeblnzGes~\S*~\S+~WES'
                          ,'WBLZ~WärmeblnzGes~\S*~\S+~WVB'
                          ,'KNOT~PKON-Knoten~\S*~\S+~QM'
                          ]
                 ,pFIGNrcvTxt=['Erzeugung'
                              ,'Verbrauch'
                              ,'Kontrolle DH'
                              ]

                 ,pVICsDf=pd.DataFrame({'Kundenname': ['VIC1'],'Knotenname': ['V-K007']})

                 ,CBShrink=1. # default: 0.3; ist hier wg. der ausgepraegten Querformat-Modellausdehnung zu klein
                 ,CBLabelPad=-20 # default: -50; dito zu gross

                   )
plt.show()


I am working on the file "PT3S.ipynb".
The first task is:
Bsp2: Prozentdarstellung - Klassen
Can you write Python code for it?

plt.close('all')
fig=plt.figure(dpi=2*72,linewidth=1.)
# 3Classes und FixedLimits sind standardmaessig Falsch; RefPerc ist standardmaessig Wahr
pFWVB=rm.pltNetDHUS(
                 pltTitle='pltNetDHUS: Bsp2: Prozentdarstellung - Klassen'
                ,timeDeltaToT= pd.to_timedelta('30 seconds')

                ,pFIGNrcv=['WBLZ~WärmeblnzGes~\S*~\S+~WES'
                          ,'WBLZ~WärmeblnzGes~\S*~\S+~WVB'
                          ,'KNOT~PKON-Knoten~\S*~\S+~QM'
                          ]
                 ,pFIGNrcvTxt=['Erzeugung'
                              ,'Verbrauch'
                              ,'Kontrolle DH'
                              ]
                 ,CBShrink=1. # default: 0.3; ist hier wg. der ausgepraegten Querformat-Modellausdehnung zu klein
                 ,CBLabelPad=-20 # default: -50; dito zu gross

                 ,pFWVBMeasure3Classes=True
                 ,pFWVBMeasureCBFixedLimitHigh=0.80
                 ,pFWVBMeasureCBFixedLimitLow=0.66
                   )
plt.show()


I am working on the file "PT3S.ipynb".
The first task is:
Bsp3: keine Prozentdarstellung - Klassen
Can you write Python code for it?

plt.close('all')
fig=plt.figure(dpi=2*72,linewidth=1.)
# 3Classes und FixedLimits sind standardmaessig Falsch; RefPerc ist standardmaessig Wahr
pFWVB=rm.pltNetDHUS(
                 pltTitle='pltNetDHUS: Bsp3: keine Prozentdarstellung - Klassen'
                ,pFIGNrcv=['WBLZ~WärmeblnzGes~\S*~\S+~WES'
                          ,'WBLZ~WärmeblnzGes~\S*~\S+~WVB'
                          ,'KNOT~PKON-Knoten~\S*~\S+~QM'
                          ]
                 ,pFIGNrcvTxt=['Erzeugung'
                              ,'Verbrauch'
                              ,'Kontrolle DH'
                              ]
                 ,CBShrink=1. # default: 0.3; ist hier wg. der ausgepraegten Querformat-Modellausdehnung zu klein
                 ,CBLabelPad=-20 # default: -50; dito zu gross

                 ,pFWVBMeasure3Classes=True
                 ,pFWVBMeasureInRefPerc=False
                 ,pFWVBMeasure='FWVB~*~*~*~W'

                 ,pFWVBMeasureCBFixedLimitHigh=200.
                 ,pFWVBMeasureCBFixedLimitLow=130.
                   )
plt.show()


I am working on the file "PT3S.ipynb".
The first task is:
Clean Up
Can you write Python code for it?


I am working on the file "PT3S.ipynb".
The first task is:
DH
Rename (um die Kanäle ohne die ID auswählen zu können)
Can you write Python code for it?

colDct={}
for col in tcData.columns.tolist():
    mo=re.match(Mx.reSir3sIDcompiled,col)
    colNew=mo.group('OBJTYPE')
    colNew=colNew+Mx.reSir3sIDSep+str(mo.group('NAME1'))
    colNew=colNew+Mx.reSir3sIDSep+mo.group('NAME2')
    #colNew=colNew+Mx.reSir3sIDSep+mo.group('OBJTYPE_PK')
    colNew=colNew+Mx.reSir3sIDSep+mo.group('ATTRTYPE')
    colDct[col]=colNew
df=tcData.rename(columns=colDct)

mx1DfDH=pd.concat([df['RSLW~wDH_RD_A~~XA']
                  ,df['RMES~yDH_pRL_A~~XA']
                  ,df['RSLW~wDH_MD_A~~XA']
                  ,df['RADD~yDH_pMD_A~~XA']
                  ]
                  , axis=1)

mx1DfDH.head()

plt.close()
size_DINA6quer=(5.8,4.1)
plt.rc('figure',figsize=size_DINA6quer)
fig=plt.figure()
mx1DfDH.plot()

mx1DfDHQ=pd.concat([df['RMES~QDHGes~~XA']
                  ,df['ALLG~~~LINEPACKRATE']
                  ]
                  , axis=1)

mx1DfDHQ.head()

mx1DfDHV=pd.concat([df['ALLG~~~LINEPACKGEOM']
                  ]
                  , axis=1)
mx1DfDHV=mx1DfDHV-mx1DfDHV['ALLG~~~LINEPACKGEOM'][0]
mx1DfDHV.describe()

mx1DfDHV.shape


I am working on the file "PT3S.ipynb".
The first task is:
Ergebnisse
Can you write Python code for it?

# Weg A um an Ergebnisse zu kommen:
# Modell(-Views) um MX-Ergebnisse ergaenzen; MX-Ergebnissatz des Modells wird implizit gelesen und returned
mx=xm.MxSync()

# Weg B um an Ergebnisse zu kommen:
# einen MX-Ergebnissatz voellig unabhaengig von einem Modell lesen
# z.B. den MX-Ergebnissatz des Modells ...
(wDir,modelDir,modelName,mx1File)=xm.getWDirModelDirModelName()
# ... lesen
mx=Mx.Mx(mx1File=mx1File)


I am working on the file "PT3S.ipynb".
The first task is:
Farben für Zeitkurven
Definitionen
Can you write Python code for it?

# Linienattribute für Paare von Linien
lwThick=mpl.rcParams['lines.linewidth']*2 # zuerst gezeichnete Linie (hell)
lsThin='--' # danach gezeichnete Linie (dunkel)


I am working on the file "PT3S.ipynb".
The first task is:
Festlegung der Farben
Can you write Python code for it?

# fuer A,B,C: Auswahl aus konstruierten Tönen
ntcCat=10
ntcCatSub=3
tcCm=Rm.pltMakeCategoricalCmap(catagoryColors=[idx for idx in range(ntcCat)],nOfSubCatsReq=ntcCatSub)

catA=0 # blau
catB=1 # orange
catC=2 # grün
catFromIdx={}
catFromIdx[0]=catA
catFromIdx[1]=catB
catFromIdx[2]=catC

# DH: RD,MD,VD:
tcC_SL=plt.get_cmap("tab20b").colors[3*4+2]
tcC_RL=plt.get_cmap("tab20b").colors[0*4+2]
tcC_ML=plt.get_cmap("tab20b").colors[4*4+2]

tcC_SLl=plt.get_cmap("tab20b").colors[3*4+3]
tcC_RLl=plt.get_cmap("tab20b").colors[0*4+3]
tcC_MLl=plt.get_cmap("tab20b").colors[4*4+3]

# DH: RD,MD-Paare: Sequenzen
tcC_XL=[tcC_RLl,tcC_RL,tcC_MLl,tcC_ML]
tcLs_XL=[mpl.rcParams['lines.linestyle'],lsThin,mpl.rcParams['lines.linestyle'],lsThin]
tcLw_XL=[lwThick,mpl.rcParams['lines.linewidth'],lwThick,mpl.rcParams['lines.linewidth']]

# DH: Q,Linepackrate:
tcC_QDH=plt.get_cmap("tab20b").colors[2*4+0]
tcC_QLPRate=plt.get_cmap("tab20b").colors[2*4+3]

# DH: Q,Linepackrate-Paar: Sequenz
tcC_DH=[tcC_QDH,tcC_QLPRate]
tcLs_DH=[mpl.rcParams['lines.linestyle'],lsThin]
tcLw_DH=[lwThick,mpl.rcParams['lines.linewidth']]

# DH: V
tcC_VDH=plt.get_cmap("tab20b").colors[1*4+2]


I am working on the file "PT3S.ipynb".
The first task is:
Fwes W
Can you write Python code for it?

mx1DfFwesW=tcData.filter(regex='^FWES').filter(regex='W$')
mx1DfFwesW=mx1DfFwesW.reindex(sorted(mx1DfFwesW.columns),axis=1)

mx1DfFwesW.head()

plt.close()
size_DINA6quer=(5.8,4.1)
plt.rc('figure',figsize=size_DINA6quer)
fig=plt.figure()
mx1DfFwesW.plot()

timeSumMaxMx1DfFwesW=mx1DfFwesW.sum(axis=1).idxmax()
timeSumMaxMx1DfFwesW

timeSumMinMx1DfFwesW=mx1DfFwesW.sum(axis=1).idxmin()
timeSumMinMx1DfFwesW


I am working on the file "PT3S.ipynb".
The first task is:
Install PT3S to site-packages
Can you write Python code for it?

### ggf. Rechte erforderlich:
### entweder in PowerShell: Start-Process powershell -Verb runAs
### oder RechteMausTaste WindowsSymbol: Windows PowerShell (Administrator)
### dann (ohne ! in PowerShell auf pip-Verz.):

#!pip uninstall --yes PT3S
#!pip install PT3S --no-cache-dir


I am working on the file "PT3S.ipynb".
The first task is:
Logging
Can you write Python code for it?

import logging
import os
logger = logging.getLogger('PT3S')

logFileName= r"PT3S.log"

loglevel = logging.DEBUG
logging.basicConfig(filename=logFileName
                        ,filemode='w'
                        ,level=loglevel
                        ,format="%(asctime)s ; %(name)-60s ; %(levelname)-7s ; %(message)s")

fileHandler = logging.FileHandler(logFileName)

logger.addHandler(fileHandler)

consoleHandler = logging.StreamHandler()
consoleHandler.setFormatter(logging.Formatter("%(levelname)-7s ; %(message)s"))
consoleHandler.setLevel(logging.INFO)
logger.addHandler(consoleHandler)


I am working on the file "PT3S.ipynb".
The first task is:
Modell
Can you write Python code for it?

path='.'
xmlFile=os.path.join(path,'testdata\LocalHeatingNetwork.XML')
xm=Xm.Xm(xmlFile=xmlFile)

vVBEL=xm.dataFrames['vVBEL']
vVBEL.filter(items=['BESCHREIBUNG','NAME_i','NAME_k','LAYR','L','D']).sort_index(level=1)

vVBEL.dtypes

[viewOrTable for viewOrTable in sorted(xm.dataFrames.keys())]


I am working on the file "PT3S.ipynb".
The first task is:
Modell und Ergebnis "synchronisieren"

Bezuege herstellen zwischen einem Modell und einem Ergebnissatz (Stichworte mx2Idx, NAME123)
mx=xm.MxSync() synchronisiert implizit mit dem MX-Ergebnissatz des Modells (der MX-Satz wir dabei gelesen)
xm.MxSync(mx=mx): explizite Synchronisation mit irgendeinem bereits gelesenen MX-Ergebnissatz
Can you write Python code for it?

xm.MxSync(mx=mx)

vROHR=xm.dataFrames['vROHR']

vROHR[['L','mx2NofPts','mx2Idx']]


I am working on the file "PT3S.ipynb".
The first task is:
NetworkX Beispiele
hydraulisches Prozessmodell
Can you write Python code for it?

G=nx.from_pandas_edgelist(xm.dataFrames['vVBEL'], source='NAME_i', target='NAME_k', edge_attr=True,create_using=nx.MultiGraph())

for e, datadict in G.edges.items():
    print(e)
    print(datadict)

for n, nbrsdict in G.adj.items():
    print("!{0:s}".format(n))
    for nox, mgdct in nbrsdict.items():
        print("   {0:s}".format(nox))
        for mg,edct in  mgdct.items():
            print("        {0:d}: {1:s}".format(mg,str(edct)))

print(nx.dijkstra_path(G, 'V-L', 'R-L'))

max([d for n,d in nx.degree(G)])

spmtx=nx.adjacency_matrix(G) # Return type: SciPy sparse matrix
plt.spy(spmtx)


I am working on the file "PT3S.ipynb".
The first task is:
Plot
Can you write Python code for it?

# 2 Zeiten auswählen
time1=timeSumMaxMx1DfFwesW
time2=timeSumMinMx1DfFwesW

# Ergebnisse für die 2 Zeiten holen
vAGSN_T1,vKNOT_T1,vROHR_T1,vFWVB_T1,vVBEL_T1,hpSL_T1,hpRL_T1=mxAdd(xm,mx,timeReq=time1)
vAGSN_T2,vKNOT_T2,vROHR_T2,vFWVB_T2,vVBEL_T2,hpSL_T2,hpRL_T2=mxAdd(xm,mx,timeReq=time2)

# Ref/Cmp zuordnen
timeRef=time1
timeCmp=time2

hpSLRef=hpSL_T1
hpRLRef=hpRL_T1

hpSLCmp=hpSL_T2
hpRLCmp=hpRL_T2

vROHRRef=vROHR_T1
vROHRCmp=vROHR_T2

vROHR_NFD=pd.merge(vROHRRef
         ,hpRLRef[hpRLRef.IptIdx=='S']
         ,how='left'
         ,left_on='pk'
         ,right_on='OBJID'
         ,suffixes=('','_AGSN')).filter(items=vROHRRef.columns.tolist()+['OBJID'])

plt.close()

size_DINA3quer=(16.5, 11.7)
plt.rc('figure',figsize=size_DINA3quer)
plt.rc('figure',dpi=72)
plt.rc('savefig',dpi=72*2)
fig=plt.figure()

# .............................................................
# Paramtrierung
# .............................................................
# links: 1 NFD und 1 NDD
# .............................................................
# links nimmt die ganz linke Hälfte des Dashboards ein
leftTileRight=0.5

# rechts: 3 ZKs
# .............................................................
# rechts nimmt die ganz linke Hälfte des Dashboards ein
rightTileStart=1-leftTileRight
# allerdings wird Platz für die y-Achsen benötigt
rightTileYAxisXSpace=0.125
rightTileStart=rightTileStart+rightTileYAxisXSpace
rightTileH_pad=0.5

# x-Achse
#majLocator=mdates.MinuteLocator(interval=5)
majLocator=mdates.MinuteLocator(byminute=[0,5,10,15,20,25,30,35,40,45,50,55])
majFormatter=mdates.DateFormatter('%d.%m.%y: %H:%M')

# .............................................................
# links: 1 NFD und 1 NDD
# .............................................................
gs1 = gridspec.GridSpec(2, 1)
axNfd = fig.add_subplot(gs1[0])
axHp = fig.add_subplot(gs1[1])
gs1.tight_layout(fig, rect=[0, 0, leftTileRight, 1])
    #rect : if rect is given, it is interpreted as a rectangle
    #(left, bottom, right, top) in the normalized figure...


I am working on the file "PT3S.ipynb".
The first task is:
Pump Nw
Can you write Python code for it?

mx1DfPumpWCols=[col for col in mx.df.columns.tolist()
if
re.match(Mx.reSir3sIDcompiled,col).group('NAME1') in ['wNA','wNB','wNC']
and
re.match(Mx.reSir3sIDcompiled,col).group('ATTRTYPE') in ['XA']
and
re.match(Mx.reSir3sIDcompiled,col).group('OBJTYPE') in ['RSLW']
]

mx1DfPumpNw=tcData.filter(items=mx1DfPumpWCols)
mx1DfPumpNw=mx1DfPumpNw.reindex(sorted(mx1DfPumpNw.columns),axis=1)

mx1DfPumpNw.head()


I am working on the file "PT3S.ipynb".
The first task is:
Pump Ny
Can you write Python code for it?


I am working on the file "PT3S.ipynb".
The first task is:
Pump Nyw-Pairs
Can you write Python code for it?

mx1DfPumpNywPairs=pd.concat([mx1DfPumpNy,mx1DfPumpNw],axis=1)

mx1DfPumpNywPairs=mx1DfPumpNywPairs.filter(items=list(chain.from_iterable(
    [x for x in zip(mx1DfPumpNy.columns.tolist(),mx1DfPumpNw.columns.tolist())]
                                                                         )
                                                     )
                                          )

mx1DfPumpNywPairs.head()

plt.close()
size_DINA6quer=(5.8,4.1)
plt.rc('figure',figsize=size_DINA6quer)
fig=plt.figure()
mx1DfPumpNywPairs.plot()


I am working on the file "PT3S.ipynb".
The first task is:
Pump Q
Can you write Python code for it?

mx1DfPumpQ=tcData.filter(regex='^PUMP').filter(regex='QM$')
mx1DfPumpQ=mx1DfPumpQ.reindex(sorted(mx1DfPumpQ.columns),axis=1)
mx1DfPumpQ.head()

plt.close()
size_DINA6quer=(5.8,4.1)
plt.rc('figure',figsize=size_DINA6quer)
fig=plt.figure()
mx1DfPumpQ.plot()


I am working on the file "PT3S.ipynb".
The first task is:
Rm Plot Beispiele
Can you write Python code for it?

dir(Rm)

import Rm

dir(Rm)

rm=Rm.Rm(xm=xm,mx=mx)


I am working on the file "PT3S.ipynb".
The first task is:
Rohrvektorergebnisse
Can you write Python code for it?


I am working on the file "PT3S.ipynb".
The first task is:
Singeltests

es müssen nur die Modelle (-w ...) behandelt werden, die in den Singletests auch verwendet werden
vor Singletests könnten die Modultests durchgeführt werden: nachfolgend sind diese ausgeschaltet (-m 0)
Singletests hinterlassen, so sie nicht selbst aufräumen, "Überbleibsel", die explizit gelöscht werden sollten (-t after)
Singletests erfahren im Gegensatz zu Modultests ein explizites MockUp außerhalb des eigentlichen Tests; d.h. die Tests beginnen mit "fertigen" Objekten bzw. können zumindest so beginnen; bei Modultests hingegen erfolgt das MockUp immer in der Testsequenz selbst
Details zum MockUp können mit -y (kein H5 lesen j/n) und -z (H5 schreiben j/n) angegeben werden
es können nur die Singletest auf einmal ausgeführt werden, die mit demselben MockUp zurecht kommen
MockUp hier: kein H5 lesen und kein H5-Schreiben: -y yes -z no
nur bei -z yes wird vor dem H5-Schreiben auch Xm.MxSync und .MxAdd durchgeführt
-s spezifiziert Tests per regExp
-x spezifiziert Test per regExp die aus der -s-Menge nicht durchzuführen sind

nachfolgende Tests erzeugen (wg. -q) nur bei fehlerfreiem Durchlauf _keine Ausgaben im Notebook
Can you write Python code for it?

%run test.py -l -q -m 0 \
-s Xm\. \
-x Xm\.vKNOTexpEBES \
-x Xm\.vROHRexpEBES \
-x Xm\._vRUES -x Xm\._vRSLW -x Xm\._vRSTN -x Xm\._vRXXX -x Xm\._vREdges \
-x Xm\.MxAdd \
-t both -y yes -z no \
-w OneLPipe -w LocalHeatingNetwork -w GPipe -w GPipes -w TinyWDN

%run test.py -l -q -m 0 \
-s Mx\. \
-x Mx\.FromH5 \
-x Mx\.ToH5 \
-t both -y yes -z no \
-w OneLPipe -w LocalHeatingNetwork -w GPipe -w GPipes -w TinyWDN