I am working on the file "FAFT_64-points_R2C/nbFAFT128_2D.ipynb".
The first task is:
Forward Transform
Can you write Python code for it?

# Executing FFT

cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes1, axes1, makeR2C, axesSplit_1 )
cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes0, axes0, makeC2C, axesSplit_1 )

plt.imshow( np.append( f_gpu.get()[:, :32], f33_gpu.get().real, axis=1 )/float(np.sqrt(size)),
           extent=[-p_amplitude , 0, -p_amplitude, p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -3.0, 6.2, '$Re \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-p_amplitude , 0)
plt.ylim(-p_amplitude , p_amplitude-dp)

plt.imshow( np.append( f_gpu.get()[:, 32:], f33_gpu.get().imag, axis=1 )/float(np.sqrt(size)),
           extent=[-p_amplitude , 0, -p_amplitude, p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -3.0, 6.2, '$Im \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-p_amplitude , 0)
plt.ylim(-p_amplitude , p_amplitude-dp)


# Executing FFT

cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes0, axes0, makeR2C, axesSplit_0 )
cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes1, axes1, makeC2C, axesSplit_0 )

plt.imshow( np.append( f_gpu.get()[:32, :], f33_gpu.get().real, axis=0 )/float(np.sqrt(size)),
           extent=[-p_amplitude , p_amplitude-dp, 0, p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( 0., 6.2, '$Re \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-p_amplitude , p_amplitude-dp)
plt.ylim(0 , p_amplitude)

plt.imshow( np.append( f_gpu.get()[32:, :], f33_gpu.get().imag, axis=0 )/float(np.sqrt(size)),
           extent=[-p_amplitude , p_amplitude-dp, 0, p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( 0., 6.2, '$Im \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-p_amplitude , p_amplitude-dp)
plt.ylim(0 , p_amplitude)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_2D.ipynb".
The first task is:
Initializing Data
Gaussian
Can you write Python code for it?

def Gaussian(x,sigma):
    return np.exp( - x**2/sigma**2/2.  )/(sigma*np.sqrt( 2*np.pi  ))

def fftGaussian(p,sigma):
    return np.exp( - p**2*sigma**2/2.  )

# Gaussian parameters
mu = 0
sigma = 1.

# Grid parameters
x_amplitude = 5.
p_amplitude = 6.                # With the traditional method p amplitude is fixed to: 2 * np.pi /( 2*x_amplitude )

dx = 2*x_amplitude/float(gridDIM)  # This is dx in Bailey's paper
dp = 2*p_amplitude/float(gridDIM)  # This is gamma in Bailey's paper

delta = dx*dp/(2*np.pi)

x_range = np.linspace( -x_amplitude, x_amplitude-dx, gridDIM)
p = np.linspace( -p_amplitude, p_amplitude-dp, gridDIM)

x = x_range[ np.newaxis, : ]
y = x_range[ :, np.newaxis ]

f =  Gaussian(x,sigma)*Gaussian(y,sigma)

plt.imshow( f, extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

axis_font = {'size':'24'}
plt.text( 0., 5.1, '$W$' , **axis_font)
plt.colorbar()

#plt.ylim(0,0.44)

print ' Amplitude x = ',x_amplitude
print ' Amplitude p = ',p_amplitude
print '        '

print 'sigma = ', sigma
print 'n     = ', x.size
print 'dx    = ', dx
print 'dp    = ', dp
print '           standard fft dp = ',2 * np.pi /( 2*x_amplitude ) , '     '
print '    '
print 'delta = ', delta

print '    '

print 'The Gaussian extends to the numerical error in single precision:'
print '    min = ', np.min(f)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_2D.ipynb".
The first task is:
Inverse Transform
Can you write Python code for it?

# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2C, axesSplit_1 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2R, axesSplit_1 )

plt.imshow( f_gpu.get()/float(size)  ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.figure(figsize=(10,10))

plt.plot( y, f_gpu.get()[32, :]/(f_gpu.get().size) /(43.8*f_gpu.get().size),
         'o', label='numerical')

plt.plot( y, 4*Gaussian(y, sigma) , label = 'analytical')

plt.legend(loc='upper left')

#plt.ylim(0,1.1)

plt.ylabel('$e^{- \\frac{\\sigma x^2}{2} }$',**axis_font)
plt.xlabel('$p$',**axis_font)

# LOG PLOT

plt.figure(figsize=(10,10))

plt.semilogy( y, f_gpu.get()[32, :]/(f_gpu.get().size) /(450*f_gpu.get().size)  , 'o', label='numerical')

plt.semilogy( y, f[32, :]  , '.', label='original')

plt.semilogy( y, Gaussian(y, sigma)/2.5 , label = 'analytical')

plt.legend(loc='upper left')

plt.ylim(1e-7,0.5)

plt.ylabel('$e^{- \\frac{x^2}{2 \\sigma^2 } }$',**axis_font)
plt.xlabel('$p$',**axis_font)


# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2C, axesSplit_0 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2R, axesSplit_0 )

plt.imshow( f_gpu.get()/(float(size*size))  ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude, x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 6.2, '$W$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.figure(figsize=(10,10))

plt.plot( y, f_gpu.get()[:, 32]/(f_gpu.get().size) /(43.8*f_gpu.get().size),
         'o', label='numerical')

plt.plot( y, 4*Gaussian(y, sigma) , label = 'analytical')

plt.legend(loc='upper left')

#plt.ylim(0,1.1)

plt.ylabel('$e^{- \\frac{\\sigma x^2}{2} }$',**axis_font)
plt.xlabel('$p$',**axis_font)

# LOG PLOT

plt.figure(figsize=(10,10))

plt.semilogy( y, f_gpu.get()[:, 32]/(f_gpu.get().size) /(450*f_gpu.get().size)  , 'o', label='numerical')

plt.semilogy( y, f[:, 32]  , '.', label='original')

plt.semilogy( y, Gaussian(y, sigma)/2.5 , label = 'analytical')

plt.legend(loc='upper left')

plt.ylim(1e-7,0.5)

plt.ylabel('$e^{- \\frac{x^2}{2 \\sigma^2 } }$',**axis_font)
plt.xlabel('$p$',**axis_font)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_2D.ipynb".
The first task is:
Loading FFT routines
Can you write Python code for it?

gridDIM = 64

size = gridDIM*gridDIM

axes0 = 0
axes1 = 1

makeC2C = 0
makeR2C = 1
makeC2R = 1

axesSplit_0 = 0
axesSplit_1 = 1

m = size

segment_axes0 = 0
segment_axes1 = 0

DIR_BASE = "/home/robert/Documents/new1/FFT/code/"

# FAFT
_faft128_2D = ctypes.cdll.LoadLibrary( DIR_BASE+'FAFT128_2D_R2C.so' )
_faft128_2D.FAFT128_2D_R2C.restype = int
_faft128_2D.FAFT128_2D_R2C.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                         ctypes.c_float, ctypes.c_float, ctypes.c_int,
                                         ctypes.c_int, ctypes.c_int, ctypes.c_int]

cuda_faft = _faft128_2D.FAFT128_2D_R2C

# Inv FAFT
_ifaft128_2D = ctypes.cdll.LoadLibrary( DIR_BASE+'IFAFT128_2D_C2R.so' )
_ifaft128_2D.IFAFT128_2D_C2R.restype = int
_ifaft128_2D.IFAFT128_2D_C2R.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                           ctypes.c_float, ctypes.c_float, ctypes.c_int,
                                           ctypes.c_int, ctypes.c_int, ctypes.c_int]

cuda_ifaft = _ifaft128_2D.IFAFT128_2D_C2R


I am working on the file "FAFT_64-points_R2C/nbFAFT128_3D.ipynb".
The first task is:
$W$ TRANSFORM FROM AXES-1
After the transfom, f_gpu[:, :, :64] contains real values and f_gpu[:, :, 64:] contains imaginary values. f33_gpu contains the 33th. complex values
Can you write Python code for it?

# Matrix for the 33th. complex values

f33 = np.zeros( [64, 64, 1], dtype = np.complex64 )

# One gpu array.

if 'f_gpu' in globals():
    f_gpu.gpudata.free()

if 'f33_gpu' in globals():
    f33_gpu.gpudata.free()

f_gpu = gpuarray.to_gpu( np.ascontiguousarray( f , dtype = np.float32 ) )
f33_gpu = gpuarray.to_gpu( np.ascontiguousarray( f33 , dtype = np.complex64 ) )


I am working on the file "FAFT_64-points_R2C/nbFAFT128_3D.ipynb".
The first task is:
$W$ TRANSFORM FROM AXES-2
After the transfom, f_gpu[:64, :, :] contains real values and f_gpu[64:, :, :] contains imaginary values. f33_gpu contains the 33th. complex values
Can you write Python code for it?

# Matrix for the 33th. complex values

f33 = np.zeros( [1, 64, 64], dtype = np.complex64 )

# One gpu array.

if 'f_gpu' in globals():
    f_gpu.gpudata.free()

if 'f33_gpu' in globals():
    f33_gpu.gpudata.free()

f_gpu = gpuarray.to_gpu( np.ascontiguousarray( f , dtype = np.float32 ) )
f33_gpu = gpuarray.to_gpu( np.ascontiguousarray( f33 , dtype = np.complex64 ) )


I am working on the file "FAFT_64-points_R2C/nbFAFT128_3D.ipynb".
The first task is:
Forward Transform
Can you write Python code for it?

# Executing FFT

t_init = time.time()

cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes1, axes1, makeR2C, axesSplit_1 )
cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes0, axes0, makeC2C, axesSplit_1 )
cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes2, axes2, makeC2C, axesSplit_1 )

t_end = time.time()

print 'computation time = ', t_end - t_init

plt.imshow( np.append( f_gpu.get()[:, :, :32], f33_gpu.get().real, axis=2 )[32,:,:]
           /float(np.sqrt(size)),
           extent=[-p_amplitude , 0, -p_amplitude  , p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( 0., 5.2, '$Re \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-x_amplitude , 0)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( np.append( f_gpu.get()[:, :, 32:], f33_gpu.get().imag, axis=2 )[32,:,:]
           /float(np.sqrt(size)),
           extent=[-p_amplitude , 0, -p_amplitude  , p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( 0., 5.2, '$Im \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-x_amplitude , 0)
plt.ylim(-x_amplitude  , x_amplitude-dx)


# Executing FFT

t_init = time.time()

cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes2, axes2, makeR2C, axesSplit_2 )
cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes1, axes1, makeC2C, axesSplit_2 )
cuda_faft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, delta, segment_axes0, axes0, makeC2C, axesSplit_2 )

t_end = time.time()

print 'computation time = ', t_end - t_init

plt.imshow( np.append( f_gpu.get()[:32, :, :], f33_gpu.get().real, axis=0 )[:,:,32]
           /float(np.sqrt(size)),
           extent=[-p_amplitude , p_amplitude-dp, 0, p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( 0., 5.2, '$Re \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(0 , x_amplitude-dx)

plt.imshow( np.append( f_gpu.get()[32:, :, :], f33_gpu.get().imag, axis=0 )[:,:,32]
           /float(np.sqrt(size)),
           extent=[-p_amplitude , p_amplitude-dp, 0, p_amplitude-dp] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( 0., 5.2, '$Im \\mathcal{F}(W)$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(0 , x_amplitude-dx)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_3D.ipynb".
The first task is:
Initializing Data
Gaussian
Can you write Python code for it?

def Gaussian(x,sigma):
    return np.exp( - x**2/sigma**2/2.  )/(sigma*np.sqrt( 2*np.pi  ))

def fftGaussian(p,sigma):
    return np.exp( - p**2*sigma**2/2.  )

# Gaussian parameters
mu = 0
sigma = 1.

# Grid parameters
x_amplitude = 5.
p_amplitude = 6.                # With the traditional method p amplitude is fixed to: 2 * np.pi /( 2*x_amplitude )

dx = 2*x_amplitude/float(gridDIM)  # This is dx in Bailey's paper
dp = 2*p_amplitude/float(gridDIM)  # This is gamma in Bailey's paper

delta = dx*dp/(2*np.pi)

x_range = np.linspace( -x_amplitude, x_amplitude-dx, gridDIM)
p = np.linspace( -p_amplitude, p_amplitude-dp, gridDIM)

x = x_range[ np.newaxis, np.newaxis, : ]
y = x_range[ np.newaxis, :, np.newaxis ]
z = x_range[ :, np.newaxis, np.newaxis ]

f =  Gaussian(x,sigma)*Gaussian(y,sigma)*Gaussian(z,sigma)

plt.imshow( f[:, :, 0], extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

axis_font = {'size':'24'}
plt.text( 0., 5.1, '$W$' , **axis_font)
plt.colorbar()

#plt.ylim(0,0.44)

print ' Amplitude x = ',x_amplitude
print ' Amplitude p = ',p_amplitude
print '        '

print 'sigma = ', sigma
print 'n     = ', x.size
print 'dx    = ', dx
print 'dp    = ', dp
print '           standard fft dp = ',2 * np.pi /( 2*x_amplitude ) , '     '
print '    '
print 'delta = ', delta

print '    '

print 'The Gaussian extends to the numerical error in single precision:'
print '    min = ', np.min(f)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_3D.ipynb".
The first task is:
Inverse Transform
Can you write Python code for it?

# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes2, axes2, makeC2C, axesSplit_0 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2C, axesSplit_0 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2R, axesSplit_0 )

plt.imshow( f_gpu.get()[32,:,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)


# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes2, axes2, makeC2C, axesSplit_1 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2C, axesSplit_1 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2R, axesSplit_1 )

plt.imshow( f_gpu.get()[32,:,:] /float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xy_2D.ipynb".
The first task is:
$W$ TRANSFORM FROM AXES-0
After the transfom, f_gpu[:32, :] contains real values and f_gpu[32:, :] contains imaginary values. g33_gpu contains the 33th. complex values
Can you write Python code for it?

f33 = np.zeros( [1 ,64], dtype = np.complex64 )

# One gpu array.

f_gpu = gpuarray.to_gpu( np.ascontiguousarray( f , dtype = np.float32 ) )
f33_gpu = gpuarray.to_gpu( np.ascontiguousarray( f33 , dtype = np.complex64 ) )


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xy_2D.ipynb".
The first task is:
$W$ TRANSFORM FROM AXES-1
After the transfom, f_gpu[:, :32] contains real values and f_gpu[:, 32:] contains imaginary values. f33_gpu contains the 33th. complex values
Can you write Python code for it?

f =  Gaussian(x,mu_x,sigma_x)*Gaussian(y,mu_y,sigma_y)

plt.imshow( f, extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] , origin='lower')

f33 = np.zeros( [64, 1], dtype = np.complex64 )

# One gpu array.

f_gpu = gpuarray.to_gpu( np.ascontiguousarray( f , dtype = np.float32 ) )
f33_gpu = gpuarray.to_gpu( np.ascontiguousarray( f33 , dtype = np.complex64 ) )


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xy_2D.ipynb".
The first task is:
Central Section: $p_y =0$
Can you write Python code for it?

plt.figure(figsize=(10,10))

plt.plot( p_range,
    ReconstructFFT2D_axesSplit_0( f_gpu.get() , f33_gpu.get()   )[32,:].real/float(size),
'o-' , label='Real')

plt.plot( p_range,
    ReconstructFFT2D_axesSplit_0( f_gpu.get() , f33_gpu.get()   )[32,:].imag/float(size),
'ro-' , label='Imag')

plt.xlabel('$p_x$',**axis_font)

plt.plot( p_range  ,  4*fftGaussian(p_range,mu_x,sigma_x).real ,'bx');
plt.plot( p_range  ,  4*fftGaussian(p_range,mu_x,sigma_x).imag ,'rx');

plt.legend(loc='upper left')


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xy_2D.ipynb".
The first task is:
Inverse Transform
Can you write Python code for it?

# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2C, axesSplit_0 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2R, axesSplit_0 )

plt.imshow( f_gpu.get()/(float(size*size))  ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude, x_amplitude-dx], origin='lower' )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 7.2, '$W$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.xlabel('$x$',**axis_font)
plt.ylabel('$y$',**axis_font)


# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2C, axesSplit_1 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2R, axesSplit_1 )

plt.imshow( f_gpu.get()/float(size)**2 ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] , origin='lower')

axis_font = {'size':'24'}
plt.text( 0., 7.1, '$W$' , **axis_font)
plt.colorbar()


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xy_2D.ipynb".
The first task is:
Loading FFT routines
Can you write Python code for it?

gridDIM = 64

size = gridDIM*gridDIM

axes0 = 0
axes1 = 1

makeC2C = 0
makeR2C = 1
makeC2R = 1

axesSplit_0 = 0
axesSplit_1 = 1

m = size

segment_axes0 = 0
segment_axes1 = 0

DIR_BASE = "/home/robert/Documents/new1/FFT/mycode/"

# FAFT
_faft128_2D = ctypes.cdll.LoadLibrary( DIR_BASE+'FAFT128_2D_R2C.so' )
_faft128_2D.FAFT128_2D_R2C.restype = int
_faft128_2D.FAFT128_2D_R2C.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                         ctypes.c_float, ctypes.c_float, ctypes.c_int,
                                         ctypes.c_int, ctypes.c_int, ctypes.c_int]

cuda_faft = _faft128_2D.FAFT128_2D_R2C

# Inv FAFT
_ifaft128_2D = ctypes.cdll.LoadLibrary( DIR_BASE+'IFAFT128_2D_C2R.so' )
_ifaft128_2D.IFAFT128_2D_C2R.restype = int
_ifaft128_2D.IFAFT128_2D_C2R.argtypes = [ctypes.c_void_p, ctypes.c_void_p,
                                           ctypes.c_float, ctypes.c_float, ctypes.c_int,
                                           ctypes.c_int, ctypes.c_int, ctypes.c_int]

cuda_ifaft = _ifaft128_2D.IFAFT128_2D_C2R

def fftGaussian(p,sigma):
    return np.exp( - p**2*sigma**2/2.  )


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xyz_3D.ipynb".
The first task is:
$W$ TRANSFORM FROM AXES-1
After the transfom, f_gpu[:, :, :64] contains real values and f_gpu[:, :, 64:] contains imaginary values. f33_gpu contains the 33th. complex values
Can you write Python code for it?


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xyz_3D.ipynb".
The first task is:
$W$ TRANSFORM FROM AXES-2
After the transfom, f_gpu[:64, :, :] contains real values and f_gpu[64:, :, :] contains imaginary values. f33_gpu contains the 33th. complex values
Can you write Python code for it?


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xyz_3D.ipynb".
The first task is:
Forward Transform
Can you write Python code for it?


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xyz_3D.ipynb".
The first task is:
Initializing Data
Gaussian
Can you write Python code for it?

def Gaussian(x,mu,sigma):
    return np.exp( - (x-mu)**2/sigma**2/2.  )/(sigma*np.sqrt( 2*np.pi  ))

def fftGaussian(p,mu,sigma):
    return np.exp(-1j*mu*p)*np.exp( - p**2*sigma**2/2.  )

# Gaussian parameters
mu_x = 1.5
sigma_x = 1.

mu_y = 1.5
sigma_y = 1.

mu_z = 1.5
sigma_z = 1.

# Grid parameters
x_amplitude = 5.
p_amplitude = 6.                # With the traditional method p amplitude is fixed to: 2 * np.pi /( 2*x_amplitude )

dx = 2*x_amplitude/float(gridDIM)  # This is dx in Bailey's paper
dp = 2*p_amplitude/float(gridDIM)  # This is gamma in Bailey's paper

delta = dx*dp/(2*np.pi)

x_range = np.linspace( -x_amplitude, x_amplitude-dx, gridDIM)
p = np.linspace( -p_amplitude, p_amplitude-dp, gridDIM)

x = x_range[ np.newaxis, np.newaxis, : ]
y = x_range[ np.newaxis, :, np.newaxis ]
z = x_range[ :, np.newaxis, np.newaxis ]

f =  Gaussian(x,mu_x,sigma_x)*Gaussian(y,mu_y,sigma_y)*Gaussian(z,mu_z,sigma_z)

plt.imshow( f[:, :, 0], extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

axis_font = {'size':'24'}
plt.text( 0., 5.1, '$W$' , **axis_font)
plt.colorbar()

#plt.ylim(0,0.44)

print ' Amplitude x = ',x_amplitude
print ' Amplitude p = ',p_amplitude
print '        '

print 'mu_x = ', mu_x
print 'mu_y = ', mu_y
print 'mu_z = ', mu_z
print 'sigma_x = ', sigma_x
print 'sigma_y = ', sigma_y
print 'sigma_z = ', sigma_z
print '   '

print 'n     = ', x.size
print 'dx    = ', dx
print 'dp    = ', dp
print '           standard fft dp = ',2 * np.pi /( 2*x_amplitude ) , '     '
print '    '
print 'delta = ', delta

print '    '

print 'The Gaussian extends to the numerical error in single precision:'
print '    min = ', np.min(f)


I am working on the file "FAFT_64-points_R2C/nbFAFT128_offset_xyz_3D.ipynb".
The first task is:
Inverse Transform
Can you write Python code for it?

# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes2, axes2, makeC2C, axesSplit_0 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2C, axesSplit_0 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2R, axesSplit_0 )

plt.imshow( f_gpu.get()[32,:,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{xy}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( f_gpu.get()[:,32,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{xz}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( f_gpu.get()[:,:,32]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{yz}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)


# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes2, axes2, makeC2C, axesSplit_1 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2C, axesSplit_1 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2R, axesSplit_1 )

plt.imshow( f_gpu.get()[32,:,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{xy}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( f_gpu.get()[:,32,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{xz}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( f_gpu.get()[:,:,32]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{yz}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)


# Executing iFFT

cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes0, axes0, makeC2C, axesSplit_2 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes1, axes1, makeC2C, axesSplit_2 )
cuda_ifaft( int(f_gpu.gpudata), int(f33_gpu.gpudata), dx, -delta, segment_axes2, axes2, makeC2R, axesSplit_2 )

plt.imshow( f_gpu.get()[32,:,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{xy}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( f_gpu.get()[:,32,:]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{xz}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)

plt.imshow( f_gpu.get()[:,:,32]/float(size) ,
           extent=[-x_amplitude , x_amplitude-dx, -x_amplitude , x_amplitude-dx] )

plt.colorbar()

axis_font = {'size':'24'}
plt.text( -1, 5.2, '$W_{yz}$',  **axis_font )

plt.xlim(-x_amplitude , x_amplitude-dx)
plt.ylim(-x_amplitude , x_amplitude-dx)


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
Create a batch of some real and some generated data, with appropriate labels, for the discriminator.
Can you write Python code for it?

def data_D(sz, G):
    real_img = X_train[np.random.randint(0,n,size=sz)]
    X = np.concatenate((real_img, G.predict(noise(sz))))
    return X, [0]*sz + [1]*sz

def make_trainable(net, val):
    net.trainable = val
    for l in net.layers: l.trainable = val


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
Create some random data for the generator.
Can you write Python code for it?

def noise(bs): return np.random.rand(bs,100)


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
MLP GAN
We'll keep thinks simple by making D & G plain ole' MLPs.
Can you write Python code for it?

MLP_G = Sequential([
    Dense(200, input_shape=(100,), activation='relu'),
    Dense(400, activation='relu'),
    Dense(784, activation='sigmoid'),
])

MLP_D = Sequential([
    Dense(300, input_shape=(784,), activation='relu'),
    Dense(300, activation='relu'),
    Dense(1, activation='sigmoid'),
])
MLP_D.compile(Adam(1e-4), "binary_crossentropy")

MLP_m = Sequential([MLP_G,MLP_D])
MLP_m.compile(Adam(1e-4), "binary_crossentropy")

dl,gl = train(MLP_D, MLP_G, 4000)


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
OK, so that didn't work. Can we do better?...
DCGAN
There's lots of ideas out there to make GANs train better, since they are notoriously painful to get working. The paper introducing DCGANs is the main basis for our next section. Add see https://github.com/soumith/ganhacks for many tips!
Because we're using a CNN from now on, we'll reshape our digits into proper images.
Can you write Python code for it?

X_train = X_train.reshape(n, 28, 28, 1)
X_test = X_test.reshape(len(X_test), 28, 28, 1)


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
Our generator uses a number of upsampling steps as suggested in the above papers. We use nearest neighbor upsampling rather than fractionally strided convolutions, as discussed in our style transfer notebook.
Can you write Python code for it?

CNN_G = Sequential([
    Dense(512*7*7, input_dim=100, activation=LeakyReLU()),
    BatchNormalization(mode=2),
    Reshape((7, 7, 512)),
    UpSampling2D(),
    Convolution2D(64, 3, 3, border_mode='same', activation=LeakyReLU()),
    BatchNormalization(mode=2),
    UpSampling2D(),
    Convolution2D(32, 3, 3, border_mode='same', activation=LeakyReLU()),
    BatchNormalization(mode=2),
    Convolution2D(1, 1, 1, border_mode='same', activation='sigmoid')
])


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
This is what's known in the literature as "mode collapse".
Can you write Python code for it?


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
Train a few epochs, and return the losses for D and G. In each epoch we:

Train D on one batch from data_D()
Train G to create images that the discriminator predicts as real.
Can you write Python code for it?

def train(D, G, m, nb_epoch=5000, bs=128):
    dl,gl=[],[]
    for e in tqdm(range(nb_epoch)):
        X,y = data_D(bs//2, G)
        dl.append(D.train_on_batch(X,y))
        make_trainable(D, False)
        gl.append(m.train_on_batch(noise(bs), np.zeros([bs])))
        make_trainable(D, True)
    return dl,gl


I am working on the file "FAI02_old/Lesson8/DCGAN.ipynb".
The first task is:
We train D a "little bit" so it can at least tell a real image from random noise.
Can you write Python code for it?

sz = n//200
x1 = np.concatenate([np.random.permutation(X_train)[:sz], CNN_G.predict(noise(sz))])
CNN_D.fit(x1, [0]*sz + [1]*sz, batch_size=128, nb_epoch=1, verbose=2)

CNN_m = Sequential([CNN_G, CNN_D])
CNN_m.compile(Adam(1e-4), "binary_crossentropy")

K.set_value(CNN_D.optimizer.lr, 1e-3)
K.set_value(CNN_m.optimizer.lr, 1e-3)


I am working on the file "FAI02_old/Lesson9/L9_neural_sr.ipynb".
The first task is:
We can't load Imagenet into memory, so we open the files and then pass them to the generator BcolzArrayIterator.
Can you write Python code for it?

arr_lr = bcolz.open(path + 'trn_resized_72.bc')
arr_hr = bcolz.open(path + 'trn_resized_288.bc')

pars = {'verbose':0, 'callbacks': [TQDMNotebookCallback(leave_inner=True)]}

def conv_block(x, filters, size, stride=(2,2), mode='same', act=True):
    x = Convolution2D(filters, size, size, subsample=stride, border_mode=mode)(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x) if act else x

def res_block(ip, nf=64):
    x = conv_block(ip, nf, 3, (1,1))
    x = conv_block(x, nf, 3, (1,1), act=False)
    return merge([x, ip], mode='sum')

def up_block(x, filters, size):
    x = keras.layers.UpSampling2D()(x)
    x = Convolution2D(filters, size, size, border_mode='same')(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x)

def get_model(arr):
    inp = Input(arr.shape[1:])
    x = conv_block(inp, 64, 9, (1,1))
    for i in range(4): x = res_block(x)
    x = up_block(x, 64, 3)
    x = up_block(x, 64, 3)
    x = Convolution2D(3, 9, 9, activation='tanh', border_mode='same')(x)
    outp = Lambda(lambda x: (x+1) * 127.5)(x)
    return inp, outp

def get_outp(m, λn): return m.get_layer(f'block{λn}_conv2').output

def mean_sqr_b(diff):
    dims = list(range(1, K.ndim(diff)))
    return K.expand_dims(K.sqrt(K.mean(diff**2, dims)), 0)

def content_fn(x):
    res = 0; n=len(w)
    for i in range(n): res += mean_sqr_b(x[i] - x[i+n]) * w[i]
    return res

inp, outp = get_model(arr_lr)
shp = arr_hr.shape[1:]

vgg_inp = Input(shp)
vgg = VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))
for λ in vgg.layers: λ.trainable=False

vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [1,2,3]])
vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)

w = [0.1, 0.8, 0.1]

m_sr = Model([inp, vgg_inp], Lambda(content_fn)(vgg1 + vgg2))
m_sr.compile('adam', 'mse')


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
After tarining for some time, we get some very impressive results! Look at these two images, we can see that the predicted higher resolution image has filled in a lot of detail, including the shadows under the greens and the texture of the food.
Can you write Python code for it?

plt.imshow(arr_lr[10].astype('uint8'));

plt.imshow(p[0].astype('uint8'));

top_model.save_weights(path + 'lesson9/results/' + 'sr_final.h5')

top_model.load_weights(path + 'lesson9/results/' + 'sr_final.h5')


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
An important difference in training for super resolution is the loss function. We use what's known as a perceptual loss function (which is simply the content loss for some layer).
Can you write Python code for it?

def get_outp(m, ln): return m.get_layer(f'block{ln}_conv1').output
vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [1,2,3]])
vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)

# ln = 1
# print(f'block{ln}_conv1'.format(ln))

def mean_sqr_b(diff):
    dims = list(range(1, K.ndim(diff)))
    return K.expand_dims(K.sqrt(K.mean(diff**2, dims)), 0)

w = [0.1, 0.8, 0.1]
def content_fn(x):
    res = 0; n=len(w)
    for i in range(n): res += mean_sqr_b(x[i]-x[i+n]) * w[i]
    return res

m_sr = Model([inp, vgg_inp], Lambda(content_fn)(vgg1+vgg2))
targ = np.zeros((arr_hr.shape[0], 1))


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Fast Style Transfer
The original paper showing the above approach to our super resolution also used this approach to createa a much faster style transfer system (for a specific style). Take a look at the paper and the very helpful supplementary material.
Reflection Padding
The supplementary material mentions that they found reflection padding helpful - we have implemented this as a Keras layer. All the other layers and blocks are already defined above.
Can you write Python code for it?

class ReflectionPadding2D(Layer):
    def __init__(self, padding=(1,1), **kwargs):
        self.padding = tuple(padding)
        self.input_spec = [InputSpec(ndim=4)]
        super(ReflectionPadding2D, self).__init__(**kwargs)

    def get_output_shape_for(self, s):
        return (s[0], s[1] + 2 * self.padding[0], s[2] + 2 * self.padding[1], s[3])

    def call(self, x, mask=None):
        w_pad, h_pad = self.padding
        return tf.pad(x, [[0,0], [h_pad, h_pad], [w_pad, w_pad], [0,0] ], 'REFLECT')


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Finally we compile this chain of models and we can pass it the original lores image as well as the hires to train on. We also define a zero vector as a target parameter, which is a necessary parameter when calling fit on a keras model.
Can you write Python code for it?

m_sr.compile('adam','mse')
m_sr.fit([arr_lr, arr_hr], targ, 8, 2, **pars)

m_sr.save_weights(path + 'lesson9/results/' + 'sr_final.h5')


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Here we alter the super resolution approach by adding style outputs:
Can you write Python code for it?

style_targs = [K.variable(o) for o in
               vgg_content.predict(np.expand_dims(style, 0))]

[K.eval(K.shape(o)) for o in style_targs]

vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Main Algorithm
This approach is exactly the same as super resoltuion, except now the loss includes the style loss.
Can you write Python code for it?

shp = arr_hr.shape[1:]

style = Image.open(path + 'nst/starry-night.png')
# style = style.resize(np.divide(style.size, 3.5).astype('int32'))
style = np.array(style)[:shp[0], :shp[1], :shp[2]]
plt.imshow(style);

def res_crop_block(ip, nf=64):
    x = conv_block(ip, nf, 3, (1,1,), 'valid')
    x = conv_block(x, nf, 3, (1,1), 'valid', False)
    ip = Lambda(lambda x: x[:, 2:-2, 2:-2])(ip)
    return merge([x, ip], mode='sum')

inp=Input(shp)
x=ReflectionPadding2D((40,40))(inp)
x=conv_block(x, 64, 9, (1,1))
x=conv_block(x, 64, 3)
x=conv_block(x, 64, 3)
for i in range(5): x=res_crop_block(x)
x=up_block(x, 64, 3)
x=up_block(x, 64, 3)
x=Convolution2D(3, 9, 9, activation='tanh', border_mode='same')(x)
outp=Lambda(lambda x: (x+1)*128.5)(x)

vgg_inp=Input(shp)
vgg=VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))
for λ in vgg.layers: λ.trainable=False

def get_outp(m, ln): return m.get_layer(f'block{ln}_conv2').output
vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [2,3,4,5]])


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Our loss now includes the MSE for the content loss and the Gram Matrix for the style
Can you write Python code for it?

def gram_matrix_b(x):
    x = K.permute_dimensions(x, (0, 3, 1, 2))
    s = K.shape(x)
    feat = K.reshape(x, (s[0], s[1], s[2]*s[3]))
    return K.batch_dot(feat, K.permute_dimensions(feat, (0, 2, 1))
                      ) / K.prod(K.cast(s[1:], K.floatx()))

w = [0.1, 0.2, 0.6, 0.1]
def tot_loss(x):
    loss = 0; n = len(style_targs)
    for i in range(n):
        loss += mean_sqr_b(gram_matrix_b(x[i+n]) - gram_matrix_b(style_targs[i])) / 2.
        loss += mean_sqr_b(x[i]-x[i+n]) * w[i]
    return loss

loss = Lambda(tot_loss)(vgg1 + vgg2)
m_style = Model([inp, vgg_inp], loss)
targ = np.zeros((arr_hr.shape[0], 1))

m_style.compile('adam', 'mse')
m_style.fit([arr_hr, arr_hr], targ, 4, 2, **pars)

m_style.save_weights(path + 'lesson9/results/' + 'style_final.h5')

K.set_value(m_style.optimizer.lr, 1e-4)
m_style.fit([arr_hr, arr_hr], targ, 4, 1, **pars)

top_model = Model(inp, outp)


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Since we only want to learn the "upsampling network", and are just using VGG to calculate the loss function, we set the Vgg layers to not be trainable:
Can you write Python code for it?


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Testing the reflection padding layer:
Can you write Python code for it?

inp = Input((288, 288, 3))
ref_model = Model(inp, ReflectionPadding2D((40, 10))(inp))
ref_model.compile('adam', 'mse')

p = ref_model.predict(arr_hr[10:11])
plt.imshow(p[0].astype('uint8'))


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
The important thing to take away here is that as opposed to our earlier approaches, this type of approach results in a model that can create the desired image and is a scalable implementation.
Note that we haven't used a test ste here, so we don't know if the above result is due to over-fitting.
Can you write Python code for it?

# well, since you mention it:
mofolo_jup = Image.open(path + 'sr-imgs/Jupiter-Juno-LR.jpeg')
mofolo_jup = np.expand_dims(np.array(mofolo_jup), 0)

p = top_model.predict(mofolo_jup)

# lores Jupiter
plt.imshow(mofolo_jup[0].astype('uint8'));

# superes Jupiter
plt.imshow(p[0].astype('uint8'))

# original hires jupiter:
mofohi_jup = Image.open(path + 'sr-imgs/Jupiter-Juno-HR.jpg')
plt.imshow(mofohi_jup)


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
The method of training this network is almost exactly the same as training the pixels from our previous implementations. The idea here is we're going to feed two images to Vgg16 and compare their convolutional outputs at some layer. These two images are the target image (which in our case is the same as the original but at a higher resolution), and the output of the previous network we just defined, which we hope will learn to output a high resolution image.
The key then is to train this other network to produce an image that minimizes the loss between the outputs of some convoltuional layer in Vgg16 (which the paper refers to as "perceptual loss"). In doing so, we're able to train a network that can upsample an image and recreate the higher resolution details.
Can you write Python code for it?

vgg_inp = Input(shp)
vgg = VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
To start we'll define some of the building blocks of our network. In particular recall the residual block (as used in ResNet), which is just a sequence of 2 Convolutional layers that is added to the initial block input. We also have a de-Convolutional layer (aka. a "Transposed Convolution" or "Fractionally-Strided Convolution"), whose purpose is to learn to 'undo' the Convolutional function. It does this by padding the smaller image in such a way to apply filters on it to produce a larger image.
Can you write Python code for it?

def conv_block(x, filters, size, stride=(2,2), mode='same', act=True):
    x = Convolution2D(filters, size, size, subsample=stride, border_mode=mode)(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x) if act else x
def res_block(ip, nf=64):
    x = conv_block(ip, nf, 3, (1,1))
    x = conv_block(x, nf, 3, (1,1), act=False)
    return merge([x, ip], mode='sum')
# def deconv_block(x, filters, size, shape, stride=(2,2)):
#     x = Deconvolution2D(filters, size, size, subsample=stride,
#                         border_mode='same', output_shape=(None,)+shape)(x)
#     x = BatchNormalization(mode=2)(x)
#     return Activation('relu')(x)
def up_block(x, filters, size):
    x = keras.layers.UpSampling2D()(x)
    x = Convolution2D(filters, size, size, border_mode='same')(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x)


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
Use Content Loss to Create a Super-Resolution Network
So far we've demonstrated how to achieve successful results in style transfer. However, there's an obvious drawback to our implementation, namely that we're training an image, not a network, and therefore every new image requires us to retrain. It's not a feasible menthod for any sort of real-time application. Fortunately, we can address this issue by using a fully covolutional network (FCN), and in particular we'll look at this implementation for Super Resolution. We're following the approach in this paper
Can you write Python code for it?

rn_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32)
preproc = lambda x: (x - rn_mean)[:, :, :, ::-1]

deproc = lambda x,s: np.clip(x.reshape(s)[:, :, :, ::-1] + rn_mean, 0, 255)

arr_lr = bcolz.open(path + 'trn_resized_72.bc')[:]
arr_hr = bcolz.open(path + 'trn_resized_288.bc')[:]

pars = {'verbose': 0, 'callbacks': [TQDMNotebookCallback(leave_inner=True)]}

shp = arr_hr.shape[1:]

arr_hr.shape[1:]


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
We're only interested in the trained part of the mdoel, which does the actual upsampling.
Can you write Python code for it?

top_model = Model(inp, outp)

p = top_model.predict(arr_lr[10:11])


I am working on the file "FAI02_old/Lesson9/Lesson9_SR_CodeAlong.ipynb".
The first task is:
this is kind of hillarious
Can you write Python code for it?

top_model.save_weights(path + 'lesson9/results/style_final.h5')
# top_model.load_weights(path + 'lesson9/results/style_final.h5')


I am working on the file "FAI02_old/Lesson9/Neural-Super-Resolution-codealong.ipynb".
The first task is:
All code is identical to the implementation shown in the neural-style notebook, with the exception of the BcolzArrayIterator training implementation.
Can you write Python code for it?

rn_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32)
preproc = lambda x: (x - rn_mean)[:,:,:,::-1]
deproc  = lambda x,s: np.clip(x.reshape(s)[:,:,:,::-1] + rn_mean, 0, 255)


I am working on the file "FAI02_old/Lesson9/Neural-Super-Resolution-codealong.ipynb".
The first task is:
Our training implementation has been altered to accomodate the BcolzArrayIterator.
We're unable to use model.fit_generator() because that function call expects the generator to return a tuple of inputs and targets.
Our generator howeve,r yields two inputs. We can work around this by separately pulling out our inputs from the generator and then using model.train_on_batch() with our inputs from the generator and our dummy targets. model.train_on_batch() simply does one gradient update on the batch of data.
This technique of creating your own training loop is useful when you are working with various iterators or complicated inputs that don't conform to Keras' standard fitting methods.
Can you write Python code for it?

def train(bs, niter=10):
    targ = np.zeros((bs, 1))
    bc = BcolzArrayIterator(arr_hr, arr_lr, batch_size=bs)
    for i in range(niter):
        hr, λr = next(bc)
        m_sr.train_on_batch([λr[:bs], hr[:bs]], targ)

its = len(arr_hr)//16; its

%time train(16, 18000)

K.set_value(m_sr.optimizer.lr, 1e-4)
train(16, 18000)

top_model = Model(inp, outp)

p = top_model.predict(arr_lr[:20])

# new_file = '----/ILSVRC2012_val_00005642.JPEG''

img = Image.open(new_file).resize((288, 288))
img_dat = np.expand_dims(np.array(img),0)


I am working on the file "FAI02_old/Lesson9/Neural-Super-Resolution-codealong.ipynb".
The first task is:
The model is working as expected.
Can you write Python code for it?

idx=4
plt.imshow(arr_hr[idx].astype('uint8'));

plt.imshow(arr_lr[idx].astype('uint8'));

plt.imshow(p[idx].astype('uint8'));


I am working on the file "FAI02_old/Lesson9/Neural-Super-Resolution-codealong.ipynb".
The first task is:
We can't load ImageNet into memory, so we open the files and then pass them to the generator BcolzArrayIterator.
Can you write Python code for it?

arr_lr = bcolz.open(dpath + 'trn_resized_72_r.bc')
arr_hr = bcolz.open(path  + 'results/trn_resized_288_r.bc')

pars = {'verbose': 0, 'callbacks': [TQDMNotebookCallback(leave_inner=True)]}

def conv_block(x, filters, size, stride=(2,2), mode='same', act=True):
    x = Convolution2D(filters, size, size, subsample=stride, border_mode=mode)(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x) if act else x

def res_block(ip, nf=64):
    x = conv_block(ip, nf, 3, (1,1))
    x = conv_block(x, nf, 3, (1,1), act=False)
    return merge([x, ip], mode='sum')

def up_block(x, filters, size):
    x = keras.layers.UpSampling2D()(x)
    x = Convolution2D(filters, size, size, border_mode='same')(x)
    x = BatchNormalization(mode=2)(x)
    return Activation('relu')(x)

def get_model(arr):
    inp=Input(arr.shape[1:])
    x=conv_block(inp, 64, 9, (1,1))
    for i in range(4): x=res_block(x)
    x=up_block(x, 64, 3)
    x=up_block(x, 64, 3)
    x=Convolution2D(3, 9, 9, activation='tanh', border_mode='same')(x)
    outp=Lambda(lambda x: (x+1)*127.5)(x)
    return inp,outp

inp, outp - get_model(arr_lr)
shp = arr_hr.shape[1:]

vgg_inp=Input(shp)
vgg=VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))
for λ in vgg.layers: λ.trainable=False

def get_outp(m, λn): return m.get_layer(f'block{λn}_conv2').output

vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [1,2,3]])
vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)

def mean_sqr_b(diff):
    dims = list(range(1, K.ndim(diff)))
    return K.expand_dims(K.sqrt(K.mean(diff**2, dims)), 0)

w = [0.1, 0.8, 0.1]

def content_fn(x):
    res = 0; n = len(w)
    for i in range(n): res += mean_sqr_b(x[i] - x[i+n]) * w[i]
    return res

m_sr = Model([inp, vgg_inp], Lambda(content_fn)(vgg1 + vgg2))
m_sr.compile('adam', 'mae')


I am working on the file "FAI02_old/Lesson9/neural-style-GPU.ipynb".
The first task is:
Create Model
Can you write Python code for it?

def download_convert_vgg16_model():
    model_url = 'http://cs.stanford.edu/people/jcjohns/fast-neural-style/models/vgg16.t7'
    file = get_file(model_url, cache_subdir='models')
    vgglua = load_lua(file).parameters()
    vgg = models.VGGFeature()
    for (src, dst) in zip(vgglua[0], vgg.parameters()): dst[:] = src[:]
    torch.save(vgg.state_dict(), path + 'vgg16_feature.pth')

url = 'https://s3-us-west-2.amazonaws.com/jcjohns-models/'
fname = 'vgg16-00b39a1b.pth'
file = get_file(fname, url+fname, cache_subdir='models')

vgg = models.vgg.vgg16()
vgg.load_state_dict(torch.load(file))
optimizer = optim.Adam(vgg.parameters())

vgg.cuda();

arr_lr = bcolz.open(path + 'trn_resized_72.bc')[:]
arr_hr = bcolz.open(path + 'trn_resized_288.bc')[:]

arr = bcolz.open(dpath + 'trn_resized.bc')[:]

x = Variable(arr[0])
y = model(x)

url = 'http://www.files.fast.ai/models/'
fname = 'imagenet_class_index.json'
fpath = get_file(fname, url + fname, cache_subdir='models')

class ResidualBlock(nn.Module):
    def __init__(self, num):
        super(ResideualBlock, self).__init__()
        self.c1 = nn.Conv2d(num, num, kernel_size=3, stride=1, padding=1)
        self.c2 = nn.Conv2d(num, num, kernel_size=3, stride=1, padding=1)
        self.b1 = nn.BatchNorm2d(num)
        self.b2 = nn.BatchNorm2d(num)

    def forward(self, x):
        h = F.relu(self.b1(self.c1(x)))
        h = self.b2(self.c2(h))
        return h + x

class FastStyleNet(nn.Module):
    def __init__(self):
        super(FastStyleNet, self).__init__()
        self.cs = [nn.Conv2d(3, 32, kernel_size=9, stride=1, padding=4),
                   nn.Conv2d(32, 64, kernel_size=4, stride=2, padding=1),
                   nn.Conv2d(64, 128, kernel_size=4, stride=2, padding1)]
        self.b1s = [nn.BatchNorm2d(i) for i in [32, 64, 128]]
        self.rs = [ResidualBlock(128) for i in range(5)]
        self.ds = [nn.ConvTranspose2d(64, 32, kernel_size=4, stride=2, padding=1),
                   nn.ConvTranspose2d(64, 32, kernel_size=4, stride=2, padding=1)]
       ...


I am working on the file "FAI02_old/Lesson9/neural-style-GPU.ipynb".
The first task is:
Setup
Can you write Python code for it?

path = '../data/nst/'
fnames = pickle.load(open(path+'fnames.pkl','rb'))

img = Image.open(path + fnames[0]); img

rn_mean = np.array([123.68, 116.779, 103.939], dtype=np.float32).reshape((1,1,1,3))
preproc = lambda x: (x - rn_mean)[:,:,:,::-1]

img_arr = preproc(np.expand_dims(np.array(img),0))
shp = img_arr.shape

deproc = lambda x: x[:,:,:,::-1] + rn_mena


I am working on the file "FAI02_old/Lesson9/neural-style-GPU_weird-things.ipynb".
The first task is:
Content Recreation
Can you write Python code for it?

# had to fix some compatibility issues w/ Keras 1 -> Keras 2
import vgg16_avg
importlib.reload(vgg16_avg)
from vgg16_avg import VGG16_Avg

model = VGG16_Avg(include_top=False)
# grabbing activations from near the end of the CNN model
layer = model.get_layer('block5_conv1').output
# calculating layer's target activations
layer_model = Model(model.input, layer)
targ = K.variable(layer_model.predict(img_arr))


I am working on the file "FAI02_old/Lesson9/neural-style-GPU_weird-things.ipynb".
The first task is:
In this implementation, need to define an object that'll allow us to separately access the loss function and gradients of a function,
Can you write Python code for it?

class Evaluator(object):
    def __init__(self, f, shp): self.f, self.shp = f, shp

    def loss(self, x):
        loss_, self.grad_values = self.f([x.reshape(self.shp)])
        return loss_.astype(np.float64)

    def grads(self, x): return self.grad_values.flatten().astype(np.float64)

# Define loss function to calc MSE betwn the 2 outputs at specfd Conv layer
loss = metrics.mse(layer, targ)
grads = K.gradients(loss, model.input)
fn = K.function([model.input], [loss]+grads)
evaluator = Evaluator(fn, shp)

# optimize loss fn w/ deterministic approach using Line Search
def solve_image(eval_obj, niter, x):
    for i in range(niter):
        x, min_val, info = fmin_l_bfgs_b(eval_obj.loss, x.flatten(),
                                         fprime=eval_obj.grads, maxfun=20)
        x = np.clip(x, -127,127)
        print('Current loss value:', min_val)
        imsave(f'{path}/results/res_at_iteration_{i}.png', deproc(x.copy(), shp)[0])
    return x

# generating a random image:
rand_img = lambda shape: np.random.uniform(-2.5,2.5,shape)/100
x = rand_img(shp)
plt.imshow(x[0])

iterations = 10
x = solve_image(evaluator, iterations, x)

Image.open(path + 'results/res_at_iteration_1.png')

# Looking at result for earlier Conv block (4):
layer = model.get_layer('block4_conv1').output
layer_model = Model(model.input, layer)
targ = K.variable(layer_model.predict(img_arr))

loss = metrics.mse(layer, targ)
grads = K.gradients(loss, model.input)
fn = K.function([model.input], [loss]+grads)
evaluator = Evaluator(fn, shp)

x = solve_image(evaluator, iterations, x)

Image.open(path + 'results/res_at_iteration_9.png')


I am working on the file "FAI02_old/Lesson9/neural_sr_attempt2.ipynb".
The first task is:
Finally starting to understand this problem. So ResourceExhaustedError isn't system memory (or at least not only) but graphics memory. The card (obviously) cannot handle a batch size of 64. But batch size must be a multiple of chunk length, which here is 64.. so I have to find a way to reduce the chunk length down to something my system can handle: no more than 8.
Can you write Python code for it?

arr_lr_c8 = bcolz.carray(arr_lr, chunklen=8, rootdir=path+'trn_resized_72_c8.bc')
arr_lr_c8.flush()

arr_hr_c8 = bcolz.carray(arr_hr, chunklen=8, rootdir=path+'trn_resized_288_c8.bc')
arr_hr_c8.flush()

arr_lr_c8.chunklen, arr_hr_c8.chunklen


I am working on the file "FAI02_old/Lesson9/neural_sr_attempt2.ipynb".
The first task is:
That looks successful, now to redo the whole thing with the _c8 versions:
Can you write Python code for it?

arr_lr_c8 = bcolz.open(path+'trn_resized_72_c8.bc')
arr_hr_c8 = bcolz.open(path+'trn_resized_288_c8.bc')

inp,outp=get_model(arr_lr_c8)

shp = arr_hr_c8.shape[1:]

vgg_inp=Input(shp)
vgg= VGG16(include_top=False, input_tensor=Lambda(preproc)(vgg_inp))
for l in vgg.layers: l.trainable=False

vgg_content = Model(vgg_inp, [get_outp(vgg, o) for o in [1,2,3]])
vgg1 = vgg_content(vgg_inp)
vgg2 = vgg_content(outp)

m_sr = Model([inp, vgg_inp], Lambda(content_fn)(vgg1+vgg2))
m_sr.compile('adam', 'mae')

def train(bs, niter=10):
    targ = np.zeros((bs, 1))
    bc = BcolzArrayIterator(arr_hr_c8, arr_lr_c8, batch_size=bs)
    for i in range(niter):
        hr,lr = next(bc)
        m_sr.train_on_batch([lr[:bs], hr[:bs]], targ)

%time train(8, 18000) # not sure what exactly the '18000' is for

arr_lr.shape, arr_hr.shape, arr_lr_c8.shape, arr_hr_c8.shape

# 19439//8 = 2429
%time train(8, 2430)


I am working on the file "FAI02_old/layer-test.ipynb".
The first task is:
Aaaahaaaaaaaa. Okay. so the .output parameter for kears.models.Model(..) will take all layers of a model up to and including the layer specified.
It does NOT create a model of only the layer specified. input is a keras tensor (with attributes: name, shape, dtype). output is also a keras tensor with attributes (name-&-activation-fn, shape, dtype).
So, a model is created consisting of all layers between and including input and output layers.
Can you write Python code for it?


I am working on the file "FAI_old/Lesson6/L6_Theano_RNN_Notes.ipynb".
The first task is:
At the end of learning (via the loop above), we create a new Theano fn which takes some piece of input along with some initial hidden state, and produce not the loss, but the output.
This is to do testing: the fn goes from out inputs to our vector of outputs v_y.
Our predictions preds, will be to take that fn f_y, pass it our initial hidden state, np.zeros(n_hidden), and some input, say X[6], and that'll give us some predictions.
Can you write Python code for it?

f_y = theano.function([t_h0, t_inp], v_y, allow_input_downcast=True)

pred = np.argmax(f_y(np.zeros(n_hidden), X[6]), axis=1)

# grabbing sample input text
act = np.argmax(X[6], axis=1)

# displaying input text
[indices_char[o] for o in act]

# displaying expected outputs
[indices_char[o] for o in pred]


I am working on the file "FAI_old/Lesson6/L6_Theano_RNN_Notes.ipynb".
The first task is:
By this point we've described how to execute a whole sequence of stesp for an RNN. We haven't given it any data to do it; we've just set up the computation.
When that computation is run, it's going to return 2 things bc step returns 2 things: the hidden state v_h, and our output activations v_y.
Now we need to calculate our error. Using cat-crossent we compare the output of our scan, v_y to some matrix t_outp. Then add it all together.
We want to apply SGD after every step, meaning we have to take the derivative of w_all wrt. all the weights, and use that along with the learning rate to update all the weights. Theano has a simple function call to do that: T.grad(..)
Can you write Python code for it?

error = nnet.categorical_crossentropy(v_y, t_outp).sum()
g_all = T.grad(error, w_all)
# "please tell me the gradient, `g_all`, of the function `error`,
# wrt. these gradients: `w_all`"
# Theano will symbolically calculate all the derivatives for you.


I am working on the file "FAI_old/Lesson6/L6_Theano_RNN_Notes.ipynb".
The first task is:
Python has a function, chain.from_iterable(..), that takes a list of tuples and turns them into one big list.
The next thing we have to do is tell Theano what happens each time we take a single step of this RNN.
There's no such thing as a for-loop on a GPU bc the GPU is built to & wants to parallelize things & do multiple things at the same time.
There is something very similar to a for-loop that you can parallelize, it's called a scan operation.
A scan operation is smth where you call some function (step) for every element (t_inp) of some sequence, and at every point the function returns some output, and the next time thru that function is called, it gets the output of the previous time to you called it along with the next element of the sequence.
Can you write Python code for it?

# example of scan:
def scan(fn, start, seq):
    res = [] # array of results
    prev = start
    for s in seq:
        app = fn(prev, s) # apply function to previous result & next elem in sequence
        res.append(app)
        prev = app
    return res

scan(lambda prev,curr: prev+curr, 0, range(5))

# output:
# [0, 1, 3, 6, 10]

# the scan operation defines a cumulative sum.

# It is possible to write a parallel version of this.
# If you can turn you algorithm into a scan, you can run it quickly
# on a GPU.


I am working on the file "FAI_old/Lesson6/L6_Theano_RNN_Notes.ipynb".
The first task is:
The fucntion we'll call on ea. step thru is a fn called step. step takes the input, x, does a dot-product by the weight-matrix we created earlier, W_x, adds on the bias term we created earlier, b_x. then we do the same thing, taking our previous hidden state h, multiplying it by the hidden weight-matrix W_h and adding the biases b_h; then putting that whole thing through the activation function, nnet.relu.
After we do that we want to create an output each time. Output is exactly the same thing. It takes the result of h, the hidden state, multiply it by the outputs weight-vector, adding on the bias; and this time we use nnnet.softmax.
At the end of that, we return the hidden state we have so far, and the output, T.flatten(y, 1)
All that happens each step.
Can you write Python code for it?

def step(x, h, W_h, b_h, W_x, b_x, W_y, b_y):
    h = nnet.relu(T.dot(x, W_x) + b_x + T.dot(h, W_h) + b_h)
    y = nnet.softmax(T.dot(h, W_y) + b_y)
    return h, T.flatten(y, 1)


I am working on the file "FAI_old/Lesson6/L6_Theano_RNN_Notes.ipynb".
The first task is:
To create the weights & biases, up above we have a function called wgts_and_bias(..) in which we tell Theano the size of the matrix we want to create.
The matrix that goes from input to hidden has n_input rows and n_hidden collumns.
wgts_and_bias returns a tuple of weights and biases.
To create the weights, we first calculate the Glorot number, sqrt(2/n) -- the scale of the random numbers we're going to use -- then we create those random numbers using the Numpy normal(..) random number function, and then we use a special Theano keyword called shared. shared(..) tells Theano that the data inside is something we want it to pass off to the GPU later and keep track of.
So once you wrap something in shared, it kind of belongs to Theano now.
Can you write Python code for it?

# weights & bias to hidden layer
W_h = id_and_bias(b_hidden)
# weights & bias to input
W_x = wgts_and_bias(n_input, n_hidden)
# weights & bias to output
W_y = wgts_and_bias(n_hidden, n_output)
# stick all manually constructed weight matrices and bias vectors
# in a list:
w_all = list(chain.from_iterable([W_h, W_x, W_y]))


I am working on the file "FAI_old/Lesson6/L6_Theano_RNN_Notes.ipynb".
The first task is:
We create our one-hot encoded X's and Y's, and we have to manually create our own loop; Theano has nothing built-in for this.
We go through every element of our input X, and call that function that we just created above, and pass in all its inputs, all_args, for initial hidden state t_h0, input t_inp, target t_outp, and learning rate lr.
Initial hidden state, a bunch of zeros: np.zeros(n_hidden).
The condition if i % 1000 == 999: ... just prints out the error every 1000 loops.
(we're using stochastic gradient descent with a minibatch size of 1)
gradient descent w/o stochastic means your using a minibatch size of the whole dataset
so this is 'online' gradient descent
Can you write Python code for it?

X = oh_x_rnn
Y = oh_y_rnn
# X.shape, Y.shape

err=0.0; l_rate=0.01
for i in range(len(X)):
    err+=fn(np.zeros(n_hidden), X[i], Y[i], l_rate)
    if i % 1000 == 999:
        print("Error:{:.3f}".format(err/1000))
        err=0.0


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
3 Char Model
Create Inputs
Create a list of every 4th character, starting at the 0th, 1st, 2nd, then 3rd characters.
We're going to build a model that attempts to predict the 4th character from the previous 3. To do that we're going ot go through our whole list of indexes from 0 to the end minus 3, and we'll create a whole list of the 0th, 4th, 8th, 12th, etc characters; the 1st, 5th, 9th, etc; and 2nd, 6th, 10th, & so forth..
Can you write Python code for it?

cs = 3
c1_dat = [idx[i] for i in xrange(0, len(idx)-1-cs, cs)]
c2_dat = [idx[i+1] for i in xrange(0, len(idx)-1-cs, cs)]
c3_dat = [idx[i+2] for i in xrange(0, len(idx)-1-cs, cs)]
c4_dat = [idx[i+3] for i in xrange(0, len(idx)-1-cs, cs)] # <-- gonna predict this


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
For each 0 thru 7, create a list of every 8th character with that starting point. These will be the 8 inputs to our model.
Can you write Python code for it?

c_in_dat = [[idx[i+n] for i in xrange(0, len(idx)-1-cs, cs)] for n in range(cs)]


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Map from chars to indices and back again
Can you write Python code for it?

char_indices = dict((c, i) for i, c in enumerate(chars))
indices_char = dict((i, c) for i, c in enumerate(chars))


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
NOTE: it's almost the same as the 1-7th characters in the first row of xs. The final character in ea. sequence is the same as the first character of this sequence. It's almost the same as our previous data, just done in a more flexible way.
Create and train Model
Can you write Python code for it?

n_fac = 42
def embedding_input(name, n_in, n_out):
    inp = Input(shape=(1,), dtype='int64', name=name+'_in')
    emb = Embedding(n_in, n_out, input_length=1, name=name+'_emb')(inp)
    return inp, Flatten()(emb)

c_ins = [embedding_input('c'+str(n), vocab_size, n_fac) for n in range(cs)]
n_hidden = 256

dense_in = Dense(n_hidden, activation='relu')
dense_hidden = Dense(n_hidden, activation='relu', init='identity')
dense_out = Dense(vocab_size, activation='softmax')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Note 8 outputs. What TimeDistributed does is create 8 copies of the weight matrix for each output.
NOTE: in Keras anytime you specfy return_sequences=True, any dense layers after that must have TimeDistributed wrapped around them. - Bc, in this case, we want to create not 1 dense layer, but 8.
Can you write Python code for it?

model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())

# just some dimensionality changes required; otherwise same
x_rnn = np.stack(np.squeeze(xs), axis=1)
y_rnn = np.stack(ys, axis=1)

x_rnn.shape, y_rnn.shape

model.fit(x_rnn, y_rnn, batch_size=64, nb_epoch=8)

def get_nexts_keras(inp):
    idxs = [char_indices[c] for c in inp]
    arr = np.array(idxs)[np.newaxis,:]
    p = model.predict(arr)[0]
    print(list(inp))
    return [chars[np.argmax(o)] for o in p]

get_nexts_keras(' this is')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Now our y dataset looks exactly like our x dataset did before, but everything's shifted over by 1 character.
Create and train the model:
Can you write Python code for it?

dense_in = Dense(n_hidden, activation='relu')
dense_out = Dense(vocab_size, activation='softmax', name='output')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Now we're ready to create our initial weight matrices.
Can you write Python code for it?

W_h = id_and_bias(n_hidden)
W_x = wgts_and_bias(n_input, n_hidden)
W_y = wgts_and_bias(n_hidden, n_output)
w_all = list(chain.from_iterable([W_h, W_x, W_y]))


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Now when we fit, we add the array of zeros to the start of our inputs; our ouputs are going to be those lists of 8, offset by 1. We get 8 losses instead of 1 bc ea. one of those 8 outputs has its own loss. You'll see the model's ability to predict the 1st character from a bunch of zeros is very limited and flattens out; but predicting the 8th char with the context of 7 is much better and keeps improving.
Can you write Python code for it?

model.fit([zeros]+xs, ys, batch_size=64, nb_epoch=12)


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Our first hidden activation is simpmly this function applied to the result of the embedding of the first character.
Can you write Python code for it?


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Our inputs
Can you write Python code for it?

# we can turn these into Numpy arrays just by stacking them up together
x1 = np.stack(c1_dat[:-2]) # 1st chars
x2 = np.stack(c2_dat[:-2]) # 2nd chars
x3 = np.stack(c3_dat[:-2]) # 3rd chars
# for every 4 character peice of this - collected works


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Our output
Can you write Python code for it?

# labels will just be the 4th characters
y = np.stack(c4_dat[:-2])


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Our second and third hidden activations sum up the previous hidden state (after applying dense_hidden) to the new input state.
Can you write Python code for it?

c2_dense = dense_in(c2) # char-2 embedding thru greenarrow
hidden_2 = dense_hidden(c1_hidden) # output of char-1's hidden state thru orangearrow
c2_hidden = merge([c2_dense, hidden_2]) # merge the two together (default: sum)

c3_dense = dense_in(c3)
hidden_3 = dense_hidden(c2_hidden)
c3_hidden = merge([c3_dense, hidden_3])


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Returning Sequences
Create Inputs
Here, c_out_dat is identical to c_in_dat, but moved across 1 character.
So now, in ea sequece, the 1st char will be used to predict the 2nd, the 1st & 2nd to predict the 3rd, and so on. A lot more predictions going on --> a lot more opportunity for the model to learn.
Can you write Python code for it?

# c_in_dat = [[idx[i+n] for i in xrange(0, len(idx)-1-cs, cs)] for n in range(cs)]
c_out_dat = [[idx[i+n] for i in xrange(1, len(idx)-cs, cs)] for n in range(cs)]

ys = [np.stack(c[:-2]) for c in c_out_dat]

[xs[n][:cs] for n in range(cs)]

[ys[n][:cs] for n in range(cs)]


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Sequence model with Keras
return_sequences=True says: rather than put the triangle outside the loop, put it inside the recurrent loop; ie: return an output every time you go to another time-step, intead of just a single output at the end.
Can you write Python code for it?


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Setup
We're going to download the collected works of Nietzsche to use as our data for this class.
Can you write Python code for it?

path = get_file('nietzsche.txt', origin="https://s3.amazonaws.com/text-datasets/nietzsche.txt")
text = open(path).read()
print('corpus length:', len(text))

chars = sorted(list(set(text)))
vocab_size = len(chars) + 1
print('total chars:', vocab_size)


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
So each column below is one series of 8 characters from the text.
Can you write Python code for it?

# visualizing xs:
[xs[n][:cs] for n in range(cs)]


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
So now we can create our model
Can you write Python code for it?

model = Model([c[0] for c in c_ins], c_out)
model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())

model.fit(xs, y, batch_size=64, nb_epoch=12)


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Sometimes it's useful to have a zero value in the dataset, eg. for padding
Can you write Python code for it?

chars.insert(0, "\0")

''.join(chars[1:-6])


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Test model
We test it by creating a function that we pass 3 letters into. Turn those letters into character indices (by looking them up in char_indices); turn each of those into a Numpy array; call model.predict on those 3 arrays -- that gives us 86 outputs; We then do an argmax to find which index of those 86 is the highest: and that's the character number we want to return.
So basically: we give it 3 letters, it gives us back the letter it thinks is most likely next.
Can you write Python code for it?

def get_next(inp):
    idxs = [char_indices[c] for c in inp]
    arrs = [np.array(i)[np.newaxis] for i in idxs]
    p = model.predict(arrs)
    i = np.argmax(p)
    return chars[i]

get_next('phi')

get_next(' th')

get_next(' an')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
The LSTM model takes much longer to run than the regular RNN because it isn't in parallel: each operation has to be run in order.
Can you write Python code for it?

model.fit(x_rnn[:mx], y_rnn[:mx], batch_size=bs, nb_epoch=4, shuffle=False)

model.optimizer.lr=1e-4
model.fit(x_rnn[:mx], y_rnn[:mx], batch_size=bs, nb_epoch=4, shuffle=False)


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
The first 4 inputs and ouputs
Can you write Python code for it?

# 1st, 2nd, 3rd chars of text
x1[:4], x2[:4], x3[:4]

# 4th char of text
y[:3]


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
The first column in each row is the 1st 8 characters of our test.
...and this is the next character after each sequence:
Can you write Python code for it?


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
The number of latent factors to create (ie. the size of our 3 character inputs)
Can you write Python code for it?

# we're going to turn these into embeddings
n_fac = 42

# by creating an embedding matrix
def embedding_input(name, n_in, n_out):
    inp = Input(shape=(1,), dtype='int64', name=name)
    emb = Embedding(n_in, n_out, input_length=1)(inp)
    return inp, Flatten()(emb)

c1_in, c1 = embedding_input('c1', vocab_size, n_fac)
c2_in, c2 = embedding_input('c2', vocab_size, n_fac)
c3_in, c3 = embedding_input('c3', vocab_size, n_fac)
# c1, c2, c3 represent result of putting each char through the embedding &
# getting out 42 latent vectors. <-- those are input to greenarrow.


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Theano RNN
Sometimes you just need more control over an artificial mind.
Can you write Python code for it?

n_input = vocab_size
n_output = vocab_size


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
Then for each successive layer we combine the output of dense_in() on the ext character with the output of dense_hidden() on the current hidden state, to create the new hidden state.
Can you write Python code for it?

for i in range(1,cs):
    c_dense = dense_in(c_ins[i][1]) #green arrow
    hidden = dense_hidden(hidden)   #orange arrow
    hidden = merge([c_dense, hidden]) #merge the two together


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
This is the 'blue arrow' from our diagram - the layer operation from hidden to hidden.
Can you write Python code for it?

dense_out = Dense(vocab_size, activation='softmax') #output size: 86 <-- vocab_size


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
This is the 'green arrow' from our diagram - the layer operation from input to hidden.
Can you write Python code for it?

dense_in = Dense(n_hidden, activation='relu')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
This is the 'orange arrow' from our diagram - the layer operation from hidden to hidden.
Can you write Python code for it?

dense_hidden = Dense(n_hidden, activation='tanh')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
This is what a sequence model looks like. We pass in a sequence and after every character, it returns a guess.
Test Model:
Can you write Python code for it?

def get_nexts(inp):
    idxs = [char_indices[c] for c in inp]
    arrs = [np.array(i)[np.newaxis] for i in idxs]
    p = model.predict([np.zeros(n_fac)[np.newaxis,:]] + arrs)
    print(list(inp))
    return [chars[np.argmax(o)] for o in p]

get_nexts(' this is')

get_nexts(' part of')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
To convert our previous ekras model into a sequence model, simply add the return_sequences=True parameter, and TimeDistributed() around our dense layer.
Can you write Python code for it?

model = Sequential([
        Embedding(vocab_size, n_fac, input_length=cs),
        SimpleRNN(n_hidden, return_sequences=True, activation='relu', inner_init='identity'),
        TimeDistributed(Dense(vocab_size, activation='softmax')),
])

model.summary()


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
We now need to tell Theano what happens each time we take a single step of this RNN.

Theano handles looping by using the GPU scan operation. We have to tell theano what to do at each step through the scan - this is the function we'll use, which does a single forward pass for one character.
Can you write Python code for it?

def step(x, h, W_h, b_h, W_x, b_x, W_y, b_y):
    # Calculate the hidden activations
    h = nnet.relu(T.dot(x, W_x) + b_x + T.dot(h, W_h) + b_h)
    # Calculate the output activations
    y = nnet.softmax(T.dot(h, W_y) + b_y)
    # Return both (the 'Flatten()' is to work around a theano bug)
    return h, T.flatten(y, 1)


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
We're going to pass a vector of all zeros as our starting point - here's our input layers for that:
Can you write Python code for it?

# our char1 input is moved within the diagram's loop-box; so now need
# initialized input (zeros)
inp1 = Input(shape=(n_fac,), name='zeros')
hidden = dense_in(inp1)

outs = []

for i in range(cs):
    c_dense = dense_in(c_ins[i][1])
    hidden = dense_hidden(hidden)
    hidden = merge([c_dense, hidden], mode='sum')
    # every layer now has an output
    outs.append(dense_out(hidden))

# our loop is identical to before, except at the end of every loop,
# we're going to append this output; so now we're going to have
# 8 outputs for every sequence instead of just 1.

# model now has vector of 0s: [inp1], and array of outputs: outs
model = Model([inp1] + [c[0] for c in c_ins], outs)
model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())

zeros = np.tile(np.zeros(n_fac), (len(xs[0]),1))
zeros.shape


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
With 8 pieces of context instead of 3, we'd expect it to do better; and we see a loss of ~1.8 instead of ~2.0
Test model:
Can you write Python code for it?

def get_next(inp):
    idxs = [np.array(char_indices[c])[np.newaxis] for c in inp]
    p = model.predict(idxs)
    return chars[np.argmax(p)]

get_next('for thos')

get_next('part of ')

get_next('queens a')


I am working on the file "FAI_old/Lesson6/Lesson6_CodeAlong.ipynb".
The first task is:
idx will be the data we use from now on -- it simply converts all the characters to their index (based on the mapping above)
Can you write Python code for it?

idx = [char_indices[c] for c in text]
# the 1st 10 characters:
idx[:10]

''.join(indices_char[i] for i in idx[:70])


I am working on the file "FAI_old/Lesson6/Theano_RNN_test_4.ipynb".
The first task is:
It's a little better but no where near good enough. I need to find out what's wrong..
Can you write Python code for it?

xs3 = [np.stack(x[:]) for x in xs2]
xs3[0].shape

xs3


I am working on the file "FAI_old/Lesson6/Theano_RNN_test_4.ipynb".
The first task is:
xs3 is in the proper format.. so I have to look at the oh_xs3 and oh_xs3_rnn versions
Can you write Python code for it?


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
Also need to define a scan function. Not concerned w/ running things in parallel here, so it's simpe to implement:
scan(..): go thru a sequences, 1 step at a time; calling a function on ea. element of the sequence. Each time, the function gets 2 things: the next elem of the seq as well as the previous result of the call.
NOTE: Theano doesn't use a for-loop, it creates a tree that simultaneously & gradually combines everything together.
Can you write Python code for it?

# using relu as activation
act = relu
act_d = relu_d
# using crossentropy as loss function
loss = x_entropy
loss_d = x_entropy_d

def scan(fn, start, seq):
    res = []
    prev = start
    for s in seq:
        app = fn(prev, s)
        res.append(app)
        prev = app
    return res

# test: scan on add 2 things together, on integers 0..4
scan(lambda prev, curr: prev + curr, 0, range(5))


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
Because there're these reset & update gates, the GRU-NN has the ability to learn these special sets of weights (rW_h, uW_h, ...), to make sure it throws away or keeps more state when that's a good idea.
Everything here onwards is identical to the simple Theano RNN.
Can you write Python code for it?

[v_h, v_y], _ = theano.scan(step, sequences=t_inp,
                            outputs_info=[t_h0, None], non_sequences=w_all)
error = nnet.categorical_crossentropy(v_y, t_outp).sum()
g_all = T.grad(error, w_all)

upd = upd_dict(w_all, g_all, lr)
fn = theano.function(all_args, error, updates=upd, allow_input_downcast=True)

err = 0.0; l_rate = 0.1
for i in xrange(len(X)):
    err += fn(np.zeros(n_hidden), X[i], Y[i], l_rate)
    if i % 1000 == 999:
        l_rate *= 0.95
        print("Error:{:.2f}".format(err/1000))
        err = 0.0


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
IV. Keras GRU
Gated Recurrent Unit
Can you write Python code for it?

# you can implement a GRU in Keras by just replacing `SimpleRNN` with `GRU`:
model = Sequential([
            GRU(n_hidden, return_sequences=True, input_shape=(cs, vocab_size),
                   activation='relu', inner_init='identity'),
            TimeDistributed(Dense(vocab_size, activation='softmax')),
        ])
model.compile(loss='categorical_crossentropy', optimizer=Adam())
model.fit(oh_x_rnn, oh_y_rnn, batch_size = 64, nb_epoch=8)

def get_nexts_oh(inp):
    idxs = np.array([char_indices[c] for c in inp])
    arr = to_categorical(idxs, vocab_size)
    p = model.predict(arr[np.newaxis,:])[0]
    print(list(inp))
    return [chars[np.argmax(o)] for o in p]

get_nexts_oh(' this is')


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
Set up training:
Building the functions to do the Forward & Backward passes of our RNN. 1st, define data & shape:
Can you write Python code for it?

# 8-char seqs 1Hencoded
inp = oh_x_rnn
# 8-char seqs 1Henc moved aXs by 1
outp = oh_y_rnn

n_input = vocab_size; n_output = vocab_size; n_hidden = 256
inp.shape, outp.shape


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
Some tests: Check manual answers same as Theano:
Can you write Python code for it?

test_preds = np.array([0.2, 0.7, 0.1])
test_actuals = np.array([0.,1.,0.])
# Theano
nnet.categorical_crossentropy(test_preds, test_actuals).eval()

# Manual
x_entropy(test_preds, test_actuals)

test_inp = T.dvector()
test_out = nnet.categorical_crossentropy(test_inp, test_actuals)
test_grad = theano.function([test_inp], T.grad(test_out, test_inp))
# Theano
test_grad(test_preds)

# Manual
x_entropy_d(test_preds, test_actuals)

loss = x_entropy
loss_d = x_entropy_d
act = relu
act_d = relu_d

pre_pred = random(oh_x_rnn[0][0].shape)
preds = softmax(pre_pred)
actual = oh_x_rnn[0][0]

np.allclose(softmax_d(pre_pred).dot(loss_d(preds, actual)), preds - actual)

print (softmax(test_preds))
print (nnet.softmax(test_preds).eval())

print()

test_out = T.flatten(nnet.softmax(test_inp))
test_grad = theano.function([test_inp], theano.gradient.jacobian(test_out, test_inp))

print(test_grad(test_preds))
print()
print(softmax_d(test_preds))


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
The Step function is nearly identical to before, except that it multiplies the hidden state by the reset gate, and updates the hidden state based on the update gate.
Can you write Python code for it?

def step(x, h, W_h, b_h, W_y, b_y,
         uW_x, ub_x, rW_x, rb_x, W_x, uW_h, rW_h):
    reset = gate(x, h, rW_h, rW_x, rb_x) # reset gate is little NN
    update = gate(x, h, uW_h, uW_x, ub_x)
    h_new = gate(x, h * reset, W_h, W_x, b_h) # hidden states X by reset gate
    h = update * h + (1 - update) * h_new # actual new hidden state
    y = nnet.softmax(T.dot(h, W_y) + b_y) # output = hidden state X hidden weight matrix + the biases
    return h, T.flatten(y, 1)
# update gate decides how much h_new will replace h


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
The functino to do a single forward pass of an RNN, for a single character:
Can you write Python code for it?

# Forward Pass
def one_char(prev, item):
    # Previous state:
    tot_loss, pre_hidden, pre_pred, hidden, ypred = prev
    # Current inputs and output
    x, y = item
    pre_hidden = np.dot(x, w_x) + np.dot(hidden, w_h)
    hidden = act(pre_hidden)
    pre_pred = np.dot(hidden, w_y)
    ypred = softmax(pre_pred)
    return (
        # Keep track of loss so we can report it
        tot_loss + loss(ypred, y),
        # Used in Backprop
        pre_hidden, pre_pred,
        # Used in next iteration
        hidden,
        # to provide predictions
        ypred)

# We use scan to apply the above to a whole sequences of characters:
def get_chars(n): return zip(inp[n], outp[n])
# scan thru all chars in nth phrase, the input & output, calling some function
def one_fwd(n): return scan(one_char, (0,0,0,np.zeros(n_hidden),0), get_chars(n))


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
The key step done automatically above by Theano is gradients calculation.

III. Python RNN
Will need to define all functions used:
Can you write Python code for it?

def sigmoid(x): return 1 / (1 + np.exp(-x))
def sigmoid_d(x):
    output = sigmoid(x)
    return output * (1 - output)

def relu(x): return np.maximum(0., x)
def relu_d(x): return (x > 0.) * 1.

# relu fn test
relu(np.array([3.,-3.])), relu_d(np.array([3.,-3.]))

# Euclidean distance
def dist(a, b): return pow(a - b, 2)
def dist_d(a, b): return 2 * (a - b)

import pdb

# CrossEntropy
eps = 1e-7
def x_entropy(pred, actual):
    return -np.sum(actual * np.log(np.clip(pred, eps, 1 - eps))) # clipping for infs
def x_entropy_d(pred, actual): return -actual / pred

def softmax(x): return np.exp(x) / np.exp(x).sum()
def softmax_d(x):
    sm = softmax(x)
    res = np.expand_dims(-sm, -1) * sm
    res[np.diag_indices_from(res)] = sm * (1 - sm)
    return res


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
The loop looks much like the Theano loop in the prevs section, except that we have to call the backwards step manually:
Can you write Python code for it?

overallError = 0
alpha = 0.0001
for n in range(10000):
    # run 1 F Step
    res = one_fwd(n)
    overallError += res[-1][0]
    # run 1 B Step
    deriv = one_bkwd(res, n)
    # print update
    if (n % 1000 == 999):
        print ("Error:{:.4f}; Gradient:{:.5f}".format(
                overallError/1000, np.linalg.norm(deriv)))
        overallError = 0

# Currently, the Forward Step is passing to scan an initial state of zeros every
# update. So it's resetting state on each step. To run statefully: just edit
# `one_fwd(n)` to return its final state and keep track of it to feed back into it
# the next time thru the loop. This however, will risk exploding Gradients &
# Activations unless implemented as an LSTM or GRU.


I am working on the file "FAI_old/Lesson7/TheanoRNN_PythonRNN_TheanoGRU.ipynb".
The first task is:
V. Theano GRU
Separate Weights
The Theano GRU looks like like the simple Theano RNN, except for the use of the reset and update gates. Each of these gates requires its own hidden & input weights, so we add those to our weight matrices.
Can you write Python code for it?

W_h = id_and_bias(n_hidden)
W_x = init_wgts(n_input, n_hidden)
W_y = wgts_and_bias(n_hidden, n_output)

# reset gate NN weights
rW_h = init_wgts(n_hidden, n_hidden)
rW_x = wgts_and_bias(n_input, n_hidden)

# update gate NN weights
uW_h = init_wgts(n_hidden, n_hidden)
uW_x = wgts_and_bias(n_input, n_hidden)

w_all = list(chain.from_iterable([W_h, W_y, uW_x, rW_x]))
w_all.extend([W_x, uW_h, rW_h])


I am working on the file "FAI_old/char_rnn_CodeAlong.ipynb".
The first task is:
In Lesson 6: improved: an RNN feeding into an RNN. See lecture at ~ 1:10:00
Can you write Python code for it?

model = Sequential([
            Embedding(vocab_size, n_fac, input_length=maxlen),
            LSTM(512, input_dim=n_fac, return_sequences=True, dropout_U=0.2, dropout_W=0.2,
                 consume_less='gpu'),
            Dropout(0.2),
            LSTM(512, return_sequences=True, dropout_U=0.2, dropout_W=0.2,
                 consume_less='gpu'),
            Dropout(0.2),
            TimeDistributed(Dense(vocab_size)),
            Activation('softmax')
        ])

model.compile(loss='sparse_categorical_crossentropy', optimizer=Adam())


I am working on the file "FAI_old/char_rnn_CodeAlong.ipynb".
The first task is:
Preprocess and create model
Can you write Python code for it?

maxlen = 40
sentences = []
next_chars = []
for i in xrange(0, len(idx) - maxlen + 1):
    sentences.append(idx[i: i + maxlen])
    next_chars.append(idx[i + 1: i + maxlen + 1])
print('nb sequences:', len(sentences))

sentences = np.concatenate([[np.array(o)] for o in sentences[:-2]])
next_chars = np.concatenate([[np.array(o)] for o in next_chars[:-2]])

sentences.shape, next_chars.shape

n_fac = 24


I am working on the file "FAI_old/char_rnn_CodeAlong.ipynb".
The first task is:
Setup
We haven't really looked into the detail of how this works yet - so this is provided for self-study for those who're interested. We'll look at it closely next week.
Can you write Python code for it?

path = get_file('nietzsche.txt', origin='https://s3.amazonaws.com/text-datasets/nietzsche.txt')
text = open(path).read().lower()
print('corpus length:', len(text))

!tail {path} -n25

# unique characters
chars = sorted(list(set(text)))
vocab_size = len(chars) + 1
print('total chars:', vocab_size)

chars.insert(0, "\0")

# the unique characters (in UpLoCase corpus, add 26)
''.join(chars[1:-6])

# create a mapping from char to index in which it appears
char_indices = dict((c, i) for i, c in enumerate(chars))
# create a mapping from index to char
indices_char = dict((i, c) for i, c in enumerate(chars))


I am working on the file "FAI_old/char_rnn_CodeAlong.ipynb".
The first task is:
Train
Can you write Python code for it?

def print_example():
    seed_string="ethics is a basic foundation of all that"
    for i in range(320):
        x=np.array([char_indices[c] for c in seed_string[-40:]])[np.newaxis,:]
        preds = model.predict(x, verbose=0)[0][-1]
        preds = preds/np.sum(preds)
        next_char = choice(chars, p=preds)
        seed_string = seed_string + next_char
    print(seed_string)

model.fit(sentences, np.expand_dims(next_chars, -1), batch_size=64, nb_epoch=1)

print_example()

model.fit(sentences, np.expand_dims(next_chars, -1), batch_size=64, nb_epoch=1)

print_example()

model.optimizer.lr=1e-3
model.fit(sentences, np.expand_dims(next_chars, -1), batch_size=128, nb_epoch=1)

print_example()

model.optimizer.lr=1e-4
model.fit(sentences, np.expand_dims(next_chars, -1), batch_size=256, nb_epoch=1)

print_example()

%mkdir -p 'data/char_rnn/'
model.save_weights('data/char_rnn.h5')

model.optimizer.lr=1e-5
model.fit(sentences, np.expand_dims(next_chars, -1), batch_size=256, nb_epoch=1)

print_example()

model.fit(sentences, np.expand_dims(next_chars, -1), batch_size=128, nb_epoch=1)

print_example()

print_example()

model.save_weights('data/char_rnn.h5')

def print_example(seed_string=''):
    if not seed_string:
        seed_string="ethics is a basic foundation of all that"
    for i in range(320):
        x=np.array([char_indices[c] for c in seed_string[-40:]])[np.newaxis,:]
        preds = model.predict(x, verbose=0)[0][-1]
        preds = preds/np.sum(preds)
        next_char = choice(chars, p=preds)
        seed_string = seed_string + next_char
    print(seed_string)

text ='so um first i was afraid i was petrified'
print_example(text)


I am working on the file "FAI_old/char_rnn_CodeAlong.ipynb".
The first task is:
^ This allows us to take the text & convert into a list of numbers, where the number represents the index in which the char appears in the unique-char list.
Can you write Python code for it?

idx = [char_indices[c] for c in text]
idx[:10]

''.join(indices_char[i] for i in idx[:70])


I am working on the file "FAI_old/conv_test_Asus.ipynb".
The first task is:
As expected (& which motivated this) the full set of convolutional test features does not fit at once in memory.
Can you write Python code for it?

fname = path + 'results/conv_test_feat.dat'

x = bcolz.open(fname)
len(x)


I am working on the file "FAI_old/conv_test_Asus.ipynb".
The first task is:
Loading train/valid features; defining & fitting NN model
Can you write Python code for it?

# conv_train_feat_batches = get_batches(path + '/results/conv_feat.dat')
# conv_valid_feat_batches = get_batches(path + '/results/conv_val_feat.dat')
conv_trn_feat = load_array(path + '/results/conv_feat.dat')
conv_val_feat = load_array(path + '/results/conv_val_feat.dat')

(val_classes, trn_classes, val_labels, trn_labels,
    val_filenames, filenames, test_filenames) = get_classes(path)

p = 0.8
bn_model = Sequential([
        MaxPooling2D(input_shape=conv_layers[-1].output_shape[1:]),
        Flatten(),
        Dropout(p/2),
        Dense(128, activation='relu'),
        BatchNormalization(),
        Dropout(p/2),
        Dense(128, activation='relu'),
        BatchNormalization(),
        Dropout(p),
        Dense(10, activation='softmax')
        ])
bn_model.compile(Adam(lr=1e-3), loss='categorical_crossentropy', metrics=['accuracy'])

# Sequential.fit_generator(self, generator, samples_per_epoch, nb_epoch, verbose=1, callbacks=None, validation_data=None, nb_val_samples=None, class_weight=None, max_q_size=10, nb_worker=1, pickle_safe=False, initial_epoch=0, **kwargs)
# bn_model.fit_generator((conv_train_feat_batches, trn_labels), conv_train_feat_batches.nb_sample, nb_epoch=1,
#                        validation_data=(conv_valid_feat_batches, val_labels), nb_val_samples=conv_valid_feat_batches.nb_sample)

bn_model.fit(conv_trn_feat, trn_labels, batch_size=batch_size, nb_epoch=1,
             validation_data = (conv_val_feat, val_labels))

bn_model.optimizer.lr=1e-2
bn_model.fit(conv_trn_feat, trn_labels, batch_size=batch_size, nb_epoch=4,
             validation_data = (conv_val_feat, val_labels))

# bn_model.save_weights(path + 'models/da_conv8.h5')
bn_model.load_weights(path + 'models/da_conv8.h5')

# conv_test_feat_batches = bcolz.iterblocks(path + fname)
fname = path + 'results/conv_test_feat.dat'
idx, inc = 0, 4096
preds = []
while idx < test_batches.n - inc:
    conv_test_feat = bcolz.open(fname)[idx:idx+inc]
    idx += inc
    if len(preds):
        next_preds = bn_model.predict(conv_test_feat,...


I am working on the file "FAI_old/conv_test_Asus.ipynb".
The first task is:
Made a mistake on the last loop above. The penultimate batch -- the last full 4096-image batch -- was added onto the end of the predictions array twice. The final 2194 image predictions were never run.
Easy enough to fix: modify the above code to work perfectly. Then either:
* create entirely new predictions from scratch (~ 1 hour)
* remove the last increment (4096) of predictions from the array, and add the last batch.
Gonna take option 2.
EDIT:
actually, option 1. preds was stored in memory, which was erased when I closed this machine for the night. So this time I'll just build the predictions array properly.
Below is testing/debugging output from the night before
Can you write Python code for it?

print(81920 - 79726)
print(79726 % 4096)
print(81920 % 4096) # <-- that's yeh problem right there, kid

x = preds[len(preds) - 4096]
print(preds[-1])
print(x)

preds[0]

# ??image.ImageDataGenerator.flow_from_directory

# ??Sequential.predict()


I am working on the file "FAI_old/conv_test_Asus.ipynb".
The first task is:
Manual iteration through test image to generate convolutional test features. Saves each batch to disk insetad of loading in memory.
Can you write Python code for it?

# conv_test_feat = conv_model.predict_generator(test_batches, test_batches.nb_sample)


I am working on the file "FAI_old/conv_test_Asus.ipynb".
The first task is:
Oh I forgot, predictions through a FC NN are fast. CNNs are where it takes a long time.
This is just quick testing that it works. Full/polished will be in the reworked statefarm-codealong (or just statefarm) JNB:
Can you write Python code for it?

def do_clip(arr, mx): return np.clip(arr, (1-mx)/9, mx)

subm = do_clip(preds, 0.93)

subm_name = path + 'results/subm01.gz'

trn_batches = get_batches(path + 'train', batch_size=batch_size, shuffle=False)

# make sure training batches defined before this:
classes = sorted(trn_batches.class_indices, key=trn_batches.class_indices.get)

import pandas as pd
submission = pd.DataFrame(subm, columns=classes)
submission.insert(0, 'img', [f[8:] for f in test_filenames])
submission.head()

submission.to_csv(subm_name, index=False, compression='gzip')

from IPython.display import FileLink
FileLink(subm_name)


I am working on the file "FAI_old/lesson1/Lesson1_recode.ipynb".
The first task is:
Create a VGG model from scratch in Keras
Model Setup
Can you write Python code for it?

from numpy.random import random, permutation
from scipy import misc, ndimage
from scipy.ndimage.interpolation import zoom

import keras
from keras import backend as K
from keras.utils.data_utils import get_file
from keras.models import Sequential, Model
from keras.layers.core import Flatten, Dense, Dropout, Lambda
from keras.layers import Input
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.optimizers import SGD, RMSprop
from keras.preprocessing import image

FILES_PATH = 'http://files.fast.ai/models/'; CLASS_FILE='imagenet_class_index.json'
# Keras' get_file () is a handy function that downloads files, and caches them for reuse later
fpath = get_file(CLASS_FILE, FILES_PATH+CLASS_FILE, cache_subdir='models')
with open(fpath) as f: class_dict = json.load(f)
# Convert dictionary with string indices into an array
classes = [class_dict[str(i)][1] for i in range(len(class_dict))]

classes[:5]


I am working on the file "FAI_old/lesson1/Lesson1_recode.ipynb".
The first task is:
Model Creation
Can you write Python code for it?

def ConvBlock(layers, model, filters):
    for i in range(layers):
        model.add(ZeroPadding2D((1,1)))
        model.add(Convolution2D(filters, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2,2), strides=(2,2)))
def FCBlock(model):
    model.add(Dense(4096, activation='relu'))
    model.add(Dropout(0.5))

# Mean of each channel as provided by VGG researchers
vgg_mean = np.array([123.68, 116.779, 103.939]).reshape((3,1,1))
def vgg_preprocess(x):
    x = x - vgg_mean     # subtract mean
    return x[:, ::-1]    # reverse axis bgr->rgb

def VGG_16():
    model = Sequential()
    model.add(Lambda(vgg_preprocess, input_shape=(3,224,224)))

    ConvBlock(2, model, 64)
    ConvBlock(2, model, 128)
    ConvBlock(3, model, 256)
    ConvBlock(3, model, 512)
    ConvBlock(3, model, 512)

    model.add(Flatten())
    FCBlock(model)
    FCBlock(model)
    model.add(Dense(1000, activation='softmax'))
    return model

model = VGG_16()

def ConvBlock(layers, model, filters):
    for i in range(layers):
        model.add(ZeroPadding2D((1, 1)))
        model.add(Convolution2D(filters, 3, 3, activation='relu'))
    model.add(MaxPooling2D((2, 2), strides=(2, 2)))

def FCBlock(model):
    model.add(Dense(4096, activation='relu'))
    model.add(Dropout(0.5))

def VGG_16():
    model = Sequential()
    model.add(Lambda(vgg_preprocess, input_shape=(3,224,224), output_shape=(3,224,224)))

    ConvBlock(2, model, 64)
    ConvBlock(2, model, 128)
    ConvBlock(3, model, 256)
    ConvBlock(3, model, 512)
    ConvBlock(3, model, 512)

    model.add(Flatten())
    FCBlock(model)
    FCBlock(model)
    model.add(Dense(1000, activation='softmax'))

    # fname = 'vgg16.h5'
    # model.load_weights(get_file(fname, self.FILE_PATH+fname, cache_subdir='models'))

model = VGG_16()


I am working on the file "FAI_old/lesson1/dogs_cats_redux.ipynb".
The first task is:
Action Plan

Create Validation and Sample sets
Rearrange image files into their respective directories
Finetune and Train model
Generate predictions
Validate predictions
Submit predictions to Kaggle

Create validation set and sample
Can you write Python code for it?

#Create directories
%cd $DATA_HOME_DIR
%mkdir valid
%mkdir results
%mkdir -p sample/train
%mkdir -p sample/test
%mkdir -p sample/valid
%mkdir -p sample/results
%mkdir -p test/unknown

%cd $DATA_HOME_DIR/train

g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(2000): os.rename(shuf[i], DATA_HOME_DIR+'/valid/' + shuf[i])

from shutil import copyfile

g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(200): copyfile(shuf[i], DATA_HOME_DIR+'/sample/train/' + shuf[i])

%cd $DATA_HOME_DIR/valid

g = glob('*.jpg')
shuf = np.random.permutation(g)
for i in range(50): copyfile(shuf[i], DATA_HOME_DIR+'/sample/valid/' + shuf[i])