# Written by Dr Daniel Buscombe, Marda Science LLC # for the SandSnap Program # # MIT License # # Copyright (c) 2020-2021, Marda Science LLC # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. ##> Release v1.4 (Aug 2021) from imports import * from matplotlib import MatplotlibDeprecationWarning import warnings warnings.filterwarnings(action="ignore",category=MatplotlibDeprecationWarning) ###=================================================== ## FUNCTIONS FOR LEARNING RATE SCHEDULER def exponential_decay(lr0, s): def exponential_decay_fn(epoch): return lr0 * 0.1 **(epoch / s) return exponential_decay_fn ###=================================================== ## IMAGE AUGMENTATION FUNCTIONS (for DO_AUG=True) # def h_flip(image): # return np.fliplr(image) def v_flip(image): return np.flipud(image) def warp_shift(image): shift= random.randint(25,200) transform = AffineTransform(translation=(0,shift)) warp_image = warp(image, transform, mode="wrap") return warp_image def apply_aug(im): return [im,v_flip(warp_shift(im))] #, clockwise_rotation(im), h_flip(im)] ##======================================================== def rescale(dat, mn, mx): ''' rescales an input dat between mn and mx ''' m = min(dat.flatten()) M = max(dat.flatten()) return (mx-mn)*(dat-m)/(M-m)+mn def do_standardize(img): #standardization using adjusted standard deviation N = np.shape(img)[0] * np.shape(img)[1] s = np.maximum(np.std(img), 1.0/np.sqrt(N)) m = np.mean(img) img = (img - m) / s img = rescale(img, 0, 1) del m, s, N return img ###=================================================== ### IMAGE BATCH GENERATOR FUNCTIONS def get_data_generator_Nvars_siso_simo(df, indices, for_training, vars, batch_size, greyscale, do_aug,#CS, standardize, tilesize): """ This function generates data for a batch of images and N associated metrics """ ##print(do_aug) if len(vars)==1: images, p1s = [], [] elif len(vars)==2: images, p1s, p2s = [], [], [] elif len(vars)==3: images, p1s, p2s, p3s = [], [], [], [] elif len(vars)==4: images, p1s, p2s, p3s, p4s = [], [], [], [], [] elif len(vars)==5: images, p1s, p2s, p3s, p4s, p5s = [], [], [], [], [], [] elif len(vars)==6: images, p1s, p2s, p3s, p4s, p5s, p6s =\ [], [], [], [], [], [], [] elif len(vars)==7: images, p1s, p2s, p3s, p4s, p5s, p6s, p7s =\ [], [], [], [], [], [], [], [] elif len(vars)==8: images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s =\ [], [], [], [], [], [], [], [], [] elif len(vars)==9: images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s =\ [], [], [], [], [], [], [], [], [], [] while True: for i in indices: r = df.iloc[i] if len(vars)==1: file, p1 = r['filenames'], r[vars[0]] if len(vars)==2: file, p1, p2 = r['filenames'], r[vars[0]], r[vars[1]] if len(vars)==3: file, p1, p2, p3 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]] if len(vars)==4: file, p1, p2, p3, p4 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]] if len(vars)==5: file, p1, p2, p3, p4, p5 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]] if len(vars)==6: file, p1, p2, p3, p4, p5, p6 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]] if len(vars)==7: file, p1, p2, p3, p4, p5, p6, p7 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]] if len(vars)==8: file, p1, p2, p3, p4, p5, p6, p7, p8 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]] elif len(vars)==9: file, p1, p2, p3, p4, p5, p6, p7, p8, p9 = \ r['filenames'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]], r[vars[8]] if greyscale==True: im = Image.open(file).convert('LA') #im = im.resize((IM_HEIGHT, IM_HEIGHT)) im = np.array(im)[:,:,0] nx,ny = np.shape(im) if (nx!=tilesize) or (ny!=tilesize): im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)] else: im = Image.open(file) #im = im.resize((IM_HEIGHT, IM_HEIGHT)) im = np.array(im) nx,ny,nz = np.shape(im) if (nx!=tilesize) or (ny!=tilesize): im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)] if standardize==True: im = do_standardize(im) else: im = np.array(im) / 255.0 #if np.ndim(im)==2: # im = np.dstack((im, im , im)) ##np.expand_dims(im[:,:,0], axis=2) #im = im[:,:,:3] if greyscale==True: if do_aug==True: aug = apply_aug(im) images.append(aug) if len(vars)==1: p1s.append([p1 for k in range(2)]) elif len(vars)==2: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) elif len(vars)==3: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); elif len(vars)==4: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) elif len(vars)==5: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); elif len(vars)==6: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) elif len(vars)==7: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) p7s.append([p7 for k in range(2)]); elif len(vars)==8: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)]) elif len(vars)==9: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)]) p9s.append([p9 for k in range(2)]) else: images.append(np.expand_dims(im, axis=2)) if len(vars)==1: p1s.append(p1) elif len(vars)==2: p1s.append(p1); p2s.append(p2) elif len(vars)==3: p1s.append(p1); p2s.append(p2) p3s.append(p3); elif len(vars)==4: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) elif len(vars)==5: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); elif len(vars)==6: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) elif len(vars)==7: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) p7s.append(p7); elif len(vars)==8: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) p7s.append(p7); p8s.append(p8) elif len(vars)==9: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) p7s.append(p7); p8s.append(p8) p9s.append(p9) else: if do_aug==True: aug = apply_aug(im) images.append(aug) if len(vars)==1: p1s.append([p1 for k in range(2)]) elif len(vars)==2: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) elif len(vars)==3: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); elif len(vars)==4: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) elif len(vars)==5: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); elif len(vars)==6: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) elif len(vars)==7: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) p7s.append([p7 for k in range(2)]); elif len(vars)==8: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)]) elif len(vars)==9: p1s.append([p1 for k in range(2)]); p2s.append([p2 for k in range(2)]) p3s.append([p3 for k in range(2)]); p4s.append([p4 for k in range(2)]) p5s.append([p5 for k in range(2)]); p6s.append([p6 for k in range(2)]) p7s.append([p7 for k in range(2)]); p8s.append([p8 for k in range(2)]) p9s.append([p9 for k in range(2)]) else: images.append(im) if len(vars)==1: p1s.append(p1) elif len(vars)==2: p1s.append(p1); p2s.append(p2) elif len(vars)==3: p1s.append(p1); p2s.append(p2) p3s.append(p3); elif len(vars)==4: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) elif len(vars)==5: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); elif len(vars)==6: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) elif len(vars)==7: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) p7s.append(p7); elif len(vars)==8: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) p7s.append(p7); p8s.append(p8) elif len(vars)==9: p1s.append(p1); p2s.append(p2) p3s.append(p3); p4s.append(p4) p5s.append(p5); p6s.append(p6) p7s.append(p7); p8s.append(p8) p9s.append(p9) if len(images) >= batch_size: if len(vars)==1: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) yield images,[p1s] else: if len(images) >= batch_size: #p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) yield np.array(images),[np.array(p1s)] images, p1s = [], [] elif len(vars)==2: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) yield images,[p1s, p2s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s)] images, p1s, p2s = [], [], [] elif len(vars)==3: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) yield images,[p1s, p2s, p3s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s)] images, p1s, p2s, p3s = [], [], [], [] elif len(vars)==4: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) p4s = np.squeeze(np.array(p4s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1) yield images,[p1s, p2s, p3s, p4s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s)] images, p1s, p2s, p3s, p4s = [], [], [], [], [] elif len(vars)==5: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) p4s = np.squeeze(np.array(p4s)) p5s = np.squeeze(np.array(p5s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1) p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1) yield images,[p1s, p2s, p3s, p4s, p5s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s), np.array(p5s)] images, p1s, p2s, p3s, p4s, p5s = [], [], [], [], [], [] elif len(vars)==6: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) p4s = np.squeeze(np.array(p4s)) p5s = np.squeeze(np.array(p5s)) p6s = np.squeeze(np.array(p6s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1) p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1) p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1) yield images,[p1s, p2s, p3s, p4s, p5s, p6s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s), np.array(p5s), np.array(p6s)] images, p1s, p2s, p3s, p4s, p5s, p6s = \ [], [], [], [], [], [], [] elif len(vars)==7: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) p4s = np.squeeze(np.array(p4s)) p5s = np.squeeze(np.array(p5s)) p6s = np.squeeze(np.array(p6s)) p7s = np.squeeze(np.array(p7s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1) p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1) p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1) p7s = np.expand_dims(np.vstack(p7s).flatten(),axis=-1) yield images,[p1s, p2s, p3s, p4s, p5s, p6s, p7s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s), np.array(p5s), np.array(p6s), np.array(p7s)] images, p1s, p2s, p3s, p4s, p5s, p6s, p7s = \ [], [], [], [], [], [], [], [] elif len(vars)==8: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) p4s = np.squeeze(np.array(p4s)) p5s = np.squeeze(np.array(p5s)) p6s = np.squeeze(np.array(p6s)) p7s = np.squeeze(np.array(p7s)) p8s = np.squeeze(np.array(p8s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1))) # p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1))) if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1) p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1) p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1) p7s = np.expand_dims(np.vstack(p7s).flatten(),axis=-1) p8s = np.expand_dims(np.vstack(p8s).flatten(),axis=-1) yield images,[p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s), np.array(p5s), np.array(p6s), np.array(p7s), np.array(p8s)] images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s = \ [], [], [], [], [], [], [], [], [] elif len(vars)==9: # if len(CS)==0: p1s = np.squeeze(np.array(p1s)) p2s = np.squeeze(np.array(p2s)) p3s = np.squeeze(np.array(p3s)) p4s = np.squeeze(np.array(p4s)) p5s = np.squeeze(np.array(p5s)) p6s = np.squeeze(np.array(p6s)) p7s = np.squeeze(np.array(p7s)) p8s = np.squeeze(np.array(p8s)) p9s = np.squeeze(np.array(p9s)) # else: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1))) # p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1))) # p9s = np.squeeze(CS[8].transform(np.array(p9s).reshape(-1, 1))) try: if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) p1s = np.expand_dims(np.vstack(p1s).flatten(),axis=-1) p2s = np.expand_dims(np.vstack(p2s).flatten(),axis=-1) p3s = np.expand_dims(np.vstack(p3s).flatten(),axis=-1) p4s = np.expand_dims(np.vstack(p4s).flatten(),axis=-1) p5s = np.expand_dims(np.vstack(p5s).flatten(),axis=-1) p6s = np.expand_dims(np.vstack(p6s).flatten(),axis=-1) p7s = np.expand_dims(np.vstack(p7s).flatten(),axis=-1) p8s = np.expand_dims(np.vstack(p8s).flatten(),axis=-1) p9s = np.expand_dims(np.vstack(p9s).flatten(),axis=-1) yield images,[p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s] else: if len(images) >= batch_size: yield np.array(images),[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s), np.array(p5s), np.array(p6s), np.array(p7s), np.array(p8s), np.array(p9s)] except GeneratorExit: print("") images, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s = \ [], [], [], [], [], [], [], [], [], [] if not for_training: break ###=================================================== def get_data_generator_1image(df, indices, for_training, ID_MAP, var, batch_size, greyscale, do_aug, standardize, tilesize): """ This function creates a dataset generator consisting of batches of images and corresponding one-hot-encoded labels describing the sediment in each image """ try: ID_MAP2 = dict((g, i) for i, g in ID_MAP.items()) except: ID_MAP = dict(zip(np.arange(ID_MAP), [str(k) for k in range(ID_MAP)])) ID_MAP2 = dict((g, i) for i, g in ID_MAP.items()) images, pops = [], [] while True: for i in indices: r = df.iloc[i] file, pop = r['filenames'], r[var] # if greyscale==True: # im = Image.open(file).convert('LA') # else: # im = Image.open(file) # im = im.resize((IM_HEIGHT, IM_HEIGHT)) # im = np.array(im) / 255.0 if greyscale==True: im = Image.open(file).convert('LA') #im = im.resize((IM_HEIGHT, IM_HEIGHT)) im = np.array(im)[:,:,0] nx,ny = np.shape(im) if (nx!=tilesize) or (ny!=tilesize): im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)] else: im = Image.open(file) #im = im.resize((IM_HEIGHT, IM_HEIGHT)) im = np.array(im) nx,ny,nz = np.shape(im) if (nx!=tilesize) or (ny!=tilesize): im = im[int(nx/2)-int(tilesize/2):int(nx/2)+int(tilesize/2), int(ny/2)-int(tilesize/2):int(ny/2)+int(tilesize/2)] if standardize==True: im = do_standardize(im) # if np.ndim(im)==2: # im = np.dstack((im, im , im)) ##np.expand_dims(im[:,:,0], axis=2) # im = im[:,:,:3] if greyscale==True: if do_aug==True: aug = apply_aug(im[:,:,0]) images.append(aug) pops.append([to_categorical(pop, len(ID_MAP2)) for k in range(2)]) #3 else: images.append(np.expand_dims(im[:,:,0], axis=2)) else: if do_aug==True: aug = apply_aug(im) images.append(aug) pops.append([to_categorical(pop, len(ID_MAP2)) for k in range(2)]) else: images.append(im) pops.append(to_categorical(pop, len(ID_MAP2))) try: if do_aug==True: if len(images) >= batch_size: if greyscale==False: images = np.array(np.vstack(images)) pops = np.array(np.vstack(pops)) else: images = np.expand_dims(np.array(np.vstack(images)), axis=-1) pops = np.array(np.vstack(pops)) yield images, pops images, pops = [], [] else: if len(images) >= batch_size: yield np.squeeze(np.array(images)),np.array(pops) #[np.array(pops)] images, pops = [], [] except GeneratorExit: print("") #pass if not for_training: break ###=================================================== ### PLOT TRAINING HISTORY FUNCTIONS def plot_train_history_1var(history): """ This function plots loss and accuracy curves from the model training """ fig, axes = plt.subplots(1, 2, figsize=(10, 10)) print(history.history.keys()) axes[0].plot(history.history['loss'], label='Training loss') axes[0].plot(history.history['val_loss'], label='Validation loss') axes[0].set_xlabel('Epochs') axes[0].legend() try: axes[1].plot(history.history['acc'], label='pop train accuracy') axes[1].plot(history.history['val_acc'], label='pop test accuracy') except: axes[1].plot(history.history['accuracy'], label='pop train accuracy') axes[1].plot(history.history['val_accuracy'], label='pop test accuracy') axes[1].set_xlabel('Epochs') axes[1].legend() ###=================================================== def plot_train_history_Nvar(history, varuse, N): """ This function makes a plot of error train/validation history for 9 variables, plus overall loss functions """ fig, axes = plt.subplots(1, N+1, figsize=(20, 5)) for k in range(N): try: axes[k].plot(history.history[varuse[k]+'_output_mean_absolute_error'], label=varuse[k]+' Train MAE') axes[k].plot(history.history['val_'+varuse[k]+'_output_mean_absolute_error'], label=varuse[k]+' Val MAE') except: axes[k].plot(history.history[varuse[k]+'_output_mae'], label=varuse[k]+' Train MAE') axes[k].plot(history.history['val_'+varuse[k]+'_output_mae'], label=varuse[k]+' Val MAE') axes[k].set_xlabel('Epochs') axes[k].legend() axes[N].plot(history.history['loss'], label='Training loss') axes[N].plot(history.history['val_loss'], label='Validation loss') axes[N].set_xlabel('Epochs') axes[N].legend() ###=================================================== def plot_train_history_1var_mae(history): """ This function plots loss and accuracy curves from the model training """ print(history.history.keys()) fig, axes = plt.subplots(1, 2, figsize=(10, 10)) axes[0].plot(history.history['loss'], label='Training loss') axes[0].plot(history.history['val_loss'], label='Validation loss') axes[0].set_xlabel('Epochs') axes[0].legend() try: axes[1].plot(history.history['mean_absolute_error'], label='pop train MAE') axes[1].plot(history.history['val_mean_absolute_error'], label='pop test MAE') except: axes[1].plot(history.history['mae'], label='pop train MAE') axes[1].plot(history.history['val_mae'], label='pop test MAE') axes[1].set_xlabel('Epochs') axes[1].legend() ###=================================================== ### PLOT CONFUSION MATRIX FUNCTIONS ###=================================================== def plot_confusion_matrix(cm, classes, normalize=False, cmap=plt.cm.Purples, dolabels=True): """ This function prints and plots the confusion matrix. Normalization can be applied by setting `normalize=True`. """ if normalize: cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] cm[np.isnan(cm)] = 0 plt.imshow(cm, interpolation='nearest', cmap=cmap, vmax=1, vmin=0) fmt = '.2f' if normalize else 'd' thresh = cm.max() / 2. if dolabels==True: tick_marks = np.arange(len(classes)) plt.xticks(tick_marks, classes, fontsize=3) #, rotation=60 plt.yticks(tick_marks, classes, fontsize=3) plt.ylabel('True label',fontsize=4) plt.xlabel('Estimated label',fontsize=4) else: plt.axis('off') for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])): if cm[i, j]>0: plt.text(j, i, str(cm[i, j])[:4],fontsize=5, horizontalalignment="center", color="white" if cm[i, j] > 0.6 else "black") #plt.tight_layout() plt.xlim(-0.5, len(classes)) plt.ylim(-0.5, len(classes)) return cm ###=================================================== def plot_confmat(y_pred, y_true, prefix, classes): """ This function generates and plots a confusion matrix """ base = prefix+'_' y = y_pred.copy() del y_pred l = y_true.copy() del y_true l = l.astype('float') ytrue = l.flatten() ypred = y.flatten() ytrue = ytrue[~np.isnan(ytrue)] ypred = ypred[~np.isnan(ypred)] cm = confusion_matrix(ytrue, ypred) cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis] cm[np.isnan(cm)] = 0 fig=plt.figure() plt.subplot(221) plot_confusion_matrix(cm, classes=classes) ###=================================================== ### PREDICTION FUNCTIONS def predict_test_train_cat(train_df, test_df, train_idx, test_idx, var, SM, classes, weights_path, greyscale, name, do_aug, tilesize): """ This function creates makes predictions on test and train data, prints a classification report, and prints confusion matrices """ if type(SM) == list: counter = 0 for s,wp in zip(SM, weights_path): exec('SM[counter].load_weights(wp)') counter += 1 else: SM.load_weights(weights_path) ##============================================== ## make predictions on training data for_training = False train_gen = get_data_generator_1image(train_df, train_idx, for_training, len(classes), var, len(train_idx), greyscale, #np.min((200, len(train_idx))), do_aug, standardize, tilesize) x_train, (trueT)= next(train_gen) PT = [] if type(SM) == list: #counter = 0 for s in SM: tmp=s.predict(x_train, batch_size=8) exec( 'PT.append(np.asarray(np.squeeze(tmp)))' ) del tmp predT = np.median(PT, axis=0) #predT = np.squeeze(np.asarray(PT)) del PT K.clear_session() gc.collect() else: predT = SM.predict(x_train, batch_size=8) #predT = np.asarray(predT).argmax(axis=-1) del train_gen, x_train if test_df is not None: ## make predictions on testing data for_training = False do_aug = False test_gen = get_data_generator_1image(test_df, test_idx, for_training, len(classes), var, len(test_idx), greyscale, #np.min((200, len(test_idx))), do_aug, standardize, tilesize) #no augmentation on validation data x_test, (true)= next(test_gen) PT = [] if type(SM) == list: #counter = 0 for s in SM: tmp=s.predict(x_test, batch_size=8) exec( 'PT.append(np.asarray(np.squeeze(tmp)))' ) del tmp pred = np.median(PT, axis=0) #pred = np.squeeze(np.asarray(PT)) del PT K.clear_session() gc.collect() else: pred = SM.predict(x_test, batch_size=8) #1) #pred = np.asarray(pred).argmax(axis=-1) del test_gen, x_test trueT = np.squeeze(np.asarray(trueT).argmax(axis=-1) ) predT = np.squeeze(np.asarray(predT).argmax(axis=-1))#[0]) if test_df is not None: pred = np.squeeze(np.asarray(pred).argmax(axis=-1))#[0]) true = np.squeeze(np.asarray(true).argmax(axis=-1) ) ##============================================== ## print a classification report to screen, showing f1, precision, recall and accuracy print("==========================================") print("Classification report for "+var) print(classification_report(true, pred)) fig = plt.figure() ##============================================== ## create figures showing confusion matrices for train and test data sets if type(SM) == list: if test_df is not None: plot_confmat(pred, true, var, classes) plt.savefig(weights_path[0].replace('.hdf5','_cm.png').\ replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))), dpi=300, bbox_inches='tight') plt.close('all') plot_confmat(predT, trueT, var+'T',classes) plt.savefig(weights_path[0].replace('.hdf5','_cmT.png').\ replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))), dpi=300, bbox_inches='tight') plt.close('all') else: if test_df is not None: plot_confmat(pred, true, var, classes) plt.savefig(weights_path.replace('.hdf5','_cm.png'), dpi=300, bbox_inches='tight') plt.close('all') plot_confmat(predT, trueT, var+'T',classes) plt.savefig(weights_path.replace('.hdf5','_cmT.png'), dpi=300, bbox_inches='tight') plt.close('all') plt.close() del fig ###=================================================== def predict_train_siso_simo(a, b, vars, SM, weights_path, name, mode, greyscale, dropout,do_aug, standardize,#CS,# scale, count_in): """ This function creates makes predcitions on test and train data """ ##============================================== ## make predictions on training data if type(SM) == list: counter = 0 for s,wp in zip(SM, weights_path): exec('SM[counter].load_weights(wp)') counter += 1 else: SM.load_weights(weights_path) # if scale == True: # # if len(vars)>1: # counter = 0 # for v in vars: # exec( # v+\ # '_trueT = np.squeeze(CS[counter].inverse_transform(b[counter].reshape(-1,1)))' # ) # counter +=1 # else: # exec( # vars[0]+\ # '_trueT = np.squeeze(CS[0].inverse_transform(b[0].reshape(-1,1)))' # ) # # else: if len(vars)>1: counter = 0 for v in vars: exec( v+\ '_trueT = np.squeeze(b[counter])' ) counter +=1 else: exec( vars[0]+\ '_trueT = np.squeeze(b)' ) del b for v in vars: exec(v+'_PT = []') # if scale == True: # # if type(SM) == list: # counter = 0 #model iterator # for s in SM: # train_vals=s.predict(a, batch_size=8) # # if len(vars)>1: # counter2 = 0 #variable iterator # for v in vars: # exec( # v+\ # '_PT.append(np.squeeze(CS[counter].inverse_transform(train_vals[counter2].reshape(-1,1))))' # ) # counter2 +=1 # else: # exec( # vars[0]+\ # '_PT.append(np.asarray(np.squeeze(CS[0].inverse_transform(train_vals.reshape(-1,1)))))' # ) # # del train_vals # # if len(vars)>1: # #counter = 0 # for v in vars: # exec( # v+\ # '_PT = np.median('+v+'_PT, axis=0)' # ) # #counter +=1 # else: # exec( # vars[0]+\ # '_PT = np.median('+v+'_PT, axis=0)' # ) # # else: # train_vals = SM.predict(a, batch_size=8) #128) # # if len(vars)>1: # counter = 0 # for v in vars: # exec( # v+\ # '_PT.append(np.squeeze(CS[counter].inverse_transform(train_vals[counter].reshape(-1,1))))' # ) # counter +=1 # else: # exec( # vars[0]+\ # '_PT.append(np.asarray(np.squeeze(CS[0].inverse_transform(train_vals.reshape(-1,1)))))' # ) # # del train_vals # # else: if type(SM) == list: #counter = 0 for s in SM: train_vals=s.predict(a, batch_size=8) if len(vars)>1: counter2 = 0 for v in vars: exec( v+\ '_PT.append(np.squeeze(train_vals[counter2]))' ) counter2 +=1 else: exec( vars[0]+\ '_PT.append(np.asarray(train_vals))' ) del train_vals if len(vars)>1: #counter = 0 for v in vars: exec( v+\ '_PT = np.median('+v+'_PT, axis=0)' ) #counter +=1 else: exec( vars[0]+\ '_PT = np.median('+v+'_PT, axis=0)' ) else: train_vals = SM.predict(a, batch_size=1)#8) #128) if len(vars)>1: counter = 0 for v in vars: exec( v+\ '_PT.append(np.squeeze(train_vals[counter]))' ) counter +=1 else: exec( vars[0]+\ '_PT.append(np.asarray(np.squeeze(train_vals)))' ) del train_vals if len(vars)>1: for k in range(len(vars)): exec(vars[k]+'_predT = np.squeeze(np.asarray('+vars[k]+'_PT))') else: exec(vars[0]+'_predT = np.squeeze(np.asarray('+vars[0]+'_PT))') for v in vars: exec('del '+v+'_PT') del a #train_gen, if len(vars)==9: nrows = 3; ncols = 3 elif len(vars)==8: nrows = 2; ncols = 4 elif len(vars)==7: nrows = 3; ncols = 3 elif len(vars)==6: nrows = 3; ncols = 2 elif len(vars)==5: nrows = 3; ncols = 2 elif len(vars)==4: nrows = 2; ncols = 2 elif len(vars)==3: nrows = 2; ncols = 2 elif len(vars)==2: nrows = 1; ncols = 2 elif len(vars)==1: nrows = 1; ncols = 1 out = dict() ## make a plot fig = plt.figure(figsize=(6*nrows,6*ncols)) labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' for k in range(1,1+(nrows*ncols)): # try: plt.subplot(nrows,ncols,k) x1 = eval(vars[k-1]+'_trueT') y1 = eval(vars[k-1]+'_predT') out[vars[k-1]+'_trueT'] = eval(vars[k-1]+'_trueT') out[vars[k-1]+'_predT'] = eval(vars[k-1]+'_predT') plt.plot(x1, y1, 'ko', markersize=5) plt.plot([ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], [ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], 'k', lw=2) plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),'Train : '+\ str(np.nanmean(100*(np.abs(y1-x1) / x1)))[:5]+\ ' %', fontsize=10) plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left') #varstring = ''.join([str(k)+'_' for k in vars]) varstring = str(len(vars))+'vars' # except: # pass if type(SM) == list: plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble'+str(count_in)+'.png').\ replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))), dpi=300, bbox_inches='tight') else: plt.savefig(weights_path.replace('.hdf5', '_skill'+str(count_in)+'.png'), dpi=300, bbox_inches='tight') plt.close() del fig np.savez_compressed(weights_path.replace('.hdf5', '_out'+str(count_in)+'.npz'),**out) del out if len(vars)==9: nrows = 3; ncols = 3 elif len(vars)==8: nrows = 2; ncols = 4 elif len(vars)==7: nrows = 3; ncols = 3 elif len(vars)==6: nrows = 3; ncols = 2 elif len(vars)==5: nrows = 3; ncols = 2 elif len(vars)==4: nrows = 2; ncols = 2 elif len(vars)==3: nrows = 2; ncols = 2 elif len(vars)==2: nrows = 1; ncols = 2 elif len(vars)==1: nrows = 1; ncols = 1 out = dict() ## make a plot fig = plt.figure(figsize=(6*nrows,6*ncols)) labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' for k in range(1,1+(nrows*ncols)): # try: plt.subplot(nrows,ncols,k) x1 = eval(vars[k-1]+'_trueT') y1 = eval(vars[k-1]+'_predT') out[vars[k-1]+'_trueT'] = eval(vars[k-1]+'_trueT') out[vars[k-1]+'_predT'] = eval(vars[k-1]+'_predT') plt.plot(x1, y1, 'ko', markersize=5) plt.plot([ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], [ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], 'k', lw=2) plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),'Train : '+\ str(np.nanmean(100*(np.abs(y1-x1) / x1)))[:5]+\ ' %', fontsize=10) plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left') #varstring = ''.join([str(k)+'_' for k in vars]) varstring = str(len(vars))+'vars' # except: # pass if type(SM) == list: plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble'+str(count_in)+'.png').\ replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))), dpi=300, bbox_inches='tight') else: plt.savefig(weights_path.replace('.hdf5', '_skill'+str(count_in)+'.png'), dpi=300, bbox_inches='tight') plt.close() del fig np.savez_compressed(weights_path.replace('.hdf5', '_out'+str(count_in)+'.npz'),**out) del out ###============================================================ def plot_all_save_all(weights_path, vars): if type(weights_path) == list: npz_files = glob(weights_path[0].replace('.hdf5', '*.npz')) else: npz_files = glob(weights_path.replace('.hdf5', '*.npz')) npz_files = [n for n in npz_files if '_all.npz' not in n] print("Found %i npz files "%(len(npz_files))) if len(vars)==9: nrows = 3; ncols = 3 elif len(vars)==8: nrows = 2; ncols = 4 elif len(vars)==7: nrows = 3; ncols = 3 elif len(vars)==6: nrows = 3; ncols = 2 elif len(vars)==5: nrows = 3; ncols = 2 elif len(vars)==4: nrows = 2; ncols = 2 elif len(vars)==3: nrows = 2; ncols = 2 elif len(vars)==2: nrows = 1; ncols = 2 elif len(vars)==1: nrows = 1; ncols = 1 ## make a plot fig = plt.figure(figsize=(6*nrows,6*ncols)) for counter,file in enumerate(npz_files): out = dict() with np.load(file, allow_pickle=True) as dat: for k in dat.keys(): out[k] = dat[k] labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' X = []; Y=[] Xt = []; Yt=[] for k in range(1,1+(nrows*ncols)): # try: plt.subplot(nrows,ncols,k) x1 = out[vars[k-1]+'_trueT'] y1 = out[vars[k-1]+'_predT'] X.append(x1.flatten()); Y.append(y1.flatten()) del x1, y1 x1 = np.array(X) y1 = np.array(Y) plt.plot(x1, y1, 'ko', markersize=5) if counter==len(npz_files)-1: plt.plot([ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], [ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], 'k', lw=2) plt.text(np.nanmin(x1), 0.8*np.max(np.hstack((x1,y1))),'Train : '+\ str(np.mean(100*(np.abs(y1-x1) / x1)))[:5]+\ ' %', fontsize=12, color='r') plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left') out[vars[k-1]+'_trueT'] = x1 #eval(vars[k-1]+'_trueT') out[vars[k-1]+'_predT'] = y1 #eval(vars[k-1]+'_predT') if counter==len(npz_files)-1: plt.plot([ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], [ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], 'k', lw=2) plt.text(np.nanmin(x1), 0.8*np.max(np.hstack((x1,y1))),'Train : '+\ str(np.mean(100*(np.abs(y1-x1) / x1)))[:5]+\ ' %', fontsize=12, color='r') try: plt.text(np.nanmin(x2), 0.8*np.max(np.hstack((x2,y2))),'Test : '+\ str(np.mean(100*(np.abs(y2-x2) / x2)))[:5]+\ ' %', fontsize=12, color='r') except: pass plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left') if type(weights_path) == list: plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble.png').\ replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))), dpi=300, bbox_inches='tight') else: plt.savefig(weights_path.replace('.hdf5', '_skill.png'), dpi=300, bbox_inches='tight') plt.close() del fig np.savez_compressed(weights_path.replace('.hdf5', '_out_all.npz'),**out) del out ###=================================================== ### MISC. UTILITIES def tidy(name,res_folder): """ This function moves training outputs to a specific folder """ pngfiles = glob('*'+name+'*.png') jsonfiles = glob('*'+name+'*.json') hfiles = glob('*'+name+'*.hdf5') tfiles = glob('*'+name+'*.txt') #pfiles = glob('*'+name+'*.pkl') nfiles = glob('*'+name+'*.npz') try: [shutil.move(k, res_folder) for k in pngfiles] [shutil.move(k, res_folder) for k in hfiles] [shutil.move(k, res_folder) for k in jsonfiles] [shutil.move(k, res_folder) for k in tfiles] #[shutil.move(k, res_folder) for k in pfiles] [shutil.move(k, res_folder) for k in nfiles] except: pass ###=================================================== def get_df(csvfile,fortrain=False): """ This function reads a csvfile with image names and labels and returns random indices """ ###=================================================== ## read the data set in, clean and modify the pathnames so they are absolute df = pd.read_csv(csvfile) num_split = 50 if fortrain==False: if len(df)>num_split: #print('Spliting into chunks') df = np.array_split(df, int(np.round(len(df)/num_split))) split = True else: split = False else: split = False if split: for k in range(len(df)): df[k]['filenames'] = [k.strip() for k in df[k]['filenames']] else: df['filenames'] = [k.strip() for k in df['filenames']] if split: for k in range(len(df)): df[k]['filenames'] = [os.getcwd()+os.sep+f.replace('\\',os.sep) for f in df[k]['filenames']] else: df['filenames'] = [os.getcwd()+os.sep+f.replace('\\',os.sep) for f in df['filenames']] np.random.seed(2021) if type(df)==list: idx = [np.random.permutation(len(d)) for d in df] else: idx = np.random.permutation(len(df)) return idx, df, split # # ###=================================================== # def predict_test_siso_simo(a, b, vars, # SM, weights_path, name, mode, greyscale, # CS, dropout, scale, do_aug, standardize, # count_in): # # # # # ## make predictions on testing data # # if d is not None: # # do_aug = False # # for_training = False # # # test_gen = get_data_generator_Nvars_siso_simo(test_df, test_idx, for_training, # # # vars, len(test_idx), greyscale, CS, do_aug, standardize) #np.min((200, len(test_idx))) # # # # # # x_test, vals = next(test_gen) # # # # if scale == True: # # # # if len(vars)>1: # # counter = 0 # # for v in vars: # # exec( # # v+\ # # '_true = np.squeeze(CS[counter].inverse_transform(d[counter].reshape(-1,1)))' # # ) # # counter +=1 # # else: # # exec( # # vars[0]+\ # # '_true = np.squeeze(CS[0].inverse_transform(d[0].reshape(-1,1)))' # # ) # # # # else: # # if len(vars)>1: # # counter = 0 # # for v in vars: # # exec( # # v+\ # # '_true = np.squeeze(d[counter])' # # ) # # counter +=1 # # else: # # exec( # # vars[0]+\ # # '_true = np.squeeze(d)' # # ) # # # # # # del d # # # # for v in vars: # # exec(v+'_P = []') # # # # if scale == True: # # # # if type(SM) == list: # # #counter = 0 # # for s in SM: # # test_vals=s.predict(c, batch_size=8) # # # # if len(vars)>1: # # counter = 0 # # for v in vars: # # exec( # # v+\ # # '_P.append(np.squeeze(CS[counter].inverse_transform(test_vals[counter].reshape(-1,1))))' # # ) # # counter +=1 # # else: # # exec( # # vars[0]+\ # # '_P.append(np.asarray(np.squeeze(CS[0].inverse_transform(test_vals.reshape(-1,1)))))' # # ) # # # # del test_vals # # # # if len(vars)>1: # # #counter = 0 # # for v in vars: # # exec( # # v+\ # # '_P = np.median('+v+'_P, axis=0)' # # ) # # #counter +=1 # # else: # # exec( # # vars[0]+\ # # '_P = np.median('+v+'_P, axis=0)' # # ) # # # # else: # # # # test_vals = SM.predict(c, batch_size=8) #128) # # if len(vars)>1: # # counter = 0 # # for v in vars: # # exec( # # v+\ # # '_P.append(np.squeeze(CS[counter].inverse_transform(test_vals[counter].reshape(-1,1))))' # # ) # # counter +=1 # # else: # # exec( # # vars[0]+\ # # '_P.append(np.asarray(np.squeeze(CS[0].inverse_transform(test_vals.reshape(-1,1)))))' # # ) # # # # del test_vals # # # # # # else: #no scale # # # # if type(SM) == list: # # counter = 0 # # for s in SM: # # test_vals=s.predict(c, batch_size=8) # # # # if len(vars)>1: # # counter = 0 # # for v in vars: # # exec( # # v+\ # # '_P.append(np.squeeze(test_vals[counter]))' # # ) # # counter +=1 # # else: # # exec( # # vars[0]+\ # # '_P.append(np.asarray(np.squeeze(test_vals)))' # # ) # # # # del test_vals # # # # if len(vars)>1: # # #counter = 0 # # for v in vars: # # exec( # # v+\ # # '_P = np.median('+v+'_P, axis=0)' # # ) # # #counter +=1 # # else: # # exec( # # vars[0]+\ # # '_P = np.median('+v+'_P, axis=0)' # # ) # # # # else: # # # # test_vals = SM.predict(c, batch_size=8) #128) # # if len(vars)>1: # # counter = 0 # # for v in vars: # # exec( # # v+\ # # '_P.append(np.squeeze(test_vals[counter]))' # # ) # # counter +=1 # # else: # # exec( # # vars[0]+\ # # '_P.append(np.asarray(np.squeeze(test_vals)))' # # ) # # # # # del test_vals # # # # # # del c #test_gen, # # # if len(vars)>1: # # for k in range(len(vars)): # # exec(vars[k]+'_pred = np.squeeze(np.asarray('+vars[k]+'_P))') # # else: # # exec(vars[0]+'_pred = np.squeeze(np.asarray('+vars[0]+'_P))') # # # # for v in vars: # # exec('del '+v+'_P') # # # ## write out results to text files # # if len(vars)>1: # # for k in range(len(vars)): # # exec('np.savetxt("'+name+'_test'+vars[k]+'.txt", ('+vars[k]+'_pred))') #','+vars[k]+'_true))') # # exec('np.savetxt("'+name+'_train'+vars[k]+'.txt", ('+vars[k]+'_predT))') #,'+vars[k]+'_trueT))') # # np.savetxt(name+"_test_files.txt", np.asarray(test_df.files.values), fmt="%s") # # np.savetxt(name+"_train_files.txt", np.asarray(train_df.files.values), fmt="%s") # # # # else: # # exec('np.savetxt("'+name+'_test'+vars[0]+'.txt", ('+vars[0]+'_pred))') #','+vars[k]+'_true))') # # exec('np.savetxt("'+name+'_train'+vars[0]+'.txt", ('+vars[0]+'_predT))') #,'+vars[k]+'_trueT))') # # np.savetxt(name+"_test_files.txt", np.asarray(test_df.files.values), fmt="%s") # # np.savetxt(name+"_train_files.txt", np.asarray(train_df.files.values), fmt="%s") # # if len(vars)==9: # nrows = 3; ncols = 3 # elif len(vars)==8: # nrows = 2; ncols = 4 # elif len(vars)==7: # nrows = 3; ncols = 3 # elif len(vars)==6: # nrows = 3; ncols = 2 # elif len(vars)==5: # nrows = 3; ncols = 2 # elif len(vars)==4: # nrows = 2; ncols = 2 # elif len(vars)==3: # nrows = 2; ncols = 2 # elif len(vars)==2: # nrows = 1; ncols = 2 # elif len(vars)==1: # nrows = 1; ncols = 1 # # out = dict() # # ## make a plot # fig = plt.figure(figsize=(6*nrows,6*ncols)) # labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' # for k in range(1,1+(nrows*ncols)): # # try: # plt.subplot(nrows,ncols,k) # x1 = eval(vars[k-1]+'_trueT') # y1 = eval(vars[k-1]+'_predT') # out[vars[k-1]+'_trueT'] = eval(vars[k-1]+'_trueT') # out[vars[k-1]+'_predT'] = eval(vars[k-1]+'_predT') # # # z = np.polyfit(y1,x1, 1) # # Z.append(z) # # # # y1 = np.polyval(z,y1) # # y1 = np.abs(y1) # # plt.plot(x1, y1, 'ko', markersize=5) # plt.plot([ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], # [ np.min(np.hstack((x1,y1))), np.max(np.hstack((x1,y1)))], # 'k', lw=2) # plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),'Train : '+\ # str(np.nanmean(100*(np.abs(y1-x1) / x1)))[:5]+\ # ' %', fontsize=10) # # # if test_vals is not None: # # x2 = eval(vars[k-1]+'_true') # # y2 = eval(vars[k-1]+'_pred') # # # y2 = np.abs(np.polyval(z,y2)) # # # # plt.plot(x2, y2, 'bx', markersize=5) # # # # if test_vals is not None: # # plt.text(np.nanmin(x2), 0.75*np.max(np.hstack((x2,y2))),'Test : '+\ # # str(np.mean(100*(np.abs(y2-x2) / x2)))[:5]+\ # # ' %', fontsize=10, color='b') # # else: # # plt.text(np.nanmin(x1), 0.7*np.max(np.hstack((x1,y1))),''+\ # # str(np.mean(100*(np.abs(y1-x1) / x1)))[:5]+\ # # ' %', fontsize=10) # plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left') # # #varstring = ''.join([str(k)+'_' for k in vars]) # varstring = str(len(vars))+'vars' # # # except: # # pass # if type(SM) == list: # plt.savefig(weights_path[0].replace('.hdf5', '_skill_ensemble'+str(count_in)+'.png').\ # replace('batch','_'.join(np.asarray(BATCH_SIZE, dtype='str'))), # dpi=300, bbox_inches='tight') # # else: # plt.savefig(weights_path.replace('.hdf5', '_skill'+str(count_in)+'.png'), # dpi=300, bbox_inches='tight') # # plt.close() # del fig # # np.savez_compressed(weights_path.replace('.hdf5', '_out'+str(count_in)+'.npz'),**out) # del out # # ###=================================================== # def predict_test_train_miso_mimo(train_df, test_df, train_idx, test_idx, # vars, auxin, SM, weights_path, name, mode, # greyscale, CS, CSaux): # """ # This function creates makes predcitions on test and train data # """ # ##============================================== # ## make predictions on training data # # SM.load_weights(weights_path) # # train_gen = get_data_generator_Nvars_miso_mimo(train_df, train_idx, False, # vars, auxin,aux_mean, aux_std, len(train_idx), greyscale) # # x_train, tmp = next(train_gen) # # if len(vars)>1: # counter = 0 # for v in vars: # exec( # v+\ # '_trueT = np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1)))' # ) # counter +=1 # else: # exec( # vars[0]+\ # '_trueT = np.squeeze(CS[0].inverse_transform(tmp[0].reshape(-1,1)))' # ) # # for v in vars: # exec(v+'_PT = []') # # del tmp # tmp = SM.predict(x_train, batch_size=8) #128) # if len(vars)>1: # counter = 0 # for v in vars: # exec( # v+\ # '_PT.append(np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1))))' # ) # counter +=1 # else: # exec( # vars[0]+\ # '_PT.append(np.asarray(np.squeeze(CS[0].inverse_transform(tmp.reshape(-1,1)))))' # ) # # # if len(vars)>1: # for k in range(len(vars)): # exec( # vars[k]+\ # '_predT = np.squeeze(np.mean(np.asarray('+vars[k]+'_PT), axis=0))' # ) # else: # exec( # vars[0]+\ # '_predT = np.squeeze(np.mean(np.asarray('+vars[0]+'_PT), axis=0))' # ) # # ## make predictions on testing data # test_gen = get_data_generator_Nvars_miso_mimo(test_df, test_idx, False, # vars, auxin, aux_mean, aux_std, len(test_idx), greyscale) # # del tmp # x_test, tmp = next(test_gen) # if len(vars)>1: # counter = 0 # for v in vars: # exec(v+\ # '_true = np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1)))' # ) # counter +=1 # else: # exec(vars[0]+\ # '_true = np.squeeze(CS[0].inverse_transform(tmp[0].reshape(-1,1)))' # ) # # for v in vars: # exec(v+'_P = []') # # del tmp # tmp = SM.predict(x_test, batch_size=8) #128) # if len(vars)>1: # counter = 0 # for v in vars: # exec( # v+\ # '_P.append(np.squeeze(CS[counter].inverse_transform(tmp[counter].reshape(-1,1))))' # ) # counter +=1 # else: # exec( # vars[0]+\ # '_P.append(np.asarray(np.squeeze(CS[0].inverse_transform(tmp.reshape(-1,1)))))' # ) # # if len(vars)>1: # for k in range(len(vars)): # exec( # vars[k]+\ # '_pred = np.squeeze(np.mean(np.asarray('+vars[k]+'_P), axis=0))' # ) # else: # exec( # vars[0]+\ # '_pred = np.squeeze(np.mean(np.asarray('+vars[0]+'_P), axis=0))' # ) # # # if len(vars)==9: # nrows = 3; ncols = 3 # elif len(vars)==8: # nrows = 4; ncols = 2 # elif len(vars)==7: # nrows = 4; ncols = 2 # elif len(vars)==6: # nrows = 3; ncols = 2 # elif len(vars)==5: # nrows = 3; ncols = 2 # elif len(vars)==4: # nrows = 2; ncols = 2 # elif len(vars)==3: # nrows = 3; ncols = 1 # elif len(vars)==2: # nrows = 2; ncols = 1 # elif len(vars)==1: # nrows = 1; ncols = 1 # # ## make a plot # fig = plt.figure(figsize=(4*nrows,4*ncols)) # labs = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' # for k in range(1,1+(nrows*ncols)): # plt.subplot(nrows,ncols,k) # x = eval(vars[k-1]+'_trueT') # y = eval(vars[k-1]+'_predT') # plt.plot(x, y, 'ko', markersize=5) # plt.plot(eval(vars[k-1]+'_true'), eval(vars[k-1]+'_pred'), # 'bx', markersize=5) # plt.plot([ np.min(np.hstack((x,y))), np.max(np.hstack((x,y)))], # [ np.min(np.hstack((x,y))), np.max(np.hstack((x,y)))], 'k', lw=2) # # plt.text(np.nanmin(x), 0.96*np.max(np.hstack((x,y))),'Test : '+\ # str(np.mean(100*(np.abs(eval(vars[k-1]+'_pred') -\ # eval(vars[k-1]+'_true')) / eval(vars[k-1]+'_true'))))[:5]+\ # ' %', fontsize=8, color='b') # plt.text(np.nanmin(x), np.max(np.hstack((x,y))),'Train : '+\ # str(np.mean(100*(np.abs(eval(vars[k-1]+'_predT') -\ # eval(vars[k-1]+'_trueT')) / eval(vars[k-1]+'_trueT'))))[:5]+\ # ' %', fontsize=8) # plt.title(r''+labs[k-1]+') '+vars[k-1], fontsize=8, loc='left') # # varstring = ''.join([str(k)+'_' for k in vars]) # # plt.savefig(weights_path.replace('.hdf5', '_skill.png'), # dpi=300, bbox_inches='tight') # plt.close() # del fig # # # ###=================================================== # def get_data_generator_Nvars_miso_mimo(df, indices, for_training, vars, auxin, # batch_size, greyscale, CS, CSaux): ##BATCH_SIZE # """ # This function generates data for a batch of images and 1 auxilliary variable, # and N associated output metrics # """ # if len(vars)==1: # images, a, p1s = [], [], [] # elif len(vars)==2: # images, a, p1s, p2s = [], [], [], [] # elif len(vars)==3: # images, a, p1s, p2s, p3s = [], [], [], [], [] # elif len(vars)==4: # images, a, p1s, p2s, p3s, p4s = [], [], [], [], [], [] # elif len(vars)==5: # images, a, p1s, p2s, p3s, p4s, p5s = [], [], [], [], [], [], [] # elif len(vars)==6: # images, a, p1s, p2s, p3s, p4s, p5s, p6s = \ # [], [], [], [], [], [], [], [] # elif len(vars)==7: # images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s = \ # [], [], [], [], [], [], [], [], [] # elif len(vars)==8: # images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s = \ # [], [], [], [], [], [], [], [], [], [] # elif len(vars)==9: # images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s = \ # [], [], [], [], [], [], [], [], [], [], [] # # while True: # for i in indices: # r = df.iloc[i] # if len(vars)==1: # file, p1, aa = r['files'], r[vars[0]], r[auxin] # if len(vars)==2: # file, p1, p2, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[auxin] # if len(vars)==3: # file, p1, p2, p3, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[auxin] # if len(vars)==4: # file, p1, p2, p3, p4, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[auxin] # if len(vars)==5: # file, p1, p2, p3, p4, p5, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[auxin] # if len(vars)==6: # file, p1, p2, p3, p4, p5, p6, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[auxin] # if len(vars)==7: # file, p1, p2, p3, p4, p5, p6, p7, aa =\ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[auxin] # if len(vars)==8: # file, p1, p2, p3, p4, p5, p6, p7, p8, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]], r[auxin] # elif len(vars)==9: # file, p1, p2, p3, p4, p5, p6, p7, p8, p9, aa = \ # r['files'], r[vars[0]], r[vars[1]], r[vars[2]], r[vars[3]], r[vars[4]], r[vars[5]], r[vars[6]], r[vars[7]], r[vars[8]], r[auxin] # # if greyscale==True: # im = Image.open(file).convert('LA') # else: # im = Image.open(file) # im = im.resize((IM_HEIGHT, IM_HEIGHT)) # im = np.array(im) / 255.0 # # if np.ndim(im)==2: # im = np.dstack((im, im , im)) ##np.expand_dims(im[:,:,0], axis=2) # # im = im[:,:,:3] # # if greyscale==True: # images.append(np.expand_dims(im, axis=2)) # else: # images.append(im) # # if len(vars)==1: # p1s.append(p1); a.append(aa) # elif len(vars)==2: # p1s.append(p1); p2s.append(p2); a.append(aa) # elif len(vars)==3: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); # elif len(vars)==4: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); p4s.append(p4) # elif len(vars)==5: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); p4s.append(p4) # p5s.append(p5); # elif len(vars)==6: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); p4s.append(p4) # p5s.append(p5); p6s.append(p6) # elif len(vars)==7: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); p4s.append(p4) # p5s.append(p5); p6s.append(p6) # p7s.append(p7); # elif len(vars)==8: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); p4s.append(p4) # p5s.append(p5); p6s.append(p6) # p7s.append(p7); p8s.append(p8) # elif len(vars)==9: # p1s.append(p1); p2s.append(p2); a.append(aa) # p3s.append(p3); p4s.append(p4) # p5s.append(p5); p6s.append(p6) # p7s.append(p7); p8s.append(p8) # p9s.append(p9) # # # if len(images) >= batch_size: # if len(vars)==1: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)], [np.array(p1s)] # images, a, p1s = [], [], [] # elif len(vars)==2: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s)] # images, a, p1s, p2s = [], [], [], [] # elif len(vars)==3: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s)] # images, a, p1s, p2s, p3s = [], [], [], [], [] # elif len(vars)==4: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), np.array(p4s)] # images, a, p1s, p2s, p3s, p4s = [], [], [], [], [], [] # elif len(vars)==5: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), # np.array(p4s), np.array(p5s)] # images, a, p1s, p2s, p3s, p4s, p5s = \ # [], [], [], [], [], [], [] # elif len(vars)==6: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), # np.array(p4s), np.array(p5s), np.array(p6s)] # images, a, p1s, p2s, p3s, p4s, p5s, p6s = \ # [], [], [], [], [], [], [], [] # elif len(vars)==7: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), # np.array(p4s), np.array(p5s), np.array(p6s), np.array(p7s)] # images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s = \ # [], [], [], [], [], [], [], [], [] # elif len(vars)==8: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1))) # p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), # np.array(p4s), np.array(p5s), np.array(p6s), # np.array(p7s), np.array(p8s)] # images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s = \ # [], [], [], [], [], [], [], [], [], [] # elif len(vars)==9: # p1s = np.squeeze(CS[0].transform(np.array(p1s).reshape(-1, 1))) # p2s = np.squeeze(CS[1].transform(np.array(p2s).reshape(-1, 1))) # p3s = np.squeeze(CS[2].transform(np.array(p3s).reshape(-1, 1))) # p4s = np.squeeze(CS[3].transform(np.array(p4s).reshape(-1, 1))) # p5s = np.squeeze(CS[4].transform(np.array(p5s).reshape(-1, 1))) # p6s = np.squeeze(CS[5].transform(np.array(p6s).reshape(-1, 1))) # p7s = np.squeeze(CS[6].transform(np.array(p7s).reshape(-1, 1))) # p8s = np.squeeze(CS[7].transform(np.array(p8s).reshape(-1, 1))) # p9s = np.squeeze(CS[8].transform(np.array(p9s).reshape(-1, 1))) # a = np.squeeze(CSaux[0].transform(np.array(a).reshape(-1, 1))) # try: # yield [np.array(a), np.array(images)],[np.array(p1s), np.array(p2s), np.array(p3s), # np.array(p4s), np.array(p5s), np.array(p6s), # np.array(p7s), np.array(p8s), np.array(p9s)] # except GeneratorExit: # print(" ") #pass # images, a, p1s, p2s, p3s, p4s, p5s, p6s, p7s, p8s, p9s = \ # [], [], [], [], [], [], [], [], [], [], [] # if not for_training: # break