#NEWLINE# def create_job_fileinfo( job ) : #NEWLINE# #INDENT# parameters = job . get_parameters ( ) #NEWLINE# from .. models import WooeyFile , UserFile #NEWLINE# files = [ ] #NEWLINE# local_storage = get_storage ( local = True ) #NEWLINE# for field in parameters : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# if ( field . parameter . form_field == 'FileField' ) : #NEWLINE# #INDENT# value = field . value #NEWLINE# if ( value is None ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# if isinstance ( value , six . string_types ) : #NEWLINE# #INDENT# if local_storage . exists ( value ) : #NEWLINE# #INDENT# if ( not get_storage ( local = False ) . exists ( value ) ) : #NEWLINE# #INDENT# get_storage ( local = False ) . save ( value , File ( local_storage . open ( value ) ) ) #NEWLINE# #UNINDENT# value = field . value #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# field . force_value ( None ) #NEWLINE# try : #NEWLINE# #INDENT# with transaction . atomic ( ) : #NEWLINE# #INDENT# field . save ( ) #NEWLINE# #UNINDENT# #UNINDENT# except : #NEWLINE# #INDENT# sys . stderr . write ( '{}\n' . format ( traceback . format_exc ( ) ) ) #NEWLINE# #UNINDENT# continue #NEWLINE# #UNINDENT# #UNINDENT# d = { #NEWLINE# #INDENT# 'parameter' : field , #NEWLINE# 'file' : value , #NEWLINE# #UNINDENT# } #NEWLINE# if field . parameter . is_output : #NEWLINE# #INDENT# full_path = os . path . join ( job . save_path , os . path . split ( local_storage . path ( value ) ) [ 1 ] ) #NEWLINE# checksum = get_checksum ( value , extra = [ job . pk , full_path , 'output' ] ) #NEWLINE# d [ 'checksum' ] = checksum #NEWLINE# #UNINDENT# files . append ( d ) #NEWLINE# #UNINDENT# #UNINDENT# except ValueError : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# #UNINDENT# known_files = { i [ 'file' ] . name for i in files } #NEWLINE# file_groups = { #NEWLINE# #INDENT# 'archives' : [ ] , #NEWLINE# #UNINDENT# } #NEWLINE# absbase = os . path . join ( settings . MEDIA_ROOT , job . save_path ) #NEWLINE# for ( root , dirs , dir_files ) in os . walk ( absbase ) : #NEWLINE# #INDENT# for filename in dir_files : #NEWLINE# #INDENT# new_name = os . path . join ( job . save_path , filename ) #NEWLINE# if any ( [ i . endswith ( new_name ) for i in known_files ] ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# filepath = os . path . join ( root , filename ) #NEWLINE# if os . path . isdir ( filepath ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# full_path = os . path . join ( job . save_path , filename ) #NEWLINE# checksum = get_checksum ( filepath , extra = [ job . pk , full_path , 'output' ] ) #NEWLINE# try : #NEWLINE# #INDENT# storage_file = get_storage_object ( full_path ) #NEWLINE# #UNINDENT# except : #NEWLINE# #INDENT# sys . stderr . write ( 'Error in accessing stored file {}:\n{}' . format ( full_path , traceback . format_exc ( ) ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# d = { #NEWLINE# #INDENT# 'name' : filename , #NEWLINE# 'file' : storage_file , #NEWLINE# 'size_bytes' : storage_file . size , #NEWLINE# 'checksum' : checksum , #NEWLINE# #UNINDENT# } #NEWLINE# if ( filename . endswith ( '.tar.gz' ) or filename . endswith ( '.zip' ) ) : #NEWLINE# #INDENT# file_groups [ 'archives' ] . append ( d ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# files . append ( d ) #NEWLINE# #UNINDENT# #UNINDENT# except IOError : #NEWLINE# #INDENT# sys . stderr . write ( '{}' . format ( traceback . format_exc ( ) ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# file_groups [ 'all' ] = files #NEWLINE# file_groups [ 'image' ] = [ ] #NEWLINE# file_groups [ 'tabular' ] = [ ] #NEWLINE# file_groups [ 'fasta' ] = [ ] #NEWLINE# for filemodel in files : #NEWLINE# #INDENT# fileinfo = get_file_info ( filemodel [ 'file' ] . path ) #NEWLINE# filetype = fileinfo . get ( 'type' ) #NEWLINE# if ( filetype is not None ) : #NEWLINE# #INDENT# file_groups [ filetype ] . append ( dict ( filemodel , ** { #NEWLINE# #INDENT# 'preview' : fileinfo . get ( 'preview' ) , #NEWLINE# #UNINDENT# } ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# filemodel [ 'preview' ] = json . dumps ( None ) #NEWLINE# #UNINDENT# #UNINDENT# grouped = set ( [ i [ 'file' ] . path for ( file_type , groups ) in six . iteritems ( file_groups ) for i in groups if ( file_type != 'all' ) ] ) #NEWLINE# for ( file_type , group_files ) in six . iteritems ( file_groups ) : #NEWLINE# #INDENT# for group_file in group_files : #NEWLINE# #INDENT# if ( ( file_type == 'all' ) and ( group_file [ 'file' ] . path in grouped ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# preview = group_file . get ( 'preview' ) #NEWLINE# size_bytes = group_file . get ( 'size_bytes' ) #NEWLINE# filepath = group_file [ 'file' ] . path #NEWLINE# save_path = job . get_relative_path ( filepath ) #NEWLINE# parameter = group_file . get ( 'parameter' ) #NEWLINE# checksum = group_file . get ( 'checksum' , get_checksum ( filepath ) ) #NEWLINE# try : #NEWLINE# #INDENT# wooey_file = WooeyFile . objects . get ( checksum = checksum ) #NEWLINE# file_created = False #NEWLINE# #UNINDENT# except ObjectDoesNotExist : #NEWLINE# #INDENT# wooey_file = WooeyFile ( checksum = checksum , filetype = file_type , filepreview = preview , size_bytes = size_bytes , filepath = save_path ) #NEWLINE# file_created = True #NEWLINE# #UNINDENT# userfile_kwargs = { #NEWLINE# #INDENT# 'job' : job , #NEWLINE# 'parameter' : parameter , #NEWLINE# 'system_file' : wooey_file , #NEWLINE# 'filename' : os . path . split ( filepath ) [ 1 ] , #NEWLINE# #UNINDENT# } #NEWLINE# try : #NEWLINE# #INDENT# with transaction . atomic ( ) : #NEWLINE# #INDENT# if file_created : #NEWLINE# #INDENT# wooey_file . save ( ) #NEWLINE# #UNINDENT# job . save ( ) #NEWLINE# UserFile . objects . get_or_create ( ** userfile_kwargs ) #NEWLINE# #UNINDENT# #UNINDENT# except : #NEWLINE# #INDENT# sys . stderr . write ( 'Error in saving DJFile: {}\n' . format ( traceback . format_exc ( ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# except : #NEWLINE# #INDENT# sys . stderr . write ( 'Error in saving DJFile: {}\n' . format ( traceback . format_exc ( ) ) ) #NEWLINE# continue