#NEWLINE# def _refresh_buckets_cache_file( cache_file ) : #NEWLINE# #INDENT# '\n    Retrieve the content of all buckets and cache the metadata to the buckets\n    cache file\n    ' #NEWLINE# log . debug ( 'Refreshing buckets cache file' ) #NEWLINE# ( key , keyid , service_url , verify_ssl , kms_keyid , location ) = _get_s3_key ( ) #NEWLINE# metadata = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# def __get_s3_meta( bucket , key = key , keyid = keyid ) : #NEWLINE# #INDENT# return s3 . query ( key = key , keyid = keyid , kms_keyid = keyid , bucket = bucket , service_url = service_url , verify_ssl = verify_ssl , location = location , return_bin = False ) #NEWLINE# #UNINDENT# if _is_env_per_bucket ( ) : #NEWLINE# #INDENT# for ( saltenv , buckets ) in six . iteritems ( _get_buckets ( ) ) : #NEWLINE# #INDENT# bucket_files = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for bucket_name in buckets : #NEWLINE# #INDENT# s3_meta = __get_s3_meta ( bucket_name ) #NEWLINE# if ( not s3_meta ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# bucket_files [ bucket_name ] = [ k for k in s3_meta if ( 'Key' in k ) ] #NEWLINE# if ( len ( bucket_files [ bucket_name ] ) == 0 ) : #NEWLINE# #INDENT# meta_response = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for k in s3_meta : #NEWLINE# #INDENT# if ( ( 'Code' in k ) or ( 'Message' in k ) ) : #NEWLINE# #INDENT# meta_response . update ( k ) #NEWLINE# #UNINDENT# #UNINDENT# try : #NEWLINE# #INDENT# log . warning ( "'{0}' response for bucket '{1}'" . format ( meta_response [ 'Message' ] , bucket_name ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# except KeyError : #NEWLINE# #INDENT# if ( 'Code' in meta_response ) : #NEWLINE# #INDENT# log . warning ( "'{0}' response for bucket '{1}'" . format ( meta_response [ 'Code' ] , bucket_name ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# log . warning ( 'S3 Error! Do you have any files in your S3 bucket?' ) #NEWLINE# return { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# metadata [ saltenv ] = bucket_files #NEWLINE# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# for bucket_name in _get_buckets ( ) : #NEWLINE# #INDENT# s3_meta = __get_s3_meta ( bucket_name ) #NEWLINE# if ( not s3_meta ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# files = [ k for k in s3_meta if ( 'Key' in k ) ] #NEWLINE# if ( len ( files ) == 0 ) : #NEWLINE# #INDENT# meta_response = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# for k in s3_meta : #NEWLINE# #INDENT# if ( ( 'Code' in k ) or ( 'Message' in k ) ) : #NEWLINE# #INDENT# meta_response . update ( k ) #NEWLINE# #UNINDENT# #UNINDENT# try : #NEWLINE# #INDENT# log . warning ( "'{0}' response for bucket '{1}'" . format ( meta_response [ 'Message' ] , bucket_name ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# except KeyError : #NEWLINE# #INDENT# if ( 'Code' in meta_response ) : #NEWLINE# #INDENT# log . warning ( "'{0}' response for bucket '{1}'" . format ( meta_response [ 'Code' ] , bucket_name ) ) #NEWLINE# continue #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# log . warning ( 'S3 Error! Do you have any files in your S3 bucket?' ) #NEWLINE# return { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# environments = [ os . path . dirname ( k [ 'Key' ] ) . split ( '/' , 1 ) [ 0 ] for k in files ] #NEWLINE# environments = set ( environments ) #NEWLINE# for saltenv in environments : #NEWLINE# #INDENT# env_files = [ k for k in files if k [ 'Key' ] . startswith ( saltenv ) ] #NEWLINE# if ( saltenv not in metadata ) : #NEWLINE# #INDENT# metadata [ saltenv ] = { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } #NEWLINE# #UNINDENT# if ( bucket_name not in metadata [ saltenv ] ) : #NEWLINE# #INDENT# metadata [ saltenv ] [ bucket_name ] = [ ] #NEWLINE# #UNINDENT# metadata [ saltenv ] [ bucket_name ] += env_files #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# if os . path . isfile ( cache_file ) : #NEWLINE# #INDENT# os . remove ( cache_file ) #NEWLINE# #UNINDENT# log . debug ( 'Writing buckets cache file' ) #NEWLINE# with salt . utils . fopen ( cache_file , 'w' ) as fp_ : #NEWLINE# #INDENT# pickle . dump ( metadata , fp_ ) #NEWLINE# #UNINDENT# return metadata