#NEWLINE# def main( args , loglevel ) : #NEWLINE# #INDENT# logging . basicConfig ( format = '%(levelname)s: %(message)s' , level = loglevel ) #NEWLINE# socket . setdefaulttimeout ( args . timeout ) #NEWLINE# g = Github ( ) #NEWLINE# with open ( args . repo_file , 'r' ) as f : #NEWLINE# #INDENT# file_counter = 0 #NEWLINE# for line in f . readlines ( ) : #NEWLINE# #INDENT# logging . info ( ( 'Fetching repository: %s' % repo_str ) ) #NEWLINE# try : #NEWLINE# #INDENT# repo_str = line . rstrip ( ) . split ( 'github.com/' ) [ ( - 1 ) ] #NEWLINE# repo = g . get_repo ( repo_str ) #NEWLINE# tree = repo . get_git_tree ( 'master' , recursive = True ) #NEWLINE# files_to_download = [ ] #NEWLINE# for file in tree . tree : #NEWLINE# #INDENT# if fnmatch . fnmatch ( file . path , args . wildcard ) : #NEWLINE# #INDENT# files_to_download . append ( ( 'https://github.com/%s/raw/master/%s' % ( repo_str , file . path ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# for file in files_to_download : #NEWLINE# #INDENT# logging . info ( ( 'Downloading %s' % file ) ) #NEWLINE# file_counter += 1 #NEWLINE# filename = posixpath . basename ( urlparse . urlsplit ( file ) . path ) #NEWLINE# output_path = os . path . join ( args . output_dir , filename ) #NEWLINE# if os . path . exists ( output_path ) : #NEWLINE# #INDENT# output_path += ( '-' + str ( file_counter ) ) #NEWLINE# #UNINDENT# try : #NEWLINE# #INDENT# urllib . urlretrieve ( file , output_path ) #NEWLINE# #UNINDENT# except : #NEWLINE# #INDENT# logging . error ( ( 'Error downloading %s' % file ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# except : #NEWLINE# #INDENT# logging . error ( ( 'Error fetching repository %s' % line ) ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# args . yara_meta = os . path . join ( args . output_dir , args . yara_meta ) #NEWLINE# with open ( args . yara_meta , 'w' ) as f : #NEWLINE# #INDENT# for i in os . listdir ( args . output_dir ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# f . write ( ( ( 'include "' + i ) + '"\n' ) ) #NEWLINE# #UNINDENT# except : #NEWLINE# #INDENT# logging . error ( ( "Couldn't write to %s" % args . yara_meta ) )