#NEWLINE# def download_directory( url , target , insecure = False ) : #NEWLINE# #INDENT# def mkdir( ) : #NEWLINE# #INDENT# if ( not mkdir . done ) : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# os . mkdir ( target ) #NEWLINE# #UNINDENT# except OSError : #NEWLINE# #INDENT# pass #NEWLINE# #UNINDENT# mkdir . done = True #NEWLINE# #UNINDENT# #UNINDENT# mkdir . done = False #NEWLINE# opener = build_opener ( insecure = insecure ) #NEWLINE# response = opener . open ( url ) #NEWLINE# if ( response . info ( ) . type == 'text/html' ) : #NEWLINE# #INDENT# contents = response . read ( ) #NEWLINE# parser = ListingParser ( url ) #NEWLINE# parser . feed ( contents ) #NEWLINE# for link in parser . links : #NEWLINE# #INDENT# link = resolve_link ( link , url ) #NEWLINE# if ( link [ ( - 1 ) ] == '/' ) : #NEWLINE# #INDENT# link = link [ : ( - 1 ) ] #NEWLINE# #UNINDENT# if ( not link . startswith ( url ) ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# name = link . rsplit ( '/' , 1 ) [ 1 ] #NEWLINE# if ( '?' in name ) : #NEWLINE# #INDENT# continue #NEWLINE# #UNINDENT# mkdir ( ) #NEWLINE# download_directory ( link , os . path . join ( target , name ) , insecure ) #NEWLINE# #UNINDENT# if ( not mkdir . done ) : #NEWLINE# #INDENT# if ( url [ ( - 1 ) ] != '/' ) : #NEWLINE# #INDENT# end = target [ ( - 5 ) : ] . lower ( ) #NEWLINE# if ( not ( end . endswith ( '.htm' ) or end . endswith ( '.html' ) ) ) : #NEWLINE# #INDENT# target = ( target + '.html' ) #NEWLINE# #UNINDENT# with open ( target , 'wb' ) as fp : #NEWLINE# #INDENT# fp . write ( contents ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# else : #NEWLINE# #INDENT# buffer_size = 4096 #NEWLINE# with open ( target , 'wb' ) as fp : #NEWLINE# #INDENT# chunk = response . read ( buffer_size ) #NEWLINE# while chunk : #NEWLINE# #INDENT# fp . write ( chunk ) #NEWLINE# chunk = response . read ( buffer_size )