#NEWLINE# def download( self , source_urls , workdir , conform = None ) : #NEWLINE# #INDENT# output_files = [ ] #NEWLINE# download_path = os . path . join ( workdir , 'esri' ) #NEWLINE# mkdirsp ( download_path ) #NEWLINE# query_fields = self . field_names_to_request ( conform ) #NEWLINE# for source_url in source_urls : #NEWLINE# #INDENT# size = 0 #NEWLINE# file_path = self . get_file_path ( source_url , download_path ) #NEWLINE# if os . path . exists ( file_path ) : #NEWLINE# #INDENT# output_files . append ( file_path ) #NEWLINE# _L . debug ( 'File exists %s' , file_path ) #NEWLINE# continue #NEWLINE# #UNINDENT# metadata = self . get_layer_metadata ( source_url ) #NEWLINE# if ( query_fields is None ) : #NEWLINE# #INDENT# field_names = [ f [ 'name' ] for f in metadata [ 'fields' ] ] #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# field_names = query_fields [ : ] #NEWLINE# #UNINDENT# if ( X_FIELDNAME not in field_names ) : #NEWLINE# #INDENT# field_names . append ( X_FIELDNAME ) #NEWLINE# #UNINDENT# if ( Y_FIELDNAME not in field_names ) : #NEWLINE# #INDENT# field_names . append ( Y_FIELDNAME ) #NEWLINE# #UNINDENT# if ( GEOM_FIELDNAME not in field_names ) : #NEWLINE# #INDENT# field_names . append ( GEOM_FIELDNAME ) #NEWLINE# #UNINDENT# query_url = ( source_url + '/query' ) #NEWLINE# count_json = self . get_layer_feature_count ( query_url ) #NEWLINE# row_count = count_json . get ( 'count' ) #NEWLINE# page_size = metadata . get ( 'maxRecordCount' , 500 ) #NEWLINE# if ( page_size > 1000 ) : #NEWLINE# #INDENT# page_size = 1000 #NEWLINE# #UNINDENT# _L . info ( 'Source has {} rows' . format ( row_count ) ) #NEWLINE# page_args = [ ] #NEWLINE# if ( metadata . get ( 'supportsPagination' ) or ( metadata . get ( 'advancedQueryCapabilities' ) and metadata [ 'advancedQueryCapabilities' ] [ 'supportsPagination' ] ) ) : #NEWLINE# #INDENT# if ( query_fields and ( not self . can_handle_pagination ( query_url , query_fields ) ) ) : #NEWLINE# #INDENT# _L . info ( 'Source does not support pagination with fields specified, so querying for all fields.' ) #NEWLINE# query_fields = None #NEWLINE# #UNINDENT# for offset in range ( 0 , row_count , page_size ) : #NEWLINE# #INDENT# page_args . append ( { #NEWLINE# #INDENT# 'resultOffset' : offset , #NEWLINE# 'resultRecordCount' : page_size , #NEWLINE# 'where' : '1=1' , #NEWLINE# 'geometryPrecision' : 7 , #NEWLINE# 'returnGeometry' : 'true' , #NEWLINE# 'outSR' : 4326 , #NEWLINE# 'outFields' : ',' . join ( ( query_fields or [ '*' ] ) ) , #NEWLINE# 'f' : 'json' , #NEWLINE# #UNINDENT# } ) #NEWLINE# #UNINDENT# _L . info ( 'Built {} requests using resultOffset method' . format ( len ( page_args ) ) ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# use_oids = True #NEWLINE# if metadata . get ( 'supportsStatistics' ) : #NEWLINE# #INDENT# oid_field_name = self . find_oid_field_name ( metadata ) #NEWLINE# try : #NEWLINE# #INDENT# ( oid_min , oid_max ) = self . get_layer_min_max ( query_url , oid_field_name ) #NEWLINE# for page_min in range ( ( oid_min - 1 ) , oid_max , page_size ) : #NEWLINE# #INDENT# page_max = min ( ( page_min + page_size ) , oid_max ) #NEWLINE# page_args . append ( { #NEWLINE# #INDENT# 'where' : '{} > {} AND {} <= {}' . format ( oid_field_name , page_min , oid_field_name , page_max ) , #NEWLINE# 'geometryPrecision' : 7 , #NEWLINE# 'returnGeometry' : 'true' , #NEWLINE# 'outSR' : 4326 , #NEWLINE# 'outFields' : ',' . join ( ( query_fields or [ '*' ] ) ) , #NEWLINE# 'f' : 'json' , #NEWLINE# #UNINDENT# } ) #NEWLINE# #UNINDENT# _L . info ( 'Built {} requests using OID where clause method' . format ( len ( page_args ) ) ) #NEWLINE# use_oids = False #NEWLINE# #UNINDENT# except DownloadError : #NEWLINE# #INDENT# _L . exception ( 'Finding max/min from statistics failed. Trying OID enumeration.' ) #NEWLINE# #UNINDENT# #UNINDENT# if use_oids : #NEWLINE# #INDENT# oid_data = self . get_layer_oids ( query_url ) #NEWLINE# oids = oid_data [ 'objectIds' ] #NEWLINE# for i in range ( 0 , len ( oids ) , 100 ) : #NEWLINE# #INDENT# oid_chunk = map ( ( long if PY2 else int ) , oids [ i : ( i + 100 ) ] ) #NEWLINE# page_args . append ( { #NEWLINE# #INDENT# 'objectIds' : ',' . join ( map ( str , oid_chunk ) ) , #NEWLINE# 'geometryPrecision' : 7 , #NEWLINE# 'returnGeometry' : 'true' , #NEWLINE# 'outSR' : 4326 , #NEWLINE# 'outFields' : ',' . join ( ( query_fields or [ '*' ] ) ) , #NEWLINE# 'f' : 'json' , #NEWLINE# #UNINDENT# } ) #NEWLINE# #UNINDENT# _L . info ( 'Built {} requests using OID enumeration method' . format ( len ( page_args ) ) ) #NEWLINE# #UNINDENT# #UNINDENT# with csvopen ( file_path , 'w' , encoding = 'utf-8' ) as f : #NEWLINE# #INDENT# writer = csvDictWriter ( f , fieldnames = field_names , encoding = 'utf-8' ) #NEWLINE# writer . writeheader ( ) #NEWLINE# for query_args in page_args : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# response = request ( 'POST' , query_url , headers = self . headers , data = query_args ) #NEWLINE# data = self . handle_esri_errors ( response , 'Could not retrieve this chunk of objects from ESRI source' ) #NEWLINE# #UNINDENT# except socket . timeout as e : #NEWLINE# #INDENT# raise DownloadError ( 'Timeout when connecting to URL' , e ) #NEWLINE# #UNINDENT# except ValueError as e : #NEWLINE# #INDENT# raise DownloadError ( 'Could not parse JSON' , e ) #NEWLINE# #UNINDENT# except Exception as e : #NEWLINE# #INDENT# raise DownloadError ( 'Could not connect to URL' , e ) #NEWLINE# #UNINDENT# finally : #NEWLINE# #INDENT# f . truncate ( ) #NEWLINE# #UNINDENT# error = data . get ( 'error' ) #NEWLINE# if error : #NEWLINE# #INDENT# raise DownloadError ( 'Problem querying ESRI dataset with args {}. Server said: {}' . format ( query_args , error [ 'message' ] ) ) #NEWLINE# #UNINDENT# geometry_type = data . get ( 'geometryType' ) #NEWLINE# features = data . get ( 'features' ) #NEWLINE# for feature in features : #NEWLINE# #INDENT# try : #NEWLINE# #INDENT# ogr_geom = self . build_ogr_geometry ( geometry_type , feature ) #NEWLINE# row = feature . get ( 'attributes' , { #NEWLINE# #INDENT# #NEWLINE# #UNINDENT# } ) #NEWLINE# row [ GEOM_FIELDNAME ] = ogr_geom . ExportToWkt ( ) #NEWLINE# try : #NEWLINE# #INDENT# centroid = ogr_geom . Centroid ( ) #NEWLINE# #UNINDENT# except RuntimeError as e : #NEWLINE# #INDENT# if ( 'Invalid number of points in LinearRing found' not in str ( e ) ) : #NEWLINE# #INDENT# raise #NEWLINE# #UNINDENT# ( xmin , xmax , ymin , ymax ) = ogr_geom . GetEnvelope ( ) #NEWLINE# row [ X_FIELDNAME ] = round ( ( ( xmin / 2 ) + ( xmax / 2 ) ) , 7 ) #NEWLINE# row [ Y_FIELDNAME ] = round ( ( ( ymin / 2 ) + ( ymax / 2 ) ) , 7 ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# row [ X_FIELDNAME ] = round ( centroid . GetX ( ) , 7 ) #NEWLINE# row [ Y_FIELDNAME ] = round ( centroid . GetY ( ) , 7 ) #NEWLINE# #UNINDENT# writer . writerow ( { fn : row . get ( fn ) for fn in field_names } ) #NEWLINE# size += 1 #NEWLINE# #UNINDENT# except TypeError : #NEWLINE# #INDENT# _L . debug ( 'Skipping a geometry' , exc_info = True ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# _L . info ( 'Downloaded %s ESRI features for file %s' , size , file_path ) #NEWLINE# output_files . append ( file_path ) #NEWLINE# #UNINDENT# return output_files