#NEWLINE# def run( self , mapping ) : #NEWLINE# #INDENT# '\n        generated_filename will return a filename similar to this:\n            `20101107__wa__general__precinct.csv`\n\n        election will return a filename similar to this:\n            `20101102__wa__general__precinct`\n\n        ' #NEWLINE# generated_filename = mapping [ 'generated_filename' ] #NEWLINE# election = mapping [ 'election' ] #NEWLINE# "\n        bad_filenames[] holds the list of files who have content that's\n        hard to use (e.g. an .xls file with 10 sheets).\n\n        The edge cases will be taken care of later. The cases where there is\n        zero actual usable data will have to be rectified outside of the\n        loader module.\n\n        " #NEWLINE# bad_filenames = [ '20070821__wa__primary.xls' , '20070821__wa__primary__county.xls' , '20080219__wa__primary__adams__precinct.xls' , '20080219__wa__primary__benton__precinct.xls' , '20080219__wa__primary__congressional_district_state_legislative.xls' , '20080219__wa__primary__douglas__precinct.xls' , '20080219__wa__primary__kitsap__precinct.xls' , '20080819__wa__primary__kitsap__precinct.xls' , '20080819__wa__primary__pierce__precinct.xls' , '20081104__wa__general__congressional_district.xls' , '20081104__wa__general__adams__precinct.xls' , '20091103__wa__general__clark__precinct.xls' , '20081104__wa__general__franklin__precinct.xls' , '20081104__wa__general__kittitas__precinct.xls' , '20081104__wa__general__kitsap__precinct.xls' , '20081104__wa__general__pierce__precinct.xls' , '20081104__wa__general__precinct.xls' , '20081104__wa__general__state_legislative.xls' , '20091103__wa__general__kitsap__precinct.xls' , '20091103__wa__general__pierce__precinct.xls' , '20101102__wa__general__kittitas___precinct.xls' , '20101102__wa__general__san_juan___precinct.xls' , '20100817__wa__primary__state_legislative.xls' , '20100817__wa__primary__congressional_district.xls' , '20111108__wa__general__clark___precinct.xlsx' , '20111108__wa__general__spokane___precinct.xlsx' , '20120807__wa__primary__congressional_district.xls' , '20120807__wa__primary__state_legislative.xls' , '20121106__wa__general__congressional_district.xls' , '20121106__wa__general__state_legislative.xls' ] #NEWLINE# '\n        Could try using `generated_filename.split(.)[-1]` instead of\n        os.path.splitext(election)[-1], since all filenames are\n        standardized. This would, of course, break if the file path includes\n        a full stop (period).\n\n        ' #NEWLINE# if any ( ( ( x in generated_filename ) for x in bad_filenames ) ) : #NEWLINE# #INDENT# loader = SkipLoader ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( os . path . splitext ( generated_filename ) [ ( - 1 ) ] . lower ( ) in ( '.xls' , '.xlsx' ) ) : #NEWLINE# #INDENT# loader = WALoaderExcel ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( os . path . splitext ( generated_filename ) [ ( - 1 ) ] . lower ( ) == '.txt' ) : #NEWLINE# #INDENT# "\n            We run into issues where King County provides > 1 million line\n            .txt files that break my machine's memory. We definitely need to\n            refactor, but for the moment we'll pass over said files.\n\n            " #NEWLINE# logger . info ( 'Cannot do anything with {0}' . format ( generated_filename ) ) #NEWLINE# loader = SkipLoader ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( 'precinct' in generated_filename ) : #NEWLINE# #INDENT# loader = WALoaderPrecincts ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if any ( ( ( bad_filenames in election ) for s in [ '2000' , '2001' , '2002' , '2003' , '2004' , '2005' , '2006' ] ) ) : #NEWLINE# #INDENT# loader = WALoaderPre2007 ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# if ( os . path . splitext ( generated_filename ) [ ( - 1 ) ] . lower ( ) in ( '.csv' , '.txt' ) ) : #NEWLINE# #INDENT# loader = WALoaderPost2007 ( ) #NEWLINE# #UNINDENT# else : #NEWLINE# #INDENT# loader = SkipLoader ( ) #NEWLINE# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# #UNINDENT# "\n        * UnboundLocalError: File passes through the elif statements, but is\n          not a file we have a loader class set up to handle at this point, so\n          loader.run(mapping) is called before it's mentioned\n\n        * IOError: File in quesiton does not exist. Seen when the mapping\n          a file path that recieved a 404 error\n\n        * unicodecsv.Error: Similar to UnboundLocalError, this error means\n          that the loader tried running but the csv parser could not parse\n          the file because of a null byte. See:\n          https://github.com/jdunck/python-unicodecsv/blob/master/unicodecsv/test.py#L222\n\n        * errors.InvalidOperation: When a file has no useful data, RawResult\n          is empty and mongodb refuses to load it.\n\n        Because of the if/else flow, sometimes we'll end up with multiple\n        UnboundLocalErrors. This should be changed so we only get the error\n        once.\n\n        " #NEWLINE# try : #NEWLINE# #INDENT# loader . run ( mapping ) #NEWLINE# #UNINDENT# except UnboundLocalError : #NEWLINE# #INDENT# logger . error ( '\tUnsupported file type ({0})' . format ( 'UnboundLocalError' ) ) #NEWLINE# #UNINDENT# except IOError : #NEWLINE# #INDENT# logger . error ( '\tFile "{0}" does not exist' . format ( generated_filename ) ) #NEWLINE# #UNINDENT# except unicodecsv . Error : #NEWLINE# #INDENT# logger . error ( '\tUnsupported file type "({0})"' . format ( 'unicodecsv.Error' ) ) #NEWLINE# #UNINDENT# except errors . InvalidOperation : #NEWLINE# #INDENT# logger . error ( '\tNo raw results loaded' )