Module Import alias alias Expr Call AttributeLoad AttributeLoad NameLoad attr attr Str ImportFrom alias ImportFrom alias alias ImportFrom alias ImportFrom alias ImportFrom alias ImportFrom alias ImportFrom alias Assign NameStore NameLoad ClassDef bases NameLoad body Pass decorator_list ClassDef bases NameLoad body Pass decorator_list FunctionDef arguments args NameParam NameParam NameParam NameParam NameParam NameParam defaults body Assign NameStore SubscriptLoad NameLoad Index Str TryExcept body If CompareEq SubscriptLoad NameLoad Index Str Str body Assign SubscriptStore NameLoad Index Str NameLoad handlers ExceptHandler type NameLoad body Pass If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse If CompareEq NameLoad Str body Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad orelse Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad decorator_list FunctionDef arguments args NameParam NameParam NameParam defaults NameLoad NameLoad body If UnaryOpNot NameLoad body Assign NameStore Call NameLoad If UnaryOpNot NameLoad body Assign NameStore Call NameLoad Assign NameStore BinOpMod Str NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Assign NameStore Call NameLoad Assign NameStore Call AttributeLoad NameLoad attr Assign NameStore BinOpMod Str NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Assign NameStore Call NameLoad NameLoad Assign NameStore Call AttributeLoad NameLoad attr If UnaryOpNot NameLoad body Assign NameStore BinOpMod Str NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Raise Call NameLoad NameLoad Assign NameStore Num Assign NameStore SubscriptLoad NameLoad Index Num Assign NameStore SubscriptLoad NameLoad Index Num If NameLoad body Assign NameStore Str Expr Call AttributeLoad NameLoad attr NameLoad Raise Call NameLoad NameLoad Assign NameStore Call NameLoad NameLoad Assign NameStore Call AttributeLoad NameLoad attr Assign NameStore Call AttributeLoad NameLoad attr Assign NameStore Call AttributeLoad NameLoad attr If UnaryOpNot Call AttributeLoad NameLoad attr NameLoad body If NameLoad body Assign NameStore BinOpMod Str TupleLoad NameLoad NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Return NameLoad Assign NameStore Num While NameLoad body AugAssignAdd NameStore Num Assign NameStore Call AttributeLoad NameLoad attr If UnaryOpNot NameLoad body Break TryExcept body Assign NameStore Call AttributeLoad NameLoad attr NameLoad NameLoad Expr Call NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad NameLoad handlers ExceptHandler type NameLoad body Assign NameStore BinOpMod Str TupleLoad NameLoad NameLoad SubscriptLoad NameLoad Index Num Expr Call AttributeLoad NameLoad attr NameLoad Continue Expr Call AttributeLoad NameLoad attr Expr Call AttributeLoad NameLoad attr Assign NameStore Call AttributeLoad NameLoad attr Assign NameStore Num For NameStore NameLoad body AugAssignAdd NameStore SubscriptLoad NameLoad Index NameLoad Assign NameStore BinOpMod Str TupleLoad NameLoad NameLoad NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Assign NameStore Call NameLoad NameLoad Assign NameStore BinOpMod Str TupleLoad NameLoad NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Assign NameStore BinOpMod Str NameLoad Expr Call AttributeLoad NameLoad attr NameLoad Assign NameStore BinOpMod Str NameLoad Expr Call AttributeLoad NameLoad attr NameLoad decorator_list Expr Str
N/A N/A sys time N/A N/A N/A N/A sys path append ../../ parsing.filing filing parsing.form_parser form_parser ParserMissingError fec_alerts.utils.form_mappers * write_csv_to_db CSV_dumper fec_import_logging fec_logger hstore_helpers dict_to_hstore db_utils get_connection N/A verbose True FilingHeaderDoesNotExist N/A Exception N/A N/A N/A FilingHeaderAlreadyProcessed N/A Exception N/A N/A N/A process_body_row N/A N/A linedict filingnum header_id is_amended cd filer_id N/A N/A N/A form N/A linedict N/A form_parser N/A N/A N/A N/A N/A linedict N/A memo_code X N/A N/A N/A linedict N/A superceded_by_amendment True N/A N/A N/A KeyError N/A N/A N/A N/A form SchA N/A N/A N/A skeda_from_skedadict linedict filingnum header_id is_amended cd N/A N/A N/A form SchB N/A N/A N/A skedb_from_skedbdict linedict filingnum header_id is_amended cd N/A N/A N/A form SchE N/A N/A N/A skede_from_skededict linedict filingnum header_id is_amended cd N/A N/A N/A form F65 N/A N/A N/A skeda_from_f65 linedict filingnum header_id is_amended cd N/A N/A N/A form F56 N/A N/A N/A skeda_from_f56 linedict filingnum header_id is_amended cd N/A N/A N/A form F92 N/A N/A N/A skeda_from_f92 linedict filingnum header_id is_amended cd N/A N/A N/A form F132 N/A N/A N/A skeda_from_f132 linedict filingnum header_id is_amended cd N/A N/A N/A form F133 N/A N/A N/A skeda_from_f133 linedict filingnum header_id is_amended cd N/A N/A N/A form F57 N/A N/A N/A skede_from_f57 linedict filingnum header_id is_amended cd N/A N/A N/A otherline_from_line linedict filingnum header_id is_amended cd filer_id N/A process_filing_body N/A N/A filingnum fp logger N/A None None N/A N/A N/A fp N/A N/A fp N/A form_parser N/A N/A logger N/A N/A logger N/A fec_logger N/A msg N/A process_filing_body: Starting # %s filingnum N/A N/A N/A logger info msg N/A connection N/A get_connection N/A cursor N/A N/A connection cursor N/A cmd N/A select fec_id, is_superceded, data_is_processed from fec_alerts_new_filing where filing_number=%s filingnum N/A N/A N/A cursor execute cmd N/A cd N/A CSV_dumper connection N/A result N/A N/A cursor fetchone N/A N/A result N/A N/A msg N/A process_filing_body: Couldn't find a new_filing for filing %s filingnum N/A N/A N/A logger error msg N/A N/A FilingHeaderDoesNotExist msg N/A header_id 1 N/A is_amended N/A result N/A 1 N/A is_already_processed N/A result N/A 2 N/A is_already_processed N/A N/A msg process_filing_body: This filing has already been entered. N/A N/A N/A logger error msg N/A N/A FilingHeaderAlreadyProcessed msg N/A f1 N/A filing filingnum N/A form N/A N/A f1 get_form_type N/A version N/A N/A f1 get_version N/A filer_id N/A N/A f1 get_filer_id N/A N/A N/A N/A fp is_allowed_form form N/A N/A verbose N/A N/A msg N/A process_filing_body: Not a parseable form: %s - %s N/A form filingnum N/A N/A N/A logger error msg N/A None N/A linenum 0 N/A True N/A N/A linenum 1 N/A row N/A N/A f1 get_body_row N/A N/A row N/A N/A N/A N/A N/A linedict N/A N/A fp parse_form_line row version N/A N/A process_body_row linedict filingnum header_id is_amended cd filer_id N/A N/A N/A ParserMissingError N/A N/A msg N/A process_filing_body: Unknown line type in filing %s line %s: type=%s Skipping. N/A filingnum linenum N/A row N/A 0 N/A N/A N/A logger warn msg N/A N/A N/A N/A cd commit_all N/A N/A N/A cd close N/A counter N/A N/A cd get_counter N/A total_rows 0 N/A i counter N/A N/A total_rows N/A counter N/A i N/A msg N/A process_filing_body: Filing # %s Total rows: %s Tally is: %s N/A filingnum total_rows counter N/A N/A N/A logger info msg N/A header_data N/A dict_to_hstore counter N/A cmd N/A update fec_alerts_new_filing set lines_present='%s'::hstore where filing_number=%s N/A header_data filingnum N/A N/A N/A cursor execute cmd N/A cmd N/A update fec_alerts_new_filing set data_is_processed = True where filing_number=%s filingnum N/A N/A N/A cursor execute cmd N/A cmd N/A update summary_data_committee_overlay set is_dirty=True where fec_id='%s' filer_id N/A N/A N/A cursor execute cmd N/A N/A 
t0 = time.time()
process_filing_body(864353)
# 869853, 869866
#for fn in [869888]:
#    process_filing_body(fn, fp)
t1 = time.time()
print "total time = " + str(t1-t0)
# long one: 767168
#FAILS WITH STATE ADDRESS PROBLEM:  biggest one on file: 838168 (510 mb) - act blue - 2012-10-18         | 2012-11-26
# second biggest: 824988 (217.3mb) - act blue - 2012-10-01         | 2012-10-17 - 874K lines
# 840327 - 169MB  C00431445 - OFA   | 2012-10-18         | 2012-11-26
# 821325 - 144 mb Obama for america 2012-09-01         | 2012-09-30
# 798883 - 141 mb
# 804867 - 127 mb
# 827978 - 119 mb
# 754317 - 118 mb

