Repository: DadosAbertosDeFeira/maria-quiteria

Issue:
Coletar despesas
http://www.transparencia.feiradesantana.ba.gov.br/index.php?view=despesasinfo



Discussion:

Resolving patch:
diff --git a/scraper/scraper/settings.py b/scraper/scraper/settings.py
--- a/scraper/scraper/settings.py
+++ b/scraper/scraper/settings.py
@@ -27,7 +27,7 @@
 # Configure a delay for requests for the same website (default: 0)
 # See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
 # See also autothrottle settings and docs
-#DOWNLOAD_DELAY = 3
+# DOWNLOAD_DELAY = 3
 # The download delay setting will honor only one of:
 #CONCURRENT_REQUESTS_PER_DOMAIN = 16
 #CONCURRENT_REQUESTS_PER_IP = 16
@@ -70,7 +70,7 @@
 
 # Enable and configure the AutoThrottle extension (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/autothrottle.html
-#AUTOTHROTTLE_ENABLED = True
+# AUTOTHROTTLE_ENABLED = True
 # The initial download delay
 #AUTOTHROTTLE_START_DELAY = 5
 # The maximum download delay to be set in case of high latencies
@@ -83,8 +83,8 @@
 
 # Enable and configure HTTP caching (disabled by default)
 # See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
-#HTTPCACHE_ENABLED = True
-#HTTPCACHE_EXPIRATION_SECS = 0
+HTTPCACHE_ENABLED = True
+HTTPCACHE_EXPIRATION_SECS = 86400  # 24 hours
 #HTTPCACHE_DIR = 'httpcache'
 #HTTPCACHE_IGNORE_HTTP_CODES = []
 #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
diff --git a/scraper/scraper/spiders/cityhall.py b/scraper/scraper/spiders/cityhall.py
--- a/scraper/scraper/spiders/cityhall.py
+++ b/scraper/scraper/spiders/cityhall.py
@@ -1,9 +1,11 @@
+from datetime import datetime, date, timedelta
 import re
+
 import scrapy
 
 
 class LawsSpider(scrapy.Spider):
-    """Parse laws and acts of Feira de Santana city hall until 2015.
+    """Coleta leis e decretos de Feira de Santana até 2015.
 
     Years: 1999 to 2015
     Example: http://www.feiradesantana.ba.gov.br/seadm/leis.asp?acao=ir&p=24&ano=2015
@@ -29,12 +31,12 @@ def parse(self, response):
             for row in rows:
                 act = self.extract_acts(row)
                 yield {
-                    'titulo': act['act_and_date'],
-                    'jornal_publicado': act['journal'],
-                    'data': act['date'],
-                    'detalhes': act['details'],
+                    'title': act['act_and_date'],
+                    'published_on': act['journal'],
+                    'date': act['date'],
+                    'details': act['details'],
                     'url': act['url'],
-                    'fonte': response.url,
+                    'source': response.url,
                 }
 
             current_page = self.get_current_page(response)
@@ -232,7 +234,7 @@ def parse(self, response):
             yield scrapy.FormRequest(self.url, formdata=data, callback=self.parse_page)
 
     def parse_page(self, response):
-        """Extract contracts from a page.
+        """Extrai informações sobre um contrato.
 
         Example:
         CONTRATO N° 11-2017-1926C   REFERENTE A CONTRATAÇÃO DE EMPRESA AQUISIÇÃO DE
@@ -283,3 +285,91 @@ def clean_details(self, raw_details):
                 # assuming that all fields will be there
                 valid_details.append(details)
         return valid_details
+
+
+class PaymentsSpider(scrapy.Spider):
+    """Coleta pagamentos realizados.
+
+    http://www.transparencia.feiradesantana.ba.gov.br/index.php?view=despesa
+    """
+    name = 'payments'
+    url = 'http://www.transparencia.feiradesantana.ba.gov.br/controller/despesa.php'
+    data = {
+        'POST_PARAMETRO': 'PesquisaDespesas',
+        'POST_FASE': '',
+        'POST_UNIDADE': '',
+        'POST_DATA': '',
+        'POST_NMCREDOR': '',
+        'POST_CPFCNPJ': ''
+    }
+
+    def start_requests(self):
+        current_date = date(2010, 1, 1)  # initial date
+        today = datetime.now().date()
+
+        while current_date <= today:
+            formatted_date = current_date.strftime('%d/%m/%Y')
+            data = self.data.copy()
+            data['POST_DATA'] = f'{formatted_date} - {formatted_date}'
+            yield scrapy.FormRequest(
+                self.url, formdata=data, callback=self.parse, meta={'data': data})
+            current_date = current_date + timedelta(days=1)
+
+    def parse(self, response):
+        # ['��� Anterior', '1', '2', '33', 'Pr��ximo ���']
+        pages = response.css('div.pagination li a ::text').extract()
+        if pages:
+            last_page = int(pages[-2])
+
+            for page in range(1, last_page + 1):
+                data = response.meta['data']
+                data['POST_PAGINA'] = str(page)
+                data['POST_PAGINAS'] = str(last_page)
+                yield scrapy.FormRequest(self.url, formdata=data, callback=self.parse_page)
+
+    def parse_page(self, response):
+        """Extrai informações sobre um pagamento.
+
+        Exemplo:
+        N°: 19000215/0004 	CPF/CNPJ: 90.180.605/0001-02 	\
+            Data: 22/10/2019 		N° do processo: 010-2019
+        Bem / Serviço Prestado: REFERENTE A DESPESA COM SEGURO DE VIDA.
+        Natureza: 339039999400 - Seguros em Geral
+        Ação: 2015 - Manutencao dos serv.tecnicos administrativos
+        Função: 04 - ADMINISTRACAO
+        Subfunção: 122 - ADMINISTRACAO GERAL
+        Processo Licitatório: PREGAO
+        Fonte de Recurso: 0000 - RECURSOS ORDINARIOS
+        """
+        headlines = response.css('#editable-sample tr.accordion-toggle')
+        details = response.css('#editable-sample div.accordion-inner')
+
+        for headline, raw_details in zip(headlines, details):
+            headline = [text.strip() for text in headline.css('td ::text').extract()]
+            data = {
+                'published_at': headline[0],
+                'phase': headline[1],
+                'company_or_person': headline[2],
+                'value': headline[3],
+            }
+            details = [detail.strip() for detail in raw_details.css('td ::text').extract()]
+            mapping = {
+                'N°:': 'number',
+                'CPF/CNPJ:': 'document',
+                'Data:': 'date',
+                'N° do processo:': 'process_number',
+                'Bem / Serviço Prestado:': 'summary',
+                'Natureza:': 'group',
+                'Ação:': 'action',
+                'Função:': 'function',
+                'Subfunção:': 'subfunction',
+                'Processo Licitatório:': 'type_of_process',
+                'Fonte de Recurso:': 'resource'
+            }
+            details_copy = details.copy()
+            while details_copy:
+                key = details_copy.pop(0)
+                value = details_copy.pop(0)
+                data[mapping[key]] = value
+
+            yield data


Test patch:
diff --git a/scraper/scraper/tests/spiders/compare_crawled_and_reports.py b/scraper/scraper/tests/spiders/compare_crawled_and_reports.py
new file mode 100644
--- /dev/null
+++ b/scraper/scraper/tests/spiders/compare_crawled_and_reports.py
@@ -0,0 +1,35 @@
+"""This is a script to compared crawled data and reports from city hall website."""
+import csv
+
+import xlrd
+
+mapping = {
+    'Data de Publicacao': 'published_at',
+    'Fase ': 'phase',
+    'Numero': 'number',
+    'N do processo': 'process_number',
+    'Bem / Servico Prestado': 'summary',
+    'Credor': 'company_or_person',
+    'CPF / CNPJ': 'document',
+    'Valor': 'value',
+    'Funcao': 'function',
+    'Subfuncao': 'subfunction',
+    'Natureza': 'group',
+    'Fonte': 'resource',
+    'Modalidade': 'type_of_process',
+}
+
+from_report = xlrd.open_workbook('Relatorio-Despesa-2010.xls')
+from_crawler = csv.DictReader(open('pag-2010.csv'), delimiter=',', quotechar='"')
+
+sheet = from_report.sheet_by_index(0)
+header_from_report = sheet.row(0)
+
+print(header_from_report)
+print(from_crawler.fieldnames)
+print('----------------')
+
+for label in header_from_report:
+    print(label.value, mapping.get(label.value))
+
+# TODO check missing data