Issue 1:
Description

I'm running a Scrapy crawler from PyCharm's Python Console:

In my code (below), I export the scraped content to CSV files through CsvItemExporter. When run from PyCharm's Python Console (using both configurations above), the scraper runs fine, but doesn't write to the CSV files; they are 0 bytes long after the crawler runs. However, when I run scrapy from the command line (scrapy crawl disasters) or from PyCharm's debugger, suddenly it writes to the CSV files as intended.

I tried this using XmlItemExporter as well. Same results.

How can I run this using PyCharm's Python Console?

Here is my code:

spiders/disasters.py

from datetime import datetime as dt
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from asnscraper.items import DisasterRaw, AirportRaw
import re

RE_LISTING = r'.*/dblist\.php\?(Year)=(1920)$'
RE_RECORD = r'.*/record\.php\?id=((\d{8})-(\d+))$'
RE_AIRPORT = r'.*/airport\.php\?id=(\w+)$'

class DisasterSpider(CrawlSpider):
name = 'disasters'
allowed_domains = ['aviation-safety.net']
start_urls = ['https://aviation-safety.net/database/']
rules = [
Rule(LinkExtractor(allow=RE_LISTING), callback='parse_list', follow=True),
Rule(LinkExtractor(allow=RE_RECORD), callback='parse_record', follow=True),
Rule(LinkExtractor(allow=RE_AIRPORT), callback='parse_airport', follow=False)
]

@staticmethod
def parse_list(response):
m = re.match(RE_LISTING, response.url)
print(f"Database listing WHERE '{m[1]}' = '{m[2]}'")

@staticmethod
def parse_airport(response):
id = re.match(RE_AIRPORT, response.url)
airportraw = AirportRaw()
airportfields = response.xpath("//a[@name='general']/div[@class='infobox']//table[1]//tr")
airportname = response.xpath("//a[@name='general']/div[@class='infobox']/span/text()")
airportraw['id'] = id[1]
airportraw['name'] = airportname
airportraw['allfields'] = airportfields
print(f"Airport listing WHERE 'id' = '{airportraw['id']}'")
return airportraw

@staticmethod
def parse_record(response):
id = re.match(RE_RECORD, response.url)
disasterraw = DisasterRaw()
disasterfields = response.xpath("//div[@class='innertube']//table[1]//tr")
disasterraw['id'] = id[1]
disasterraw['date'] = dt.strptime(id[2], '%Y%m%d')
disasterraw['allfields'] = disasterfields
print(f"Acident listing WHERE 'id' = '{disasterraw['id']}'")
return disasterraw

items.py

import scrapy

class DisasterRaw(scrapy.Item):
id = scrapy.Field()
allfields = scrapy.Field()
date = scrapy.Field()
ap_from = scrapy.Field()
ap_to = scrapy.Field()

class Disaster(scrapy.Item):
id = scrapy.Field()
datetime = scrapy.Field()
type = scrapy.Field()
operator = scrapy.Field()
aircraft = scrapy.Field()
crew_deaths = scrapy.Field()
crew_total = scrapy.Field()
passenger_deaths = scrapy.Field()
passenger_total = scrapy.Field()
ground_deaths = scrapy.Field()
souls_deaths = scrapy.Field()
souls_total = scrapy.Field()
damage = scrapy.Field()
fate = scrapy.Field()
location = scrapy.Field()
flightphase = scrapy.Field()
nature = scrapy.Field()
ap_from = scrapy.Field()
ap_to = scrapy.Field()
flightnumber = scrapy.Field()
narrative = scrapy.Field()

class AirportRaw(scrapy.Item):
id = scrapy.Field()
name = scrapy.Field()
allfields = scrapy.Field()

class Airport(scrapy.Item):
id = scrapy.Field()
name = scrapy.Field()
country = scrapy.Field()
iata = scrapy.Field()
icao = scrapy.Field()
elevation = scrapy.Field()
elevation_unit = scrapy.Field()
dateopened = scrapy.Field()

pipelines.py

import os.path
from datetime import timedelta as td
from scrapy.exporters import CsvItemExporter
from scrapy.exceptions import DropItem
from asnscraper.items import Disaster, DisasterRaw, Airport, AirportRaw

import re

DATADIR = r'..\data'

class ExportPipeline:
dbs = [Airport, Disaster]
exporters = {}

@staticmethod
def dbname(db):
return db.__name__.lower()

@staticmethod
def file_path(file, folder = DATADIR):
return os.path.abspath(os.path.join(folder, file))

def open_spider(self, spider):

for db in self.dbs:
db_name = self.dbname(db)
print(f"Exporting {db_name}s to {db_name}s.csv")
e = CsvItemExporter(open(self.file_path(f"{db_name}s.csv"), 'wb'))
self.exporters[db_name] = e
self.exporters[db_name].start_exporting()

return spider

def close_spider(self, spider):
for k, e in self.exporters.items():
e.finish_exporting()

return spider

def process_item(self, item, spider):
for db in self.dbs:
if isinstance(item, db):
db_name = self.dbname(db)
self.exporters[db_name].export_item(item)
return item

class AirportPipeline:

def process_item(self, raw, spider):
if not isinstance(raw, AirportRaw):
return raw
# df = raw['allfields']
airport = Airport()
airport['id'] = raw['id']
airport['name'] = raw['name'].extract()[0].strip()
return airport

class DisasterPipeline:

def process_item(self, raw, spider):
if not isinstance(raw, DisasterRaw):
return raw
df = raw['allfields']
disaster = Disaster()
disaster['id'] = raw['id']
timestr = extract_field(field='Time', df=df)
try:
hours, minutes = re.match(r'(\d\d):(\d\d)', timestr).groups()
except (AttributeError, TypeError):
hours, minutes = (0, 0)
finally:
deltat = td(hours=(int(hours)), minutes=(int(minutes)))
total_dt = raw['date'] + deltat
disaster['datetime'] = total_dt
disaster['crew_deaths'], disaster['crew_total'] = fatalities('Crew', df=df)
disaster['passenger_deaths'], disaster['passenger_total'] = \
fatalities('Passengers', df=df)
disaster['souls_deaths'], disaster['souls_total'] = fatalities('Total', df=df)
disaster['ground_deaths'] = fatalities_ground(df=df)
disaster['damage'] = extract_field(field='Aircraft damage', df=df)
disaster['fate'] = extract_field(field='Aircraft fate', df=df)
disaster['aircraft'] = extract_field(field='Registration', df=df)
disaster['location'] = extract_field(field='Location', df=df)
disaster['flightphase'] = extract_field(field='Phase', df=df)
disaster['nature'] = extract_field(field='Nature', df=df)
disaster['ap_from'] = extract_field(field='Departure airport', df=df)
disaster['ap_to'] = extract_field(field='Destination airport', df=df)
disaster['flightnumber'] = extract_field(field='Flightnumber', df=df)
return disaster

def extract_field(field, df):
query = f'td[@class="caption" and text()="{field}:"]/following-sibling::td//text()'
query2 = f'td[@class="caption"]//*[text()="{field}:"]/' \
f'ancestor::td/following-sibling::td//text()'
ret = df.xpath(query).extract() or df.xpath(query2).extract()
if isinstance(ret, list):
if len(ret) == 0:
return
ret = [str(r) for r in ret]
return ''.join(ret).strip()
if isinstance(ret, str):
return ret.strip()
return ret

def fatalities(field, df):
fstr = extract_field(field=field, df=df)
if fstr is None:
return
fatal = re.match(r'.*[Ff]atalities:\s*(\d*)\s*/\s*[Oo]ccupants:\s*(\d*).*', fstr)
f_deaths = int(fatal[1]) if fatal[1] != '' else None
f_total = int(fatal[2]) if fatal[2] != '' else None
return (
f_deaths, f_total)

def fatalities_ground(df):
fstr = extract_field(field='Ground casualties', df=df)
if fstr is None:
return
fatal = re.match(r'.*[Ff]atalities:\s*(\d*)\s*.*', fstr)
if fatal is None:
return
f_deaths = int(fatal[1]) if fatal[1] != '' else None
return f_deaths

Steps to Reproduce

Start a blank Scrapy project and copy the code above to the relevant files. Other files are default;

Configure PyCharm to run it in the Python Console, as indicated in my screenshots;

Run the crawler;

Open the generated csv files. They are empty.

Run the crawler through PyCharm's Debugger;

Open the generated csv files again. They are not empty.

Run the crawler through the command line.

Open the generated csv files. They are the same as in step 6.

Expected behavior: Everytime I open the generated csv files, they should be exactly the same regardless of how I run it.

Actual behavior: They are not the same. When running the above crawler through PyCharm's Python Console, they are empty. Otherwise, they are not empty.

Reproduces how often: Always happens

Versions

Python version: Python 3.8.5 (default, Aug 5 2020, 09:44:06) [MSC v.1916 64 bit (AMD64)] on win32

Anaconda version:

> conda info

active environment : asn
active env location : C:\ProgramData\Anaconda3\envs\asn
shell level : 2
user config file : C:\Users\*\.condarc
populated config files :
conda version : 4.8.4
conda-build version : 3.18.11
python version : 3.7.6.final.0
virtual packages : __cuda=9.1
base environment : C:\ProgramData\Anaconda3 (writable)
channel URLs : https://repo.anaconda.com/pkgs/main/win-64
https://repo.anaconda.com/pkgs/main/noarch
https://repo.anaconda.com/pkgs/r/win-64
https://repo.anaconda.com/pkgs/r/noarch
https://repo.anaconda.com/pkgs/msys2/win-64
https://repo.anaconda.com/pkgs/msys2/noarch
package cache : C:\ProgramData\Anaconda3\pkgs
C:\Users\*\.conda\pkgs
C:\Users\*\AppData\Local\conda\conda\pkgs
envs directories : C:\ProgramData\Anaconda3\envs
C:\Users\*\.conda\envs
C:\Users\*\AppData\Local\conda\conda\envs
platform : win-64
user-agent : conda/4.8.4 requests/2.22.0 CPython/3.7.6 Windows/10 Windows/10.0.18362
administrator : False
netrc file : None
offline mode : False

I don't know why conda says the Python version is 3.7.6. It is in fact 3.8.5, as indicated above.

Scrapy version

> scrapy version --verbose
Scrapy : 1.6.0
lxml : 4.5.2.0
libxml2 : 2.9.10
cssselect : 1.1.0
parsel : 1.5.2
w3lib : 1.21.0
Twisted : 20.3.0
Python : 3.8.5 (default, Aug 5 2020, 09:44:06) [MSC v.1916 64 bit (AMD64)]
pyOpenSSL : 19.1.0 (OpenSSL 1.1.1g 21 Apr 2020)
cryptography : 2.9.2
Platform : Windows-10-10.0.18362-SP0

Additional Information

Posted to Stack Overflow on 2020-09-02, no answers so far.

Issue 2:
Proxy CONNECT support is required for https urls to work.

When proxy support was added to Scrapy, urllib still didn't supported CONNECT, but that has been fixed now:

http://bugs.python.org/issue1424152

Now we need to add support to Scrapy (which doesn't use urllib at all).

This link contains useful tips on how to add CONNECT support with Twisted:

http://twistedmatrix.com/pipermail/twisted-web/2008-August/003878.html

see http://dev.scrapy.org/ticket/159 for more info