Issue 1:
I have rules set like this

rules = [
Rule(LinkExtractor(
allow= '/topic/\d+/organize$',
restrict_xpaths = '//div[@id= "zh-topic-organize-child-editor"]'
),
process_request='request_tagPage', callback = "parse_tagPage", follow = True)
]

request_tagPage() is used to add cookie and headers to a request. I found that once I used process_request parameter, the callback function parse_tagPage() isn't get called.

Then I manually set parse_tagPage() as callback function in the request_tagPage(). Now when response is returned, parse_tagPage() is called but the spider only crawls the links from the start_urls

My full spider is here:

class ZhihuSpider(CrawlSpider):
name = "zhihu"
BASE_URL = "www.zhihu.com"
get_xsrf_url = "https://www.zhihu.com" # url to visit first to get xsrf information
login_url = "https://www.zhihu.com/login/email" # url to visit to login and get valid cookie
start_urls = [
"https://www.zhihu.com/topic/19776749/organize",

]
headers = {
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding": "gzip,deflate",
"Accept-Language": "en-US,en;q=0.8,zh-TW;q=0.6,zh;q=0.4",
"Connection": "keep-alive",
"Content-Type":" application/x-www-form-urlencoded; charset=UTF-8",
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
"Referer": "http://www.zhihu.com"
}

# Rules to enforce using cookie every time request a tag information page
#restrict_xpaths = '//div[@id= "zh-topic-organize-page-children"]/ul/li/ul[@class= "zm-topic-organize-list"]',
rules = [
Rule(LinkExtractor(
allow= '/topic/\d+/organize$',
restrict_xpaths = '//div[@id= "zh-topic-organize-child-editor"]'
),
process_request='request_tagPage', callback = "parse_tagPage", follow = True)
# 发现match rule的页面会用parse_tagPage去处理
# follow = True，这样页面被parse_tagPage处理后，还是会被抓内部的link去继续crawling
] # 使用list，rules就自动成为iterable

# 用来保存tag结构的大dictionary，会在item pipeline中得到更新
d = {"「根话题」":{}}

# 用来保存每一条tag的list，会在item pipeline中得到更新
l = []

# Function to get the login response; Only called once
# Scrapy刚启动时会call这个函数，函数的目的是拿到xsrf信息
def start_requests(self):
print("---"*5)
print("start to request for getting the hidden info and cookie")
print("---"*5)
return [Request(self.get_xsrf_url, headers= self.headers, meta= \
{"cookiejar":1}, callback= self.post_login)]

# Function to post a login form, notice it gets the xsrf string first before send the form
# 这个函数会提取只有试图login知乎时才会得到的xsrf信息来构建一个登录form，然后得到登录成功的cookie
def post_login(self, response):
print("---"*5)
print("preparing login...")
print("---"*5)
# Get the xsrf string
xsrf = Selector(response).xpath('//div[@data-za-module="SignInForm"]//form//input[@name="_xsrf"]/@value').extract()[0]
return FormRequest(self.login_url,
meta = {"cookiejar": response.meta["cookiejar"]},
headers = self.headers,
# create form
formdata = {
"_xsrf": xsrf,
"password": "zhihu_19891217",
"email": "skywalker.ljc@gmail.com",
"remeber_me": "true",
},
callback = self.after_login,
)

# After login, this function request urls in the start_urls, initiate the whole process
# 这个函数会给登录成功的cookie给start_urls, 这样start_urls也会带着cookie去request
def after_login(self, response):
for url in self.start_urls:
# No need to callback since the rules has set the process_request parameter,
# which specifies a function to send the actual request.
yield Request(url, meta = {"cookiejar": 1}, headers = self.headers, \
callback = self.parse, dont_filter = True)
# self.parse is the default parser used by CrawlSpider to apply rules
# dont_filter = True so that this start_url request won't be filtered
print("A start_url has been requested:", url)
print("---"*5)
print("All start_urls cookies are have been requested!")
print("---"*5)

# Function to request tag information page
# 这个函数是为了让scrapy爬后续的页面时也会带上cookie；
# CrawlSpider会首先用最低级的Request()去形成基础的request， 但是基础request无法通过zhihu.com反爬虫机制
# 所以在rules中要求spider再用这个函数加工基础request成带cookie和header的能通过zhihu反爬虫机制的request
def request_tagPage(self, request):
return Request(request.url, meta = {"cookiejar": 1}, \
headers = self.headers, callback=self.parse_tagPage)
# When use process_request, the callback in Rule object won't work, has to assign a callback here

# Finally, the function to actually parse the tag information page
# 这个函数才是真正接受知乎的tag结构页面并且parse页面的
# 这个函数还会根据每一个tag的信息，修改spider用来保存tag structure的dictionary
def parse_tagPage(self, response):
print("---"*5)
print("parse_tagPage is called!")
print("---"*5)
sel = Selector(response)

# tag的名字和链接
name = sel.xpath('//h1[@class= "zm-editable-content"]/text()').extract()[0]
relative_link = sel.xpath('//div[@class= "zm-topic-topbar"]//a/@href').extract()[0]

# tag的parent
parents = sel.xpath('//div[@id= "zh-topic-organize-parent-editor"]//a[@class= "zm-item-tag"]/text()').extract()
parents = [s.replace("\n", "") for s in parents]

# tag的children
children = sel.xpath('//div[@id= "zh-topic-organize-child-editor"]//a[@class= "zm-item-tag"]/text()').extract()
children = [s.replace("\n", "") for s in children]

# 新建一个tag item
item = {}

item["name"] = name
item["relative_link"] = relative_link
item["parents"] = parents
item["children"] = children

self.l.append(item)

Issue 2:
Was trying to send email through scrapy.

Settings I used are:-

#settings for mail

MAIL_FROM = '...@gmail.com'

MAIL_HOST = 'smtp.gmail.com'

MAIL_PORT = 587

MAIL_USER = '...@gmail.com'

MAIL_PASS = 'xxxxxxx'

MAIL_TLS = True

MAIL_SSL = False

Email was sent successfully. But the following exception is thrown;

2019-04-25 18:10:15 [twisted] CRITICAL: Unhandled Error

Traceback (most recent call last):

File "/lib/python3.6/site-packages/twisted/python/log.py", line 103, in callWithLogger

return callWithContext({"system": lp}, func, *args, **kw)

File "/python3.6/site-packages/twisted/python/log.py", line 86, in callWithContext

return context.call({ILogContext: newCtx}, func, *args, **kw)

File "/lib/python3.6/site-packages/twisted/python/context.py", line 122, in callWithContext

return self.currentContext().callWithContext(ctx, func, *args, **kw)

File "/lib/python3.6/site-packages/twisted/python/context.py", line 85, in callWithContext

return func(*args,**kw)

--- ---

File "/lib/python3.6/site-packages/twisted/internet/posixbase.py", line 614, in _doReadOrWrite

why = selectable.doRead()

File "/lib/python3.6/site-packages/twisted/internet/tcp.py", line 243, in doRead

return self._dataReceived(data)

File "/lib/python3.6/site-packages/twisted/internet/tcp.py", line 249, in _dataReceived

rval = self.protocol.dataReceived(data)

File "/lib/python3.6/site-packages/twisted/protocols/tls.py", line 330, in dataReceived

self._flushReceiveBIO()

File "/lib/python3.6/site-packages/twisted/protocols/tls.py", line 300, in _flushReceiveBIO

self._flushSendBIO()

File "/lib/python3.6/site-packages/twisted/protocols/tls.py", line 252, in _flushSendBIO

bytes = self._tlsConnection.bio_read(2 ** 15)

builtins.AttributeError: 'NoneType' object has no attribute 'bio_read'