Q:

Scrapy not working as expected

This is my Scrapy spider.  I'm trying to scrape some data from web.  But I don't know how to force Scrapy to follow links recursively.  Where is my mistake? 
import re
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from YellowPagesOfMoldova.items import YellowpagesofmoldovaItem
from scrapy.item import Item

class YellowSpider(CrawlSpider):
    name = 'yellow'
    allowed_domains = ['yellowpages.md']
    start_urls = [ 'http://www.yellowpages.md/eng/companies/info/8939-arc-publishing-house']
    rules = (
        Rule(SgmlLinkExtractor(allow=('eng.+')), follow=True),
        )

    def parse(self, response):

        sel = Selector(response)
        i = YellowpagesofmoldovaItem()
        i['url']            = response.url
        i['locality']       = sel.xpath("//tr[3]/td/p[1]/span[1]/text()").extract()
        i['title']          = sel.xpath('//title/text()').extract()
        i['title2']         = sel.xpath("//td/h1/text()").extract()
        i['website']        = sel.xpath("//p[2]/a/text()").extract()
        i['activity']       = sel.xpath("//tbody/tr[4]/td/p/text()").extract()
        i['street']         = sel.xpath("//tr/td/p[1]/span[2]/text()").extract()
        return i

Thanks. 
I resolved this trouble.  Now it works perfectly.  It looks so:
import re
from scrapy.selector import HtmlXPathSelector
from scrapy.contrib.spiders import CrawlSpider, Rule
from scrapy.contrib.linkextractors.sgml import SgmlLinkExtractor
from scrapy.selector import Selector
from YellowPagesOfMoldova.items import YellowpagesofmoldovaItem
from scrapy.item import Item

class YellowSpider(CrawlSpider):
    name = 'yellow'
    allowed_domains = ['yellowpages.md']
    start_urls = [ 'http://www.yellowpages.md/eng/companies/info/8939-arc-publishing-house']
    rules = (
        Rule(SgmlLinkExtractor(allow=('eng.+')),callback='parse_items', follow=True),

        )

    def parse_items(self, response):

        sel = Selector(response)
        i = YellowpagesofmoldovaItem()
        i['url']            = response.url
        i['locality']       = sel.xpath("//tr[3]/td/p[1]/span[1]/text()").extract()
        i['title']          = sel.xpath('//title/text()').extract()
        i['title2']         = sel.xpath("//td/h1/text()").extract()
        i['website']        = sel.xpath("//p[2]/a/text()").extract()
        i['activity']       = sel.xpath("//tbody/tr[4]/td/p/text()").extract()
        i['street']         = sel.xpath("//tr/td/p[1]/span[2]/text()").extract()
        return i

A:

CrawlSpider's parse method should not be overridden because that's where all the "magic" happens. ( See the warning about parse here). 
Change def parse to def parse_page and reference this callback in your Rule: Rule(SgmlLinkExtractor(allow=('eng.+')), callback='parse_page', follow=True),

