Scrapy蜘蛛（Spider）

scrapy.Spider

class scrapy.spiders.Spider

Spider参数

scrapy crawl first_scrapy -a group = accessories

import scrapy

class FirstSpider(scrapy.Spider):
    name = "first"

    def __init__(self, group=None, *args, **kwargs):
        super(FirstSpider, self).__init__(*args, **kwargs)
        self.start_urls = ["http://www.yiibai.com/group/%s" % group]

通用Spider

您可以使用通用蜘蛛来创建子类蜘蛛。他们的目的是要根据一定的规则来提取所有在网站上的所有链接的数据。

import scrapy
from scrapy.item import Item, Field
	
class First_scrapyItem(scrapy.Item):
    product_title = Field()
    product_link = Field()
    product_description = Field()

CrawlSpider

class scrapy.spiders.CrawlSpider

rules

parse_start_url(response)

它通过允许解析初步回应返回项目或请求对象。

注意: 请务必重命名等函数解析而不是编写规则，因为解析函数用于CrawlSpider来实现它的逻辑。

让我们看看下面的例子开始演示蜘蛛爬行 example.com 首页，使用 parse_items 方法收集所有页面上的链接和词组：

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor

class DemoSpider(CrawlSpider):
    name = "demo"
    allowed_domains = ["www.yiibai.com"]
    start_urls = ["http://www.yiibai.com"]

    rules = (
    Rule(LinkExtractor(allow =(), restrict_xpaths = ("//div[@class = 'next']",)), callback = "parse_item", follow = True),
    )

    def parse_item(self, response):
        item = DemoItem()
        item["product_title"] = response.xpath("a/text()").extract()
        item["product_link"] = response.xpath("a/@href").extract()
        item["product_description"]  = response.xpath("div[@class='desc']/text()").extract()
        return items

XMLFeedSpider

class scrapy.spiders.XMLFeedSpider

CSVFeedSpider

class scrapy.spiders.CSVFeedSpider

CSVFeedSpider 示例:

from scrapy.spiders import CSVFeedSpider
from demoproject.items import DemoItem

class DemoSpider(CSVFeedSpider):
    name = "demo"
    allowed_domains = ["www.yiibai.com"]
    start_urls = ["http://www.yiibai.com/feed.csv"]
    delimiter = ";"
    quotechar = "'"
    headers = ["product_title", "product_link", "product_description"]

    def parse_row(self, response, row):
        self.logger.info("This is row: %r", row)

        item = DemoItem()
        item["product_title"] = row["product_title"]
        item["product_link"] = row["product_link"]
        item["product_description"] = row["product_description"]
        return item

SitemapSpider

站点地图(sitemap)帮助蜘蛛通过 robots.txt 的定位网址并抓取网站。它有以下类：

class scrapy.spiders.SitemapSpider

SitemapSpider 示例:

from scrapy.spiders import SitemapSpider

class DemoSpider(SitemapSpider):
    urls = ["http://www.yiibai.com/sitemap.xml"]

    def parse(self, response):
        # You can scrap items here

from scrapy.spiders import SitemapSpider

class DemoSpider(SitemapSpider):
    urls = ["http://www.yiibai.com/sitemap.xml"]
    rules = [
        ("/item/", "parse_item"),
        ("/group/", "parse_group"),
    ]

    def parse_item(self, response):
        # you can scrap item here

    def parse_group(self, response):
        # you can scrap group here  

下面的代码显示了跟踪站点地图，在 robots.txt 中的网址有 /sitemap_company:

from scrapy.spiders import SitemapSpider

class DemoSpider(SitemapSpider):
    urls = ["http://www.yiibai.com/robots.txt"]
    rules = [
        ("/company/", "parse_company"),
    ]
    sitemap_follow = ["/sitemap_company"]

    def parse_company(self, response):
        # you can scrap company here

from scrapy.spiders import SitemapSpider

class DemoSpider(SitemapSpider):
    urls = ["http://www.yiibai.com/robots.txt"]
    rules = [
        ("/company/", "parse_company"),
    ]

    other_urls = ["http://www.yiibai.com/contact-us"]

    def start_requests(self):
        requests = list(super(DemoSpider, self).start_requests())
        requests += [scrapy.Request(x, self.parse_other) for x in self.other_urls]
        return requests

    def parse_company(self, response):
        # you can scrap company here...
		
    def parse_other(self, response):
        # you can scrap other here...