写入文件时出错

Question

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from Erowid.items import ErowidItem
import os

class ExperiencesSpider(CrawlSpider):
    name = "experiences"
    allowed_domains = ["www.erowid.org"]
    start_urls = ['https://www.erowid.org/experiences/exp_list.shtml']

    rules = [Rule(LinkExtractor(allow =('subs/exp_[a-zA-Z]+.shtml')),callback='parse_item', follow = True)

    Rule(LinkExtractor(allow =('subs/exp_[a-zA-Z]+.shtml')), follow = True)    

    ]

    def parse_item(self, response):
        filename = str(response.url)[44:-6]
        selectors = response.css('table')
        if not os.path.exists('drugs-%s' % (filename)): ##Make the file
            os.makedirs('drugs-%s' % (filename))
        list_of_experience = selectors.xpath('//table[@class="exp-cat-table"]/tr/td/a/@href').extract()

        for item in list_of_experience:
            request_url = str(item)
            Request(url="http://www.erowid.org" + request_url, callback = 'request_experience')
            def request_experience(self, response):
                selectors = response.css('div')
                for selector in selectors:
                    experience = ErowidItem()
                    experience['Author'] = selector.xpath('//div[@class="author"]/a/text()').extract()
                    experience['Title'] = selector.xpath('//div[@class="title"]/text()').extract()
                    experience['Substance'] = selector.xpath('//div[@class="substance"]/text()').extract()
                    experience['Text'] = selector.xpath("//div[@class = 'report-text-surround']/text()").extract()

                    title = str(experience['Substance']) + " "+ str(experience['Title'])
                    with open(os.path.join('drugs-%s' % (filename), title),"a") as fid:
                        fid.write(str(experience) + "\n")

我尝试使用scrapy从Erowid中提取数据，并且尝试格式化数据，这样对于每种物质，我都有一个以“物质-经验标题”的形式命名的文件。你知道吗

我的规则让我的蜘蛛在包含https://www.erowid.org/experiences/subs/exp_Acacia_confusa.shtml的网站列表中爬行。然后我得到了所有指向某个体验的链接，并通过第二个请求将其放入，这次的目的是从体验中收集数据。你知道吗

我打算以上述格式存储数据，即“物质-经验的标题”。对于每种物质，我想建立一个目录，其中包含来自该页面的所有文件。你知道吗

但是，我的代码生成目录，但不编写我想要的文件。你知道吗

是什么导致了这个错误？你知道吗

Answer 1

根据documentation of ^{}-

callback (callable) – the function that will be called with the response of this request (once its downloaded) as its first parameter.

回调应该是可调用函数，而不是它的字符串，在尝试将其作为Request对象的回调发送之前，还需要定义函数。你知道吗

示例-

import scrapy
from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from scrapy.selector import HtmlXPathSelector
from scrapy.http import Request
from Erowid.items import ErowidItem
import os

class ExperiencesSpider(CrawlSpider):
    name = "experiences"
    allowed_domains = ["www.erowid.org"]
    start_urls = ['https://www.erowid.org/experiences/exp_list.shtml']

    rules = [Rule(LinkExtractor(allow =('subs/exp_[a-zA-Z]+.shtml')),callback='parse_item', follow = True)

    Rule(LinkExtractor(allow =('subs/exp_[a-zA-Z]+.shtml')), follow = True)    

    ]

    def request_experience(self, response):
        selectors = response.css('div')
        for selector in selectors:
            experience = ErowidItem()
            experience['Author'] = selector.xpath('//div[@class="author"]/a/text()').extract()
            experience['Title'] = selector.xpath('//div[@class="title"]/text()').extract()
            experience['Substance'] = selector.xpath('//div[@class="substance"]/text()').extract()
            experience['Text'] = selector.xpath("//div[@class = 'report-text-surround']/text()").extract()

            title = str(experience['Substance']) + " "+ str(experience['Title'])
            with open(os.path.join('drugs-%s' % (self.filename), title),"a") as fid:
                fid.write(str(experience) + "\n")

    def parse_item(self, response):
        self.filename = str(response.url)[44:-6]
        selectors = response.css('table')
        if not os.path.exists('drugs-%s' % (self.filename)): ##Make the file
            os.makedirs('drugs-%s' % (self.filename))
        list_of_experience = selectors.xpath('//table[@class="exp-cat-table"]/tr/td/a/@href').extract()

        for item in list_of_experience:
            request_url = str(item)
            Request(url="http://www.erowid.org" + request_url, callback = self.request_experience)