class SomewebsiteProductSpider(scrapy.Spider):
name = "somewebsite"
allowed_domains = ["somewebsite.com"]
start_urls = [
]
def parse(self, response):
items = somewebsiteItem()
title = response.xpath('//h1[@id="title"]/span/text()').extract()
sale_price = response.xpath('//span[contains(@id,"ourprice") or contains(@id,"saleprice")]/text()').extract()
category = response.xpath('//a[@class="a-link-normal a-color-tertiary"]/text()').extract()
availability = response.xpath('//div[@id="availability"]//text()').extract()
items['product_name'] = ''.join(title).strip()
items['product_sale_price'] = ''.join(sale_price).strip()
items['product_category'] = ','.join(map(lambda x: x.strip(), category)).strip()
items['product_availability'] = ''.join(availability).strip()
fo = open("C:\\Users\\user1\PycharmProjects\\test.txt", "w")
fo.write("%s \n%s \n%s" % (items['product_name'], items['product_sale_price'], self.start_urls))
fo.close()
print(items)
yield items
在测试.py在
^{pr2}$如何将动态起始URL列表传递给“SomewebsiteProductSpiders”对象测试.py在启动爬行过程之前?任何帮助都将不胜感激。 谢谢您。在
process.crawl
接受传递给spider构造函数的可选参数,因此您可以从spider的__init__
填充{start_requests
过程。例如测试.py
某蜘蛛.py
^{pr2}$相关问题 更多 >
编程相关推荐