Python，Selenium和Scrapy不能一起工作吗？

Question

我想抓取英国航空公司的机票并把它储存在mongodb。 我可以通过搜索表单，但我不能抓取给定的数据。在

我的蜘蛛：

from scrapy import Spider
from scrapy.selector import Selector
from scrapy.http import FormRequest
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
import time
from flight.items import FlightItem

class BASpider(Spider):
name = "BA"
allowed_domains = ["britishairways.com"]
start_urls = [
    "http://www.britishairways.com/travel/home/public/en_za?DM1_Channel=PPC&DM1_Mkt=ZA&DM1_Campaign=APMEA_ZA_EN_PUREBRAND_MASTERBRAND&Brand=Y&gclid=CLvt24zsqMgCFUGg2wodds4Prw",
]

def __init__(self):
    self.driver = webdriver.Firefox()

def parse(self, response):
    self.driver.get(response.url)
    WebDriverWait(self.driver, 10).until(lambda s: s.find_element_by_xpath('//select[@id="depCountry"]').is_displayed())

    departCountry_form = self.driver.find_element_by_id('depCountry')
    departCity_form = self.driver.find_element_by_id('from')
    oneWay = self.driver.find_element_by_id('journeyTypeOW')
    oneWay.click()
    dest_form = self.driver.find_element_by_id('planTripFlightDestination')
    date_form = self.driver.find_element_by_id('depDate')
    butt = self.driver.find_element_by_class_name('button')
    departCountry_form.send_keys("South Africa")
    departCity_form.send_keys("Johannesburg")
    dest_form.send_keys("London")
    date_form.clear()
    date_form.send_keys("05/10/15")

    actions = ActionChains(self.driver)
    actions.click(butt)
    actions.perform()
    time.sleep(35)



def parse_post(self, response): 
    flightList = Selector(response).xpath('//table[@class="flightList directFlightsTable connectflights"]/tbody/tr')

    for flight in flightList:
        item = FlightItem()
        item['dTime'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" departure"]/div/div/span[1]/text()').extract()[0]
        item['aTime'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" arrival"]/span[1]/text()').extract()[0]
        item['flightNr'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" operator"]/div/div/span[2]/href').extract()[0]
        item['price_economy'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" priceselecter price-M ch3 col1"]/span/span[2]/label/text()').extract()[0]
        item['price_premium'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" priceselecter price-W ch3 col2"]/span/span[2]/label/text()').extract()[0]
        item['price_business'] = flight.xpath(
            'td[7]/table/tbody/tr/td[@class=" priceselecter price-C ch3 col3"]/span/span[2]/label/text()').extract()[0]
        yield item  

    self.driver.close() 

我没有出错，只是没有刮擦。在

Answer 1

您什么也得不到的原因是parse_post()方法从未被调用。在

---

实际上，您可能会直接在parse()回调中实例化Selector出的self.driver.page_source：

selector = Selector(text=self.driver.page_source)
flightList = selector.xpath('//table[@class="flightList directFlightsTable connectflights"]/tbody/tr')

for flight in flightList:
    # ...