如何限制在Selenium中通过标记名进行刮取

Question

我有以下代码从堆栈溢出中提取问题。我使用tagName定位器来抓取一个网站。然而，它把所有东西都刮到了同一个标签上，其中包括了不需要的东西。如何限制这一点

例如，如果我只想在h1标签中获取问题，那么只有scraper会在h1标签中使用stackoverflow网站名称来获取问题。如何让他们只在特定标签中刮取问题

这是我的代码，它可以清除标记中的所有内容。如何限制？：

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
import pandas as pd
import time
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import csv

def to_do():
# vars...
    csv_file_location = r"C:\Users\intel\Desktop\data_file.csv"

    user_agent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) ' \
                 'Chrome/80.0.3987.132 Safari/537.36'

    driver_exe = 'chromedriver'
    options = Options()
    options.add_argument("--headless")
    options.add_argument(f'user-agent={user_agent}')
    options.add_argument("--disable-web-security")
    options.add_argument("--allow-running-insecure-content")
    options.add_argument("--allow-cross-origin-auth-prompt")

    url = "https://stackoverflow.com/questions"

    driver = webdriver.Chrome(executable_path=r"C:\Users\intel\Downloads\setups\chromedriver.exe", options=options)
    driver.get(url)

    one_ = "A"

    two_ = "DIV"

    three_ = "A"

    try:
        element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, one_))
        )
        elements_1 = driver.find_elements_by_css_selector(one_)

        web_content_list = []
        for ele in elements_1:
            web_content_dict = {}
            web_content_dict["Title"] = ele.text
            web_content_list.append(web_content_dict)

        element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, two_))
        )
        elements_2 = driver.find_elements_by_css_selector(two_)

        for ele2 in elements_2:
            web_content_dict = {}
            web_content_dict["Title2"] = ele2.text
            web_content_list.append(web_content_dict)

        element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, three_))
        )

        elements_3 = driver.find_elements_by_css_selector(three_)

        for ele3 in elements_3:
            web_content_dict = {}
            web_content_dict["Title3"] = ele3.text
            web_content_list.append(web_content_dict)

        df = pd.DataFrame(web_content_list)
        new_df = pd.DataFrame({'Column 1': df['Title'].dropna(),
                  'Column 2': df['Title2'].dropna(),
                  'Column 3': df['Title3'].dropna()})
        new_df.to_csv(csv_file_location,
                  index=False, mode='a', encoding='utf-8')

        try:
            f = open(csv_file_location)
            print("Done !!!\n"*3)

        except IOError:
            print("File not accessible")

        finally:
            f.close()
        driver.quit()

    except:
        element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, one_))
        )
        elements_1 = driver.find_elements_by_css_selector(one_)

        element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, two_))
        )

        elements_2 = driver.find_elements_by_css_selector(two_)

        element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, three_))
        )

        elements_3 = driver.find_elements_by_css_selector(three_)

        df = pd.DataFrame({
            "Title1" : [ele for ele.text in elements_1],
            "Title2" : [ele2 for ele2.text in elements_2],
            "Title3" : [ele3 for ele3.text in elements_3],
        })
        df.to_csv(csv_file_location,
                  index=False, mode='w', encoding='utf-8')

        try:
            f = open(csv_file_location)
            print("Done !!!\n"*3)
            # Do something with the file
        except IOError:
            print("File not accessible")

        finally:
            f.close()
        driver.quit()

    finally:
        print("start")

if __name__ == "__main__":
    to_do()

任何帮助都将不胜感激

Answer 1

如果您只想获取问题下的a标记，可以使用此css选择器：

selector = '#questions a'

element = WebDriverWait(driver, 5).until(
            EC.presence_of_element_located((By.CSS_SELECTOR, selector))
        )

elements_1 = driver.find_elements_by_css_selector(selector)

这个css选择器将在下面找到'questions'的id和所有锚（'a'）标记，并且只使用这些标记

如果只需要问题而不需要其他锚定标记，请修改选择器以使用此选项：

selector = '#questions h3 a'