如何从googleplay网站的无限卷轴中删除所有链接

Question

我正试图通过无限滚动来刮取playstore中的链接。使用Selenium和BeautifulSoup，我只能获取第一页上的链接。如何继续前进，以获得所有链接的完整列表。这是我到目前为止所拥有的

from selenium import webdriver
from bs4 import BeautifulSoup
from  requests import Session
import re

urls = [
    'https://play.google.com/store/apps/collection/cluster?clp=0g4eChwKFnRvcHNlbGxpbmdfZnJlZV9EQVRJTkcQBxgD:S:ANO1ljJ0qxs&gsr=CiHSDh4KHAoWdG9wc2VsbGluZ19mcmVlX0RBVElORxAHGAM%3D:S:ANO1ljJvMRw&hl=en&gl=US'
]

def main():
    driver = webdriver.Chrome()
    for url in urls:
        driver.get(url)
        content = driver.page_source.encode('utf-8').strip()
        headers = {
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
        page = requests.get(url, headers=headers).content
        bs = BeautifulSoup(page, "html.parser")
        temp_link = []

        first_p = bs.findAll('a', {'class':'poRVub'}) 
        for link in first_p:
            temp_link.append('https://play.google.com'+link['href'])
        

        more_link = bs.findAll('div', {'class':'uMConb  V2Vq5e POHYmb-eyJpod YEDFMc-eyJpod y1APZe-eyJpod drrice'})
        print(more_link)

Answer 1

您需要实现一种滚动方式，直到google为您呈现所需的所有项目

def infinite_scroll():
    spinner_loading_class = 'xu46lf'
    while True:
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
        try:
            wait.until(EC.presence_of_element_located((By.CLASS_NAME, spinner_loading_class)))
            wait.until(EC.invisibility_of_element_located((By.CLASS_NAME, spinner_loading_class)))
        except TimeoutException:
            # Scrolling is done, all items were loaded
            return None

然后在加载url后调用此方法

for url in urls:
    driver.get(url)
    infinite_scroll()
    content = driver.page_source
    bs = BeautifulSoup(content, "html.parser")
    temp_link = []

    items = bs.findAll('a', {'class': 'poRVub'})
    for link in items:
        temp_link.append('https://play.google.com' + link['href'])

    # more_link = bs.findAll('a', {'class': 'poRVub'})
    print(temp_link)
    print(len(temp_link))

注意您将需要以下导入：

from selenium.webdriver.support.wait import WebDriverWait, TimeoutException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By

您还需要在驱动程序初始化后创建wait对象：

driver = webdriver.Chrome()
wait = WebDriverWait(driver, 5)

输出

['https://play.google.com/store/apps/details?id=co.hinge.app', 'https://play.google.com/store/apps/details?id=com.udates', 'https://play.google.com/store/apps/details?id=com.hily.app', 'https://play.google.com/store/apps/details?id=com.match.android.matchmobile', 'https://play.google.com/store/apps/details?id=com.okcupid.okcupid', 'https://play.google.com/store/apps/details?id=com.zoosk.zoosk', 'https://play.google.com/store/apps/details?id=com.affinityapps.blk', 'https://play.google.com/store/apps/details?id=com.curvydating', 'https://play.google.com/store/apps/details?id=com.affinityapps.chispa', 'https://play.google.com/store/apps/details?id=com.lovinga', 'https://play.google.com/store/apps/details?id=com.appspartner.iflirts', 'https://play.google.com/store/apps/details?id=com.app.rondevo', 'https://play.google.com/store/apps/details?id=com.infostream.seekingarrangement', 'https://play.google.com/store/apps/details?id=com.affinityapps.twozerofour', 'https://play.google.com/store/apps/details?id=com.getpure.pure', 'https://play.google.com/store/apps/details?id=com.eharmony', 'https://play.google.com/store/apps/details?id=com.mason.wooplus', 'https://play.google.com/store/apps/details?id=co.clover.clover', 'https://play.google.com/store/apps/details?id=com.taptodate', 'https://play.google.com/store/apps/details?id=com.jaumo.casual', 'https://play.google.com/store/apps/details?id=com.meetville.dating', 'https://play.google.com/store/apps/details?id=mingle.android.mingle2', 'https://play.google.com/store/apps/details?id=com.peoplemedia.ourtime', 'https://play.google.com/store/apps/details?id=com.improverllc.meetly', 'https://play.google.com/store/apps/details?id=com.idealmatch.idma', 'https://play.google.com/store/apps/details?id=com.fwbhookup.xpal', 'https://play.google.com/store/apps/details?id=com.socialnetwork.hookupsapp', 'https://play.google.com/store/apps/details?id=com.cougar.sugarmomma.hookupapps', 'https://play.google.com/store/apps/details?id=com.bi.bicupid', 'https://play.google.com/store/apps/details?id=com.InfoTalkGroup.OnIyFansApplication', 'https://play.google.com/store/apps/details?id=com.evermatch', 'https://play.google.com/store/apps/details?id=ru.fotostrana.sweetmeet', 'https://play.google.com/store/apps/details?id=singles420.entrision.com.singles420', 'https://play.google.com/store/apps/details?id=com.ashleymadison.mobile', 'https://play.google.com/store/apps/details?id=com.ace.android', 'https://play.google.com/store/apps/details?id=com.covalent.kippo', 'https://play.google.com/store/apps/details?id=com.catch.fwbhookupapp', 'https://play.google.com/store/apps/details?id=com.social.onenight', 'https://play.google.com/store/apps/details?id=com.antiland', 'https://play.google.com/store/apps/details?id=com.threesome.thirder', 'https://play.google.com/store/apps/details?id=com.cougardating.cougard', 'https://play.google.com/store/apps/details?id=com.yummytechnologies.OnIyFansApplicationtips', 'https://play.google.com/store/apps/details?id=com.amolatina', 'https://play.google.com/store/apps/details?id=dating.app.chat.flirt.wgbcv', 'https://play.google.com/store/apps/details?id=com.nibel.mocee', 'https://play.google.com/store/apps/details?id=com.strangers.chat_developers.strangerschatapp', 'https://play.google.com/store/apps/details?id=com.coffeemeetsbagel', 'https://play.google.com/store/apps/details?id=com.dartushinc.videocall', 'https://play.google.com/store/apps/details?id=com.dating.android', 'https://play.google.com/store/apps/details?id=com.cougardating.olderwomendating', 'https://play.google.com/store/apps/details?id=com.peoplemedia.blackpeoplemeet', 'https://play.google.com/store/apps/details?id=com.rvideocall.videocalladwise', 'https://play.google.com/store/apps/details?id=find.love.tonight.schat', 'https://play.google.com/store/apps/details?id=one.night.casual.secret.adult.singles.dating.meet.match.chat.free.video.hook.up', 'https://play.google.com/store/apps/details?id=com.asiandate', 'https://play.google.com/store/apps/details?id=com.discreet.adultdate', 'https://play.google.com/store/apps/details?id=co.feeld', 'https://play.google.com/store/apps/details?id=com.sdcmedia.swingers.g3', 'https://play.google.com/store/apps/details?id=com.spark.com.silversingles.app', 'https://play.google.com/store/apps/details?id=live.chat.free.dvghn', 'https://play.google.com/store/apps/details?id=com.fetlife.fetish.hookupapps', 'https://play.google.com/store/apps/details?id=com.ihappydate', 'https://play.google.com/store/apps/details?id=com.socialnetwork.sdmapp', 'https://play.google.com/store/apps/details?id=com.honeo.joefe', 'https://play.google.com/store/apps/details?id=com.random.chat.app', 'https://play.google.com/store/apps/details?id=com.chat.plus.app1009949', 'https://play.google.com/store/apps/details?id=com.connectllc.ship', 'https://play.google.com/store/apps/details?id=com.dating.suger', 'https://play.google.com/store/apps/details?id=hookup.sugarmomma.hookupapps', 'https://play.google.com/store/apps/details?id=com.freeappvideochat.sexocam', 'https://play.google.com/store/apps/details?id=uk.co.hilmy.wild', 'https://play.google.com/store/apps/details?id=kr.jungrammer.randomchat', 'https://play.google.com/store/apps/details?id=ru.loveplanet.app', 'https://play.google.com/store/apps/details?id=com.league.theleague', 'https://play.google.com/store/apps/details?id=com.youloveme.chat.qas', 'https://play.google.com/store/apps/details?id=com.lifestyle.kinky', 'https://play.google.com/store/apps/details?id=app.threesome.dating', 'https://play.google.com/store/apps/details?id=com.adore.android', 'https://play.google.com/store/apps/details?id=chat.roulette', 'https://play.google.com/store/apps/details?id=com.datemyage', 'https://play.google.com/store/apps/details?id=com.sugardaddydating.sugardaddymeet', 'https://play.google.com/store/apps/details?id=com.millionairedating.millionairematch', 'https://play.google.com/store/apps/details?id=cdff.mobileapp', 'https://play.google.com/store/apps/details?id=net.flirtyhookup.hookup', 'https://play.google.com/store/apps/details?id=datingsandmeetings.eddsa', 'https://play.google.com/store/apps/details?id=com.bbw.hookup.farmers', 'https://play.google.com/store/apps/details?id=com.muzmatch.muzmatchapp', 'https://play.google.com/store/apps/details?id=com.lizinc.adultfriend', 'https://play.google.com/store/apps/details?id=com.sudy.cougar', 'https://play.google.com/store/apps/details?id=com.hotornot.app', 'https://play.google.com/store/apps/details?id=de.affinitas.za.co.elitesingles.and', 'https://play.google.com/store/apps/details?id=com.sunlightmobile.love4youchat', 'https://play.google.com/store/apps/details?id=com.wildec.fastmeet', 'https://play.google.com/store/apps/details?id=com.naughtyvideochat.appg', 'https://play.google.com/store/apps/details?id=com.mequeres', 'https://play.google.com/store/apps/details?id=com.threesome.swingers.app.threefun', 'https://play.google.com/store/apps/details?id=girlsaround.flirtdating', 'https://play.google.com/store/apps/details?id=traveldating.findtrippartners', 'https://play.google.com/store/apps/details?id=ru.mobstudio.andgalaxy', 'https://play.google.com/store/apps/details?id=com.fwbdate.nsadating', 'https://play.google.com/store/apps/details?id=com.localhookupdating.android', 'https://play.google.com/store/apps/details?id=meet.new.people.make.friends.free.dating.video.chat', 'https://play.google.com/store/apps/details?id=ru.taboo.app', 'https://play.google.com/store/apps/details?id=com.trandrapp.trandr', 'https://play.google.com/store/apps/details?id=dotsoa.anonymous.chat', 'https://play.google.com/store/apps/details?id=com.maybeyou', 'https://play.google.com/store/apps/details?id=com.aswile.trooma', 'https://play.google.com/store/apps/details?id=com.flirtini', 'https://play.google.com/store/apps/details?id=com.ArianaDate.StrangerMeetings', 'https://play.google.com/store/apps/details?id=chat.anti', 'https://play.google.com/store/apps/details?id=com.onlinebuddies.manhuntgaychat', 'https://play.google.com/store/apps/details?id=com.shinyoctopus.friendi', 'https://play.google.com/store/apps/details?id=com.wildec.dating.meet4u', 'https://play.google.com/store/apps/details?id=com.bdsm.kinky.hookup', 'https://play.google.com/store/apps/details?id=com.farmersonly.android', 'https://play.google.com/store/apps/details?id=kr.jungrammer.secretchat', 'https://play.google.com/store/apps/details?id=love.waiter.android', 'https://play.google.com/store/apps/details?id=com.arabChat.dating', 'https://play.google.com/store/apps/details?id=de.yalwa.hashtagdating', 'https://play.google.com/store/apps/details?id=gayfriendly.gay.dating.app', 'https://play.google.com/store/apps/details?id=com.cougar.sugarmomma.sugarmummy', 'https://play.google.com/store/apps/details?id=com.td.transdr', 'https://play.google.com/store/apps/details?id=com.wildec.meet24', 'https://play.google.com/store/apps/details?id=com.casualx.hookup', 'https://play.google.com/store/apps/details?id=meetsingnear.android', 'https://play.google.com/store/apps/details?id=com.arabic_chat.app958899', 'https://play.google.com/store/apps/details?id=com.snatchable.snatchable', 'https://play.google.com/store/apps/details?id=dating.inmessage.net', 'https://play.google.com/store/apps/details?id=kink.bdsm.dating', 'https://play.google.com/store/apps/details?id=ffkxn.easydate.enjoy', 'https://play.google.com/store/apps/details?id=com.besget.gaper', 'https://play.google.com/store/apps/details?id=com.boranuonline.idates', 'https://play.google.com/store/apps/details?id=devansoft.randomchat', 'https://play.google.com/store/apps/details?id=de.appfiction.yocutiegoogle', 'https://play.google.com/store/apps/details?id=com.jaumo.mature', 'https://play.google.com/store/apps/details?id=com.lestaryp', 'https://play.google.com/store/apps/details?id=com.free.hookup.dating.apps.wild', 'https://play.google.com/store/apps/details?id=com.wqioq.asngqw', 'https://play.google.com/store/apps/details?id=com.blendr.mobile', 'https://play.google.com/store/apps/details?id=co.dilmil.android', 'https://play.google.com/store/apps/details?id=com.cupidmedia.wrapper.latinamericancupid', 'https://play.google.com/store/apps/details?id=chat.divorcedatingandmatch', 'https://play.google.com/store/apps/details?id=com.stddating.positivesingles', 'https://play.google.com/store/apps/details?id=com.threesomechat.polydating', 'https://play.google.com/store/apps/details?id=com.blackwhitemeet', 'https://play.google.com/store/apps/details?id=com.catholicsingles2', 'https://play.google.com/store/apps/details?id=com.dating.datingappcheat', 'https://play.google.com/store/apps/details?id=com.kinky.fetlifestyle', 'https://play.google.com/store/apps/details?id=com.yourchristiandate', 'https://play.google.com/store/apps/details?id=com.tabooapp.dating', 'https://play.google.com/store/apps/details?id=com.one.goodnight', 'https://play.google.com/store/apps/details?id=com.cupidmedia.wrapper.afrointroductions', 'https://play.google.com/store/apps/details?id=com.jaumo.prime', 'https://play.google.com/store/apps/details?id=datetoday.girls.meetlove', 'https://play.google.com/store/apps/details?id=com.deccanappz.livechat.indianchat', 'https://play.google.com/store/apps/details?id=com.aflami.app905537', 'https://play.google.com/store/apps/details?id=fiori.transgender', 'https://play.google.com/store/apps/details?id=com.socialnetworking.bbwmapp', 'https://play.google.com/store/apps/details?id=com.gravida.viverra', 'https://play.google.com/store/apps/details?id=com.randomsexo.camchatapp', 'https://play.google.com/store/apps/details?id=secret.benefit', 'https://play.google.com/store/apps/details?id=com.over.seniors', 'https://play.google.com/store/apps/details?id=com.momsforflirt.meet.real.women', 'https://play.google.com/store/apps/details?id=com.planetromeo.android.app', 'https://play.google.com/store/apps/details?id=com.allien.ufo.addon.invasion.mod', 'https://play.google.com/store/apps/details?id=chat.gbwastapp', 'https://play.google.com/store/apps/details?id=com.cupidmedia.wrapper.interracialcupid', 'https://play.google.com/store/apps/details?id=com.truckersucker2.app', 'https://play.google.com/store/apps/details?id=girls.chatmet', 'https://play.google.com/store/apps/details?id=com.choiceoflove.dating', 'https://play.google.com/store/apps/details?id=com.mutualapp', 'https://play.google.com/store/apps/details?id=com.bbw.Hookup.hookupapps', 'https://play.google.com/store/apps/details?id=com.onesion.emoki', 'https://play.google.com/store/apps/details?id=us.Only4Adults.HotGirlCalling', 'https://play.google.com/store/apps/details?id=com.angel.soulmatif', 'https://play.google.com/store/apps/details?id=real.sexymobilenumberforwhatsappchatcall', 'https://play.google.com/store/apps/details?id=datingorhookup.theunicorn.threesomechat', 'https://play.google.com/store/apps/details?id=com.secretarrangement.sugardating', 'https://play.google.com/store/apps/details?id=com.msd.community', 'https://play.google.com/store/apps/details?id=com.sugardaddy.date', 'https://play.google.com/store/apps/details?id=com.threesomedating.lgbt', 'https://play.google.com/store/apps/details?id=com.bi.bothx', 'https://play.google.com/store/apps/details?id=com.transgender.datingapp', 'https://play.google.com/store/apps/details?id=com.dating.meetby', 'https://play.google.com/store/apps/details?id=com.olosho.olosho', 'https://play.google.com/store/apps/details?id=com.datingapp.bbwmatch', 'https://play.google.com/store/apps/details?id=kr.jungrammer.honeytalk', 'https://play.google.com/store/apps/details?id=com.ts.transer', 'https://play.google.com/store/apps/details?id=us.GirlsLocating.HotGallery1', 'https://play.google.com/store/apps/details?id=com.arab.chat.app1015872', 'https://play.google.com/store/apps/details?id=com.chat.meet.dating', 'https://play.google.com/store/apps/details?id=com.sugardaddy.sugardaddy.sugarbaby', 'https://play.google.com/store/apps/details?id=com.fwbdating.nsameet', 'https://play.google.com/store/apps/details?id=site.olderwomandating.android5ecb361099ca7', 'https://play.google.com/store/apps/details?id=ru.chocoapp.app', 'https://play.google.com/store/apps/details?id=com.fetish.hookupapps.bdsm', 'https://play.google.com/store/apps/details?id=com.dating.threefan', 'https://play.google.com/store/apps/details?id=com.deccanappz.livechat.bhabhi', 'https://play.google.com/store/apps/details?id=com.LonelyGirls.LonelyGirls', 'https://play.google.com/store/apps/details?id=com.buzzarab.buzzarab']
200