如何使用Selenium从devtools网络面板检索“Initiator”字段？

Question

我试图从一个网站中提取所有的URL网络请求，并在它们之间建立层次关系，即如果一个URL请求正在生成另一个请求。就像一连串的请求。在

如您所知，在Network面板中，Requests表中有一个名为“Initiator”的字段，它告诉您特定请求的来源或父级（如果有）。手动操作，我可以使用浏览器，转到开发工具中的网络面板，加载网站并下载生成的HAR文件。例如：

{
        "startedDateTime": "2019-11-05T17:38:46.775Z",
        "time": 15.676000155508518,
        "request": {
          "method": "POST",
          "url": "https://www.google.com/gen_204?oq=&gs_l=psy-ab.22...0.0..847450...0.0..0.0.0.......0......gws-wiz.",
          "httpVersion": "http/2.0",
          "headers": [
            {
              "name": ":path",
              "value": "/gen_204?oq=&gs_l=psy-ab.22...0.0..847450...0.0..0.0.0.......0......gws-wiz."
            },
            {
              "name": "sec-fetch-mode",
              "value": "no-cors"
            },
            {
              "name": "origin",
              "value": "https://www.google.com"
            },
            {
              "name": "accept-encoding",
              "value": "gzip, deflate, br"
            },
            {
              "name": "accept-language",
              "value": "en-GB,en;q=0.9,en-US;q=0.8,es-US;q=0.7,es;q=0.6"
            },
            {
              "name": "user-agent",
              "value": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Ubuntu Chromium/76.0.3809.100 Chrome/76.0.3809.100 Safari/537.36"
            },
            {
              "name": "content-type",
              "value": "text/plain;charset=UTF-8"
            },
            {
              "name": "accept",
              "value": "*/*"
            },
            {
              "name": "referer",
              "value": "https://www.google.com/"
            },
            {
              "name": ":authority",
              "value": "www.google.com"
            },
            {
              "name": "cookie",
              "value": "CONSENT=YES+GB.en+20160414-00-0; SEARCH_SAMESITE=CgQIg44B; ANID=AHWqTUlE3OPRfM5R1dtW0XvyIu2NOdLWoSHEgFemsFslXQTIzFKFCL-7kTtDZAr_; NID=190=Ezp7tXRaU_Rs2BS9RprlsS9QN9-PcwpYNSLwaOVGVFFp6pWepIjDqsYlgyLqb2eATn6HwUNs-SmgzAmtEm63fgX-YWVgbOyX7GU1esPamrN-GWXfwmXyrsqsTBOOQTzsHB3Q89tATDNQE_OKGd0YgCxMp9m9QXke2BJANdKdBYujl-g5tS8ZXcq0pw; 1P_JAR=2019-11-05-17; DV=o32RqCcqMlgsAJonGalrPPWlv0DK4xZ24gV5ztaaewMAAAA"
            },
            {
              "name": ":scheme",
              "value": "https"
            },
            {
              "name": "sec-fetch-site",
              "value": "same-origin"
            },
            {
              "name": "content-length",
              "value": "0"
            },
            {
              "name": ":method",
              "value": "POST"
            }
          ],
          "queryString": [
            {
              "name": "oq",
              "value": ""
            },
            {
              "name": "gs_l",
              "value": "psy-ab.22...0.0..847450...0.0..0.0.0.......0......gws-wiz."
            }
          ],
          "cookies": [
            {
              "name": "CONSENT",
              "value": "YES+GB.en+20160414-00-0",
              "expires": null,
              "httpOnly": false,
              "secure": false
            },
            {
              "name": "SEARCH_SAMESITE",
              "value": "CgQIg44B",
              "expires": null,
              "httpOnly": false,
              "secure": false
            },
            {
              "name": "ANID",
              "value": "AHWqTUlE3OPRfM5R1dtW0XvyIu2NOdLWoSHEgFemsFslXQTIzFKFCL-7kTtDZAr_",
              "expires": null,
              "httpOnly": false,
              "secure": false
            },
            {
              "name": "NID",
              "value": "190=Ezp7tXRaU_Rs2BS9RprlsS9QN9-PcwpYNSLwaOVGVFFp6pWepIjDqsYlgyLqb2eATn6HwUNs-SmgzAmtEm63fgX-YWVgbOyX7GU1esPamrN-GWXfwmXyrsqsTBOOQTzsHB3Q89tATDNQE_OKGd0YgCxMp9m9QXke2BJANdKdBYujl-g5tS8ZXcq0pw",
              "expires": null,
              "httpOnly": false,
              "secure": false
            },
            {
              "name": "1P_JAR",
              "value": "2019-11-05-17",
              "expires": null,
              "httpOnly": false,
              "secure": false
            },
            {
              "name": "DV",
              "value": "o32RqCcqMlgsAJonGalrPPWlv0DK4xZ24gV5ztaaewMAAAA",
              "expires": null,
              "httpOnly": false,
              "secure": false
            }
          ],
          "headersSize": -1,
          "bodySize": 0
        },
        "response": {
          "status": 204,
          "statusText": "",
          "httpVersion": "http/2.0",
          "headers": [
            {
              "name": "date",
              "value": "Tue, 05 Nov 2019 17:38:46 GMT"
            },
            {
              "name": "server",
              "value": "gws"
            },
            {
              "name": "x-frame-options",
              "value": "SAMEORIGIN"
            },
            {
              "name": "content-type",
              "value": "text/html; charset=UTF-8"
            },
            {
              "name": "status",
              "value": "204"
            },
            {
              "name": "alt-svc",
              "value": "quic=\":443\"; ma=2592000; v=\"46,43\",h3-Q050=\":443\"; ma=2592000,h3-Q049=\":443\"; ma=2592000,h3-Q048=\":443\"; ma=2592000,h3-Q046=\":443\"; ma=2592000,h3-Q043=\":443\"; ma=2592000"
            },
            {
              "name": "content-length",
              "value": "0"
            },
            {
              "name": "x-xss-protection",
              "value": "0"
            }
          ],
          "cookies": [],
          "content": {
            "size": 0,
            "mimeType": "text/html"
          },
          "redirectURL": "",
          "headersSize": -1,
          "bodySize": -1,
          "_transferSize": 54
        },
        "cache": {},
        "timings": {
          "blocked": 1.1320006029605865,
          "dns": -1,
          "ssl": -1,
          "connect": -1,
          "send": 0.16199999999999992,
          "wait": 14.122000366747379,
          "receive": 0.25999918580055237,
          "_blocked_queueing": 0.5990006029605865
        },
        "serverIPAddress": "216.58.204.68",
        "_initiator": {
          "type": "script",
          "stack": {
            "callFrames": [
              {
                "functionName": "s_1pb",
                "scriptId": "129",
                "url": "https://www.google.com/xjs/_/js/k=xjs.s.en_GB.UISl_YucLj8.O/ck=xjs.s.or8k_ixGu54.L.W.O/m=Fkg7bd,HcFEGb,IvlUe,MC8mtf,OF7gzc,RMhBfe,T4BAC,TJw5qb,TbaHGc,Y33vzc,cdos,hsm,iDPoPb,jsa,mvYTse,tg8oTe,uz938c,vWNDde,ws9Tlc,yQ43ff,d,csi/am=BAAAsAjYuwOC_L8VAAQAfAYAAAFuwQYLhCGhYqwOEAE/d=1/dg=2/br=1/ct=zgms/rs=ACT90oGdwE1ooFdbHyz-Vk2BhYjwAv-QDQ",
                "lineNumber": 2323,
                "columnNumber": 376
              },

在本例中，URL https://www.google.com/gen_204?oq=&gs_l=psy-ab.22...由URL https://www.google.com/xjs/_/js/k=xjs.s.en_GB.UISl...启动。您可以在键“Initiator”->；“callFrames”->；“url”中看到此信息。在

我们的想法是获取这些信息（调用其他URL的URL）或使用Selenium自动下载HAR文件。我试过了：

1。Selenium+browsermob代理 问题：生成的HAR文件没有“Initiator”字段，无法连接启动器请求及其依赖项。在

2。Selenium性能日志 我使用以下代码从Selenium获取性能日志：

^{pr2}$

在这里，唯一可以找到“initiator”字段的方法是Network.requestWillBeSent“方法，该方法有一个URL，但与另一个URL无关。似乎每个“消息”字段都是独立的，而且同样，无法连接发起方请求及其依赖项。在

1. driver.execute_脚本 我在这个问题中看到了这个代码：How to access Network panel on google chrome developer tools with selenium?

driver = webdriver.Chrome('/path/to/chromedriver)
driver.get('https://www.google.com');
log = driver.execute_script("return window.performance.getEntries();")
#ANOTHER WAY
#log = driver.execute_script("var performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; var network = performance.getEntries() || {}; return network;")

生成的日志与HAR文件或性能日志完全不同，它没有任何可用于关联请求的信息。示例：

{
    "startTime": 0,
    "initiatorType": "navigation",
    "unloadEventStart": 0,
    "fetchStart": 69.29999962449074,
    "duration": 1311.3000001758337,
    "responseStart": 172.89999965578318,
    "nextHopProtocol": "h2",
    "transferSize": 68052,
    "connectStart": 70.19999995827675,
    "domainLookupStart": 70.19999995827675,
    "redirectStart": 0,
    "domContentLoadedEventEnd": 504.90000005811453,
    "responseEnd": 190.7999999821186,
    "requestStart": 100.49999970942736,
    "type": "navigate",
    "secureConnectionStart": 80.40000032633543,
    "connectEnd": 99.99999962747097,
    "redirectCount": 0,
    "workerStart": 0,
    "decodedBodySize": 233300,
    "loadEventStart": 1304.8000000417233,
    "encodedBodySize": 67329,
    "serverTiming": [],
    "entryType": "navigation",
    "domInteractive": 487.699999473989,
    "domContentLoadedEventStart": 487.80000023543835,
    "redirectEnd": 0,
    "name": "https://www.google.com/?gws_rd=ssl",
    "domainLookupEnd": 70.19999995827675,
    "unloadEventEnd": 0,
    "loadEventEnd": 1311.3000001758337,
    "domComplete": 1303.3999996259809,
    "toJSON": {}
  },
  {
    "initiatorType": "img",
    "fetchStart": 298.20000007748604,
    "duration": 14.100000262260437,
    "responseStart": 310.70000026375055,
    "responseEnd": 312.3000003397465,
    "transferSize": 6146,
    "connectStart": 298.20000007748604,
    "domainLookupStart": 298.20000007748604,
    "redirectStart": 0,
    "toJSON": {},
    "requestStart": 299.80000015348196,
    "secureConnectionStart": 0,
    "connectEnd": 298.20000007748604,
    "workerStart": 0,
    "decodedBodySize": 5969,
    "startTime": 298.20000007748604,
    "encodedBodySize": 5969,
    "serverTiming": [],
    "entryType": "resource",
    "redirectEnd": 0,
    "name": "https://www.google.com/images/branding/googlelogo/1x/googlelogo_color_272x92dp.png",
    "domainLookupEnd": 298.20000007748604,
    "nextHopProtocol": "h2"
  },

因此，我认为从developer tools Network面板下载的HAR文件是唯一允许在URL请求之间建立关系的信息源。因为我正在抓取许多网站，我需要自动化的过程，但我找不到一个方法来做到这一点。在

任何关于下载HAR文件的想法，我需要直接使用Selenium提取我需要的信息。在