Mitmproxy与Webdriver的奇怪问题

Question

这是我代码的简化版本，展示了webdriver和mitmproxy之间的问题：

from libmproxy import controller
from libmproxy import proxy
import os
from multiprocessing import Process
from selenium import webdriver
from selenium.webdriver.common.proxy import *
import time


# traffic inception
class HttpMitmReporter(controller.Master):
    def __init__(self, proxy_server):
        controller.Master.__init__(self, proxy_server)

    def run(self):
        try:
            return controller.Master.run(self)
        except KeyboardInterrupt:
            self.shutdown()


# setup browser with proxy settings etc.
def setup_browser():
    global driver
    proper_setup = False

    proxy_con = "127.0.0.1:8088"
    my_proxy = Proxy({'proxyType': ProxyType.MANUAL,
                      'httpProxy': proxy_con,
                      'ftpProxy': proxy_con,
                      'sslProxy': proxy_con,
                      'noProxy': ''})

    while not proper_setup:
        try:
            driver = webdriver.Firefox(proxy=my_proxy)
            driver.set_page_load_timeout(20)
            driver.set_script_timeout(20)
            proper_setup = True
        except Exception as exception:
            print exception
            try:
                driver.quit()
            except Exception:
                time.sleep(3)


driver = None

# start proxy
config = proxy.ProxyConfig(
    cacert=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
server = proxy.ProxyServer(config, 8088)
reporter = HttpMitmReporter(server)
mitm_proxy = Process(target=reporter.run)
mitm_proxy.start()

setup_browser()

# urls to test
urls = [
        'http://support.microsoft.com/common/survey.aspx?showpage=1&scid=sw%3Bde%3B3663&altStyle=narrow&renderOption=OverrideDefault&fr=1',
        'http://outlook.com/',
        ]

# visit sites
for url in urls:
    print url
    try:
        driver.get(url)
        elements = driver.find_elements_by_tag_name('a')
    except Exception as ex:
        print ex
        driver.quit()
        setup_browser()

# terminate browser and proxy
driver.quit()
mitm_proxy.terminate()

如果你运行这个代码，你会发现Firefox一直在加载outlook.com，而且超时设置从来没有被触发。
这个问题只在使用mitmproxy时出现。我测试了其他代理，它们在这些网址上可以正常工作，但我不能使用它们，因为要么无法检查SSL流量，要么只是提供原始数据输出。不过我发现mitmproxy应该是最好的解决方案，只要它能和webdriver正常配合使用。
此外，我还在命令行中启动mitmproxy进行测试——结果是一样的问题。
还有，我用Java的webdriver进行测试——结果也是一样。
而且这个问题有时也会出现在其他网址，比如fc2.com。

这是我配置的简单概述：
- Linux Mint Petra x64
- python 2.7
- Mitmproxy 0.10.1
- Selenium 2.41.0
- Firefox 28

有没有人知道怎么解决这个问题？
为什么这个问题只在mitmproxy上出现？
另外，我也想了解一些可以用来检查http和https流量的替代方案，配合selenium使用。

与FlowMaster的更新：

from libmproxy import flow
from libmproxy import proxy
import os
from multiprocessing import Process
from selenium import webdriver
from selenium.webdriver.common.proxy import *
import time


# traffic inception
class HttpMitmReporter(flow.FlowMaster):

    def run(self):
        try:
            flow.FlowMaster.run(self)
        except KeyboardInterrupt:
            self.shutdown()

    def handle_request(self, r):
        f = flow.FlowMaster.handle_request(self, r)
        if f:
            r.reply()
        return f

    def handle_response(self, r):
        f = flow.FlowMaster.handle_response(self, r)
        if f:
            r.reply()
        return f


# setup browser with proxy settings etc.
def setup_browser():
    global driver
    proper_setup = False

    proxy_con = "127.0.0.1:8088"
    my_proxy = Proxy({'proxyType': ProxyType.MANUAL,
                      'httpProxy': proxy_con,
                      'ftpProxy': proxy_con,
                      'sslProxy': proxy_con,
                      'noProxy': ''})

    while not proper_setup:
        try:
            driver = webdriver.Firefox(proxy=my_proxy)
            driver.set_page_load_timeout(20)
            driver.set_script_timeout(20)
            proper_setup = True
        except Exception as exception:
            print exception
            try:
                driver.quit()
            except Exception:
                time.sleep(3)


driver = None

# start proxy
config = proxy.ProxyConfig(
    cacert=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem")
)
state = flow.State()
server = proxy.ProxyServer(config, 8088)
reporter = HttpMitmReporter(server, state)
mitm_proxy = Process(target=reporter.run)
mitm_proxy.start()

setup_browser()

# urls to test
urls = [
        'http://support.microsoft.com/common/survey.aspx?showpage=1&scid=sw%3Bde%3B3663&altStyle=narrow&renderOption=OverrideDefault&fr=1',
        'http://outlook.com/',
        ]

# visit sites
for url in urls:
    print url
    try:
        driver.get(url)
        elements = driver.find_elements_by_tag_name('a')
    except Exception as ex:
        print ex
        driver.quit()
        setup_browser()

# terminate browser and proxy
driver.quit()
mitm_proxy.terminate()

Linux http https firefox ssl webdriver selenium mitmproxy

Mitmproxy与Webdriver的奇怪问题

1 个回答

撰写回答