在字符串中出现findall问题（预期为字符串或类字节对象）

2024-06-02 07:30:53 发布

您现在位置：Python中文网/ 问答频道 /正文
1867 3

上一个问题我做了关于这个主题被搁置作为离题，做了修改，但它仍然搁置，不知道为什么。我问的是：

我目前正在使用amazon scraper，我需要从产品中获取图像，例如：

https://www.amazon.com/gp/product/B0711BMXVB?pf_rd_p=1581d9f4-062f-453c-b69e-0f3e00ba2652&pf_rd_r=X7FDBW1DN25C8PM5A01C

我所做的是，使用xpath

//script[contains(., "ImageBlockATF")]/text()

获取一堆文本，其中包含“大型”图像的所有URL

基本上是这样的：

P.when('A').register("ImageBlockATF", function(A){
var data = {
            'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX395_.jpg":[282,395],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX500_.jpg":[357,500],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX535_.jpg":[382,535],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX575_.jpg":[410,575],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX625_.jpg":[446,625],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX675_.jpg":[481,675],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX695_.jpg":[496,695]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY395_.jpg":[395,249],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY500_.jpg":[500,316],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY535_.jpg":[535,338],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY575_.jpg":[575,363],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY625_.jpg":[625,395],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY675_.jpg":[675,426],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY695_.jpg":[695,439]},"variant":"FRNT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY395_.jpg":[395,301],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY500_.jpg":[500,381],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY535_.jpg":[535,408],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY575_.jpg":[575,438],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY625_.jpg":[625,477],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY675_.jpg":[675,515],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY695_.jpg":[695,530]},"variant":"BACK","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX395_.jpg":[146,395],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX500_.jpg":[185,500],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX535_.jpg":[198,535],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX575_.jpg":[213,575],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX625_.jpg":[231,625],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX675_.jpg":[250,675],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX695_.jpg":[257,695]},"variant":"BOTT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX395_.jpg":[139,395],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX500_.jpg":[177,500],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX535_.jpg":[189,535],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX575_.jpg":[203,575],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX625_.jpg":[221,625],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX675_.jpg":[238,675],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX695_.jpg":[245,695]},"variant":"TOPP","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX395_.jpg":[186,395],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX500_.jpg":[235,500],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX535_.jpg":[252,535],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX575_.jpg":[271,575],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX625_.jpg":[294,625],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX675_.jpg":[318,675],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX695_.jpg":[327,695]},"variant":"RGHT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX395_.jpg":[255,395],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX500_.jpg":[322,500],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX535_.jpg":[345,535],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX575_.jpg":[371,575],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX625_.jpg":[403,625],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX675_.jpg":[435,675],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX695_.jpg":[448,695]},"variant":"PAIR","lowRes":null}]},
            'colorToAsin': {'initial': {}},
            'holderRatio': 1.2,
            'holderMaxHeight': 700,
            'heroImage': {'initial': []},
            'heroVideo': {'initial': []},
            'spin360ColorData': {'initial': {}},
            'spin360ColorEnabled': {'initial': 0},
            'spin360ConfigEnabled': false,
            'spin360LazyLoadEnabled': false,
            'playVideoInImmersiveView':'false',
            'tabbedImmersiveViewTreatment':'C',
            'totalVideoCount':'0',
            'videoIngressATFSlateThumbURL':'',
            'mediaTypeCount':'0',
            'atfEnhancedHoverOverlay' : true,
            'winningAsin': 'B072596K2C',
            'weblabs' : {},
            'aibExp3Layout' : 1,
            'aibRuleName' : 'frank-powered',
            'acEnabled' : false
            };
A.trigger('P.AboveTheFold'); // trigger ATF event.
return data;
});

我把它命名为imagesString。你知道吗

我用scrapy得到了这根线：

imagesString = (response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first())

在示例URL中搜索前一个xpath时，会弹出2个文本块。使用extract_first()可以提取第一个字符串，即上面的长字符串。你知道吗

然后我必须获得网址，所以我这样做：

images = re.findall('\"large\":\"(https.*?\.jpg)\"', imagesString)

它给了我一个所有大图片的列表。你知道吗

我遇到的问题是，在程序运行的某个点上，我得到以下错误：

    Traceback (most recent call last):
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 30, in process_spider_output
for x in result:
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
      File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
      File "C:\Users\Manuel\Desktop\scrapyProject\genericScraper\genericScraper\spiders\finalClothes_spider.py", line 52, in parse
    imagenes = re.findall('\"large\":\"(https.*?\.jpg)\"', imagenesString)
     File "C:\Users\Manuel\Anaconda3\lib\re.py", line 223, in findall
        return _compile(pattern, flags).findall(string)
    TypeError: expected string or bytes-like object

我真的不知道发生了什么。我能看到的是，这个错误从来不会在进程开始时发生。如果我必须这样做30个产品，它的工作很好，但当我开始得到更多的产品，这种情况发生。你知道吗

通过使用Json方法和@mauricemayer帮助

s = response.xpath('//script[contains(., "ImageBlockATF")]/text()').extract_first()
m = re.search(r'^var data = ({.*};)', s, re.S | re.M)
data = m.groups()[0]
jsonObj = json.loads(data[:-1].replace("'", '"'))

我得到了这个错误

File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\utils\defer.py", line 102, in iter_errback
yield next(it)
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\offsite.py", line 30, in process_spider_output
for x in result:
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\referer.py", line 339, in <genexpr>
return (_set_referer(r) for r in result or ())
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\urllength.py", line 37, in <genexpr>
return (r for r in result or () if _filter(r))
  File "C:\Users\Manuel\Anaconda3\lib\site-packages\scrapy\spidermiddlewares\depth.py", line 58, in <genexpr>
return (r for r in result or () if _filter(r))
  File     "C:\Users\Manuel\Desktop\scrapyProject\genericScraper\genericScraper\spiders\finalClothes_spider.py", line 59, in parse
    data = m.groups()[0]
AttributeError: 'NoneType' object has no attribute 'groups'

编辑：添加了用户建议和新错误

EDIT2:添加了Json标记


Tags： inpyhttpscomsslamazonlineusers
1条回答
网友
1楼 · 发布于 2024-06-02 07:30:53

javascript变量data是一个JSON对象，它可能更容易处理，您可以在对象上快速迭代：

import json
import re


s = """P.when('A').register("ImageBlockATF", function(A){
var data = {
            'colorImages': { 'initial': [{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX395_.jpg":[282,395],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX500_.jpg":[357,500],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX535_.jpg":[382,535],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX575_.jpg":[410,575],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX625_.jpg":[446,625],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX675_.jpg":[481,675],"https://images-na.ssl-images-amazon.com/images/I/81Qs-sOznzL._UX695_.jpg":[496,695]},"variant":"MAIN","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY395_.jpg":[395,249],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY500_.jpg":[500,316],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY535_.jpg":[535,338],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY575_.jpg":[575,363],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY625_.jpg":[625,395],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY675_.jpg":[675,426],"https://images-na.ssl-images-amazon.com/images/I/71ZLo7ef-GL._UY695_.jpg":[695,439]},"variant":"FRNT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY395_.jpg":[395,301],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY500_.jpg":[500,381],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY535_.jpg":[535,408],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY575_.jpg":[575,438],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY625_.jpg":[625,477],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY675_.jpg":[675,515],"https://images-na.ssl-images-amazon.com/images/I/71Fny8%2BI-mL._UY695_.jpg":[695,530]},"variant":"BACK","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX395_.jpg":[146,395],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX500_.jpg":[185,500],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX535_.jpg":[198,535],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX575_.jpg":[213,575],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX625_.jpg":[231,625],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX675_.jpg":[250,675],"https://images-na.ssl-images-amazon.com/images/I/71a7BKbdD3L._UX695_.jpg":[257,695]},"variant":"BOTT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX395_.jpg":[139,395],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX500_.jpg":[177,500],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX535_.jpg":[189,535],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX575_.jpg":[203,575],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX625_.jpg":[221,625],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX675_.jpg":[238,675],"https://images-na.ssl-images-amazon.com/images/I/8139cgDppVL._UX695_.jpg":[245,695]},"variant":"TOPP","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX395_.jpg":[186,395],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX500_.jpg":[235,500],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX535_.jpg":[252,535],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX575_.jpg":[271,575],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX625_.jpg":[294,625],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX675_.jpg":[318,675],"https://images-na.ssl-images-amazon.com/images/I/81a3uUSxI%2BL._UX695_.jpg":[327,695]},"variant":"RGHT","lowRes":null},{"hiRes":"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UL1500_.jpg","thumb":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL._US40_.jpg","large":"https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg","main":{"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX395_.jpg":[255,395],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX500_.jpg":[322,500],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX535_.jpg":[345,535],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX575_.jpg":[371,575],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX625_.jpg":[403,625],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX675_.jpg":[435,675],"https://images-na.ssl-images-amazon.com/images/I/815uXTfk02L._UX695_.jpg":[448,695]},"variant":"PAIR","lowRes":null}]},
            'colorToAsin': {'initial': {}},
            'holderRatio': 1.2,
            'holderMaxHeight': 700,
            'heroImage': {'initial': []},
            'heroVideo': {'initial': []},
            'spin360ColorData': {'initial': {}},
            'spin360ColorEnabled': {'initial': 0},
            'spin360ConfigEnabled': false,
            'spin360LazyLoadEnabled': false,
            'playVideoInImmersiveView':'false',
            'tabbedImmersiveViewTreatment':'C',
            'totalVideoCount':'0',
            'videoIngressATFSlateThumbURL':'',
            'mediaTypeCount':'0',
            'atfEnhancedHoverOverlay' : true,
            'winningAsin': 'B072596K2C',
            'weblabs' : {},
            'aibExp3Layout' : 1,
            'aibRuleName' : 'frank-powered',
            'acEnabled' : false
            };
A.trigger('P.AboveTheFold'); // trigger ATF event.
return data;
});"""

m = re.search(r'^var data = ({.*};)', s, re.S | re.M)
data = m.groups()[0]
jsonObj = json.loads(data[:-1].replace("'", '"'))  # remove the last semicolon and replace single quotes!

for img in jsonObj['colorImages']['initial']:
    print (img['large'])

印刷品：

https://images-na.ssl-images-amazon.com/images/I/41rNitnJpsL.jpg
https://images-na.ssl-images-amazon.com/images/I/41Q1eJ1c1tL.jpg
https://images-na.ssl-images-amazon.com/images/I/51%2BKgvmEndL.jpg
https://images-na.ssl-images-amazon.com/images/I/31rBxkzNDgL.jpg
https://images-na.ssl-images-amazon.com/images/I/41qECXntKAL.jpg
https://images-na.ssl-images-amazon.com/images/I/41rT%2B2GI9ZL.jpg
https://images-na.ssl-images-amazon.com/images/I/419Wv4M%2B-bL.jpg

相关问题 更多 >

    编程相关推荐