尽管用户代理已更改，但未处理或不允许使用零碎的HTTP状态代码

Question

我很难在JS:https://www.jobteaser.com/en/job-offers中创建一个呈现所有页面的网站

在用调试器工具检查了请求之后，我看到我想要的所有内容都是用AJAX以json格式发送的。在

The file returning the content

所以我做了以下蜘蛛来获取特定搜索的内容：

import scrapy
from scrapy import Request
import json

class JobteaserSpider(scrapy.Spider):
    name="jobteaser"
    start_urls=['https://www.jobteaser.com/fr/job-offers?q%3Dbusiness%20analyst%26contract%3Dstage%2Cinternship%2Cwerkstudent%26location%3DFrance..France%26locale%3Dfr%2Cen']

    def parse(self,response):

        apiKey="..."

        header ={
            "requests":[
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=*&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=abroad_only&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=company_business_type&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=company_sectors&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=contract_duration&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=contract&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%5D&hitsPerPage=20&page=0&facets=locale&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=location&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=position_category&distinct=true&facetingAfterDistinct=true"},
                {"indexName":"job_offers",
                "params":"query=business%20analyst&facetFilters=%5B%5B%22contract%3Astage%22%2C%22contract%3Ainternship%22%2C%22contract%3Awerkstudent%22%5D%2C%5B%22location%3AFrance%22%5D%2C%5B%22locale%3Afr%22%2C%22locale%3Aen%22%5D%5D&hitsPerPage=20&page=0&facets=start_date&distinct=true&facetingAfterDistinct=true"},

            ],
            "apiKey":apiKey

        }
        yield scrapy.Request(
            url="https://9vcp793ivh-dsn.algolia.net/1/indexes/*/queries?x-algolia-agent=Algolia%20for%20vanilla%20JavaScript%203.30.0&x-algolia-application-id=9VCP793IVH",
            method='POST',
            body=json.dumps(header),
            headers={'Content-Type':'application/json'},
            callback=self.parse_internship)


    def parse_internship(self,response):
        yield{"E":response.body}
        pass

头也以.json格式发送。USER_AGENT已更改，ROBOTSTXT_OBEY设置为False。尽管采取了这些措施，我还是犯了这个错误：

^{pr2}$

可能是通过请求发送的URL不好，但是在彻底分析原始URL之后，我找不到正确的URL。在

谢谢！在

Answer 1

所以这比我想象的要容易得多，只需从另一个显示json数据的页面检索apiKey。然后，当设置了正确的apiKey时，页面将发送所需的内容。在