Niftyindices.com无法抓取指数数据

Question

我之前能够从印度NSE的Niftyindices.com网站上抓取数据，这些数据是我需要的历史指数数据。这个网站是www.niftyindices.com，你可以在首页的报告部分找到历史数据。在这个页面上，你需要选择指数类型为“股票”，然后选择指数为“NIFTY 100”。数据范围我一般选择从2001年1月1日到今天。

直到昨天，我都能从这个网站获取数据，但今天早上突然就无法用同样的代码获取到数据了。以下是我用来抓取数据的代码。bm_sv是一个重要的cookie，必须传给网站——这是我之前的经验。现在收到的Json数据里没有任何数据了。

谢谢大家的帮助！

import requests
import json
from datetime import datetime


print('start')

data=[]
headers = {'Content-Type': 'application/json; charset=utf-8'
           ,'Accept':'application/json, text/javascript, */*; q=0.01'
           ,'Accept-Encoding':'gzip, deflate'
           ,'Accept-Language':'en-US,en;q=0.9'
           ,'Content-Length':'100'
           ,'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'
           ,'Cookie':'bm_sv=9B706239B47F50CA0B651E20BA5CBF74~YAAQFjkgF2zWSiyOAQAAdV0dQRfVSICkZc20SfI+PnDk8taK1Ppu1ZSmjclFkHqVgsGOE0vK3WnPMHuhY5kOStjVm4OnN1wm9SBRO3nIAvXWAVCR8iN23B8R7kHpcme82M8ytCrJ/LozntCxQlQSFqzuFwLw4+ZPBjdkICfQH4piCmjvZB3AH8NvCmf+nbzT34Q4JO4zYeYadkjlKjVRVIh0lzX2BK8crljTE9W+F1DUdtZYBRBUCM83OIfmZhnH6PnDu79C~1'
           }

row=['indexname','date','open','high','low','close']
data.append(row)

payload={'name': 'NIFTY 100','startDate':'01-Jan-2001','endDate': '10-Mar-2024'}
JSonURL='https://www.niftyindices.com/Backpage.aspx/getHistoricaldatatabletoString'
r=requests.post(JSonURL, data=json.dumps(payload),headers=headers)
print(r.text)
text=r.json()
print(text)
datab=json.loads(text['d'])
sorted_data=sorted(datab,key=lambda x: datetime.strptime(x['HistoricalDate'], '%d %b %Y'), reverse=False)
print('startdata available from: ',datetime.strftime(datetime.strptime(sorted_data[0]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y'))
print('data available till',datetime.strftime(datetime.strptime(sorted_data[len(datab)-1]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y\n'))
for rec in sorted_data:
    row=[]
    row.append(rec['Index Name'])
    row.append(datetime.strptime(rec['HistoricalDate'], '%d %b %Y'))
    row.append(rec['OPEN'].replace('-','0'))
    row.append(rec['HIGH'].replace('-','0'))
    row.append(rec['LOW'].replace('-','0'))
    row.append(rec['CLOSE'])
    print(row)
    data.append(row)
        
print(data)

Answer 1

你使用的负载（数据）需要是一个嵌套字典。请查看下面这个（修改过的）代码。现在它可以正常工作了！

新增一行代码

mypayload={"cinfo":json.dumps(payload)}

修改了一行代码（使用上面新创建的负载）

r=requests.post(JSonURL, data=json.dumps(mypayload),headers=headers)

import requests
import json
from datetime import datetime


print('start')

data=[]
headers = {'Content-Type': 'application/json; charset=utf-8'
           ,'Accept':'application/json, text/javascript, */*; q=0.01'
           ,'Accept-Encoding':'gzip, deflate'
           ,'Accept-Language':'en-US,en;q=0.9'
           ,'Content-Length':'100'
           ,'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.190 Safari/537.36'
           ,'Cookie':'bm_sv=9B706239B47F50CA0B651E20BA5CBF74~YAAQFjkgF2zWSiyOAQAAdV0dQRfVSICkZc20SfI+PnDk8taK1Ppu1ZSmjclFkHqVgsGOE0vK3WnPMHuhY5kOStjVm4OnN1wm9SBRO3nIAvXWAVCR8iN23B8R7kHpcme82M8ytCrJ/LozntCxQlQSFqzuFwLw4+ZPBjdkICfQH4piCmjvZB3AH8NvCmf+nbzT34Q4JO4zYeYadkjlKjVRVIh0lzX2BK8crljTE9W+F1DUdtZYBRBUCM83OIfmZhnH6PnDu79C~1'
           }

row=['indexname','date','open','high','low','close']
data.append(row)

payload={'name': 'NIFTY 100','startDate':'01-Jan-2001','endDate': '10-Mar-2024'}
mypayload={"cinfo":json.dumps(payload)}
JSonURL='https://www.niftyindices.com/Backpage.aspx/getHistoricaldatatabletoString'
r=requests.post(JSonURL, data=json.dumps(mypayload),headers=headers)
print(r.text)
text=r.json()
print(text)
datab=json.loads(text['d'])
sorted_data=sorted(datab,key=lambda x: datetime.strptime(x['HistoricalDate'], '%d %b %Y'), reverse=False)
print('startdata available from: ',datetime.strftime(datetime.strptime(sorted_data[0]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y'))
print('data available till',datetime.strftime(datetime.strptime(sorted_data[len(datab)-1]['HistoricalDate'], '%d %b %Y'),'%d-%b-%Y\n'))
for rec in sorted_data:
    row=[]
    row.append(rec['Index Name'])
    row.append(datetime.strptime(rec['HistoricalDate'], '%d %b %Y'))
    row.append(rec['OPEN'].replace('-','0'))
    row.append(rec['HIGH'].replace('-','0'))
    row.append(rec['LOW'].replace('-','0'))
    row.append(rec['CLOSE'])
    print(row)
    data.append(row)
        
print(data)