我有一个名为def get_scraped_data():
的函数,它本质上是一个web scraper。它返回一个名为newlaw的数据帧,其中包含我所需的所有数据。整个代码如下。你知道吗
def get_scraped_data():
from bs4 import BeautifulSoup
import requests
import pandas as pd
urlh = 'http://solicitors.lawsociety.org.uk/search/results?Type=1&IncludeNlsp=True&Pro=True¶meters=%2C1%3BAPL%2C0%3B%2C1%3BPUB%2C0%3B%2C1%3BADV%2C0%3B%2C1%3BAGR%2C0%3B%2C1%3BAVI%2C0%3B%2C1%3BBAN%2C1%3B%2C1%3BBEN%2C0%3B%2C1%3BCHA%2C0%3B%2C1%3BCHI%2C0%3B%2C1%3BCLI%2C0%3B%2C1%3BCOL%2C1%3B%2C1%3BPCO%2C1%3B%2C1%3BCCL%2C0%3B%2C1%3BCOS%2C1%3B%2C1%3BCOM%2C1%3B%2C1%3BCON%2C1%3B%2C1%3BCSU%2C0%3B%2C1%3BCSF%2C0%3B%2C1%3BCSG%2C0%3B%2C1%3BCUT%2C0%3B%2C1%3BCTR%2C1%3B%2C1%3BPRE%2C0%3B%2C1%3BCFI%2C1%3B%2C1%3BCRD%2C0%3B%2C1%3BCRF%2C0%3B%2C1%3BCRG%2C0%3B%2C1%3BCRJ%2C0%3B%2C1%3BCRL%2C0%3B%2C1%3BCRM%2C0%3B%2C1%3BCRS%2C0%3B%2C1%3BCRO%2C1%3B%2C1%3BDEB%2C0%3B%2C1%3BDTR%2C1%3B%2C1%3BDEF%2C0%3B%2C1%3BDRC%2C0%3B%2C1%3BDRO%2C1%3B%2C1%3BEDU%2C0%3B%2C1%3BELC%2C0%3B%2C1%3BELH%2C0%3B%2C1%3BEMP%2C1%3B%2C1%3BENE%2C0%3B%2C1%3BENV%2C0%3B%2C1%3BEUN%2C0%3B%2C1%3BFDS%2C0%3B%2C1%3BFAM%2C0%3B%2C1%3BFAL%2C0%3B%2C1%3BFMC%2C0%3B%2C1%3BFME%2C0%3B%2C1%3BFML%2C0%3B%2C1%3BFPL%2C0%3B%2C1%3BFIS%2C0%3B%2C1%3BHRI%2C0%3B%2C1%3BIMA%2C0%3B%2C1%3BIML%2C0%3B%2C1%3BIMM%2C0%3B%2C1%3BIMG%2C0%3B%2C1%3BIMN%2C0%3B%2C1%3BITE%2C1%3B%2C1%3BINS%2C1%3B%2C1%3BIUR%2C1%3B%2C1%3BIPR%2C1%3B%2C1%3BJRW%2C0%3B%2C1%3BJRL%2C0%3B%2C1%3BLCO%2C1%3B%2C1%3BLRE%2C0%3B%2C1%3BPOA%2C0%3B%2C1%3BLIC%2C1%3B%2C1%3BLIV%2C0%3B%2C1%3BLIS%2C0%3B%2C1%3BLIT%2C0%3B%2C1%3BLPH%2C0%3B%2C1%3BLPP%2C0%3B%2C1%3BMAR%2C0%3B%2C1%3BMED%2C1%3B%2C1%3BMHE%2C0%3B%2C1%3BMHL%2C0%3B%2C1%3BMAA%2C1%3B%2C1%3BMIL%2C0%3B%2C1%3BNDI%2C0%3B%2C1%3BPEN%2C1%3B%2C1%3BPIN%2C0%3B%2C1%3BPIR%2C0%3B%2C1%3BPLA%2C0%3B%2C1%3BPRZ%2C0%3B%2C1%3BPRP%2C0%3B%2C1%3BPRT%2C0%3B%2C1%3BPRW%2C0%3B%2C1%3BPCI%2C0%3B%2C1%3BPCP%2C0%3B%2C1%3BPCT%2C0%3B%2C1%3BPCW%2C0%3B%2C1%3BPNE%2C0%3B%2C1%3BTAX%2C0%3B%2C1%3BTAC%2C1%3B%2C1%3BTAE%2C0%3B%2C1%3BTAH%2C1%3B%2C1%3BTAM%2C0%3B%2C1%3BTAP%2C0%3B%2C1%3BTAT%2C0%3B+'
r = requests.get(urlh)
soup = BeautifulSoup(r.content, 'html.parser')
names = []
roles = []
offices = []
locations = []
for i in range(1,2):
url = 'http://solicitors.lawsociety.org.uk/search/results?Type=1&IncludeNlsp=True&Pro=True¶meters=%2C1%3BAPL%2C0%3B%2C1%3BPUB%2C0%3B%2C1%3BADV%2C0%3B%2C1%3BAGR%2C0%3B%2C1%3BAVI%2C0%3B%2C1%3BBAN%2C1%3B%2C1%3BBEN%2C0%3B%2C1%3BCHA%2C0%3B%2C1%3BCHI%2C0%3B%2C1%3BCLI%2C0%3B%2C1%3BCOL%2C1%3B%2C1%3BPCO%2C1%3B%2C1%3BCCL%2C0%3B%2C1%3BCOS%2C1%3B%2C1%3BCOM%2C1%3B%2C1%3BCON%2C1%3B%2C1%3BCSU%2C0%3B%2C1%3BCSF%2C0%3B%2C1%3BCSG%2C0%3B%2C1%3BCUT%2C0%3B%2C1%3BCTR%2C1%3B%2C1%3BPRE%2C0%3B%2C1%3BCFI%2C1%3B%2C1%3BCRD%2C0%3B%2C1%3BCRF%2C0%3B%2C1%3BCRG%2C0%3B%2C1%3BCRJ%2C0%3B%2C1%3BCRL%2C0%3B%2C1%3BCRM%2C0%3B%2C1%3BCRS%2C0%3B%2C1%3BCRO%2C1%3B%2C1%3BDEB%2C0%3B%2C1%3BDTR%2C1%3B%2C1%3BDEF%2C0%3B%2C1%3BDRC%2C0%3B%2C1%3BDRO%2C1%3B%2C1%3BEDU%2C0%3B%2C1%3BELC%2C0%3B%2C1%3BELH%2C0%3B%2C1%3BEMP%2C1%3B%2C1%3BENE%2C0%3B%2C1%3BENV%2C0%3B%2C1%3BEUN%2C0%3B%2C1%3BFDS%2C0%3B%2C1%3BFAM%2C0%3B%2C1%3BFAL%2C0%3B%2C1%3BFMC%2C0%3B%2C1%3BFME%2C0%3B%2C1%3BFML%2C0%3B%2C1%3BFPL%2C0%3B%2C1%3BFIS%2C0%3B%2C1%3BHRI%2C0%3B%2C1%3BIMA%2C0%3B%2C1%3BIML%2C0%3B%2C1%3BIMM%2C0%3B%2C1%3BIMG%2C0%3B%2C1%3BIMN%2C0%3B%2C1%3BITE%2C1%3B%2C1%3BINS%2C1%3B%2C1%3BIUR%2C1%3B%2C1%3BIPR%2C1%3B%2C1%3BJRW%2C0%3B%2C1%3BJRL%2C0%3B%2C1%3BLCO%2C1%3B%2C1%3BLRE%2C0%3B%2C1%3BPOA%2C0%3B%2C1%3BLIC%2C1%3B%2C1%3BLIV%2C0%3B%2C1%3BLIS%2C0%3B%2C1%3BLIT%2C0%3B%2C1%3BLPH%2C0%3B%2C1%3BLPP%2C0%3B%2C1%3BMAR%2C0%3B%2C1%3BMED%2C1%3B%2C1%3BMHE%2C0%3B%2C1%3BMHL%2C0%3B%2C1%3BMAA%2C1%3B%2C1%3BMIL%2C0%3B%2C1%3BNDI%2C0%3B%2C1%3BPEN%2C1%3B%2C1%3BPIN%2C0%3B%2C1%3BPIR%2C0%3B%2C1%3BPLA%2C0%3B%2C1%3BPRZ%2C0%3B%2C1%3BPRP%2C0%3B%2C1%3BPRT%2C0%3B%2C1%3BPRW%2C0%3B%2C1%3BPCI%2C0%3B%2C1%3BPCP%2C0%3B%2C1%3BPCT%2C0%3B%2C1%3BPCW%2C0%3B%2C1%3BPNE%2C0%3B%2C1%3BTAX%2C0%3B%2C1%3BTAC%2C1%3B%2C1%3BTAE%2C0%3B%2C1%3BTAH%2C1%3B%2C1%3BTAM%2C0%3B%2C1%3BTAP%2C0%3B%2C1%3BTAT%2C0%3B+' + '=&Page=' + str(i)
response = requests.get(url)
response.raise_for_status()
soup = BeautifulSoup(response.content, 'html.parser')
hp_sol_data = soup.find_all('section', {'class':'solicitor'})
for sol in hp_sol_data:
try:
addy = sol.contents[7].find_all('dd', {'class':'feature highlight'})[0].text
locations.append(addy)
except IndexError:
locations.append('None Found')
try:
office_names = sol.contents[7].find_all('dd', {'class':'highlight'})[0].text
offices.append(office_names)
except IndexError:
offices.append('None Found')
for link in soup.find_all('a', href=True):
if link.get('href').startswith('/person/'):
tags = (link.get('href'))
url2 = 'http://solicitors.lawsociety.org.uk' + str(tags)
r2 = requests.get(url2)
soup = BeautifulSoup(r2.content, 'html.parser')
s_data = soup.find_all('article', {'class':'solicitor solicitor-type-individual details'})
for item in s_data:
solicitor_names = (item.contents[3].find_all('h1')[0].text)
names.append(solicitor_names)
try:
role = (item.find_all('div', {'class':'panel-half'})[1].find('dd').get_text(''))
roles.append(role)
except IndexError:
roles.append('Role not specified')
tls_solicitors = pd.DataFrame({'Solicitor_Name': names,
'Role': roles,
'Office': offices,
'Address': locations},
columns = ['Solicitor_Name', 'Office', 'Address', 'Role'])
law = tls_solicitors
newd = law['Role'].str.split('\n', n=3, expand = True)
law['Primary_Role'] = newd[0]
law['Secondary_Role'] = newd[1]
law['Other_Role'] = newd[2]
law['Other_Role_1'] = newd[3]
law['Other_Role_2'] = ''
law['Other_Role_3'] = ''
law['Other_Role_4'] = ''
newlaw = law.drop('Role', axis=1)
return newlaw
然后我编辑了我的视图.py调用函数并将其另存为变量data(data=get\u scraped\u data())。你知道吗
from django.shortcuts import render
from .tls import get_scraped_data
from django.http import HttpResponse
def law_view(request, *args, **kwargs):
data = get_scraped_data()
return render(request, "law.html", {'data': data})
在我试图显示我创建了一个模板的法律.html'并插入此代码。你知道吗
<table class="table table-striped">
<thead>
<tr>
{% for row in data %}
<th>{{ row }}</th>
{% endfor %}
</tr>
</thead>
<tbody>
<tr>
{% for data in data %}
<td>{{ data }}</td>
{% endfor %}
</tr>
</tbody>
</table>
这里的问题是只有DataFrame newlaw的头显示。 例如
律师姓名、办公室、地址等。。你知道吗
我还尝试了一个{data}},它正好显示了我想要的内容,但是我似乎找不到一种方法来处理数据并将其放入一个表中,这样它就更赏心悦目了。你知道吗
我需要帮助。你知道吗
您应该将所有代码包装在tls.py公司在函数中,它返回
newlaw
(假设这是您想要的输出)。所以:在你看来,你可以这样做:
在你的模板里(tls.html文件)然后可以循环浏览数据:
当然,您可能希望以更好的方式显示它,例如在
<table>
中。为此,只需将所有内容包装在<table>
中,显示标题行,然后遍历包装在<tr>
中的行,并遍历包装在<td>
中的每个列。你知道吗我认为
DataFrame
也有很多方法来呈现自己,所以您也可以这样做:它将数据呈现为HTML表。你知道吗
相关问题 更多 >
编程相关推荐