Python基于requests库爬取网站信息

import requests
from bs4 import BeautifulSoup
r=requests.get(\'http://www.baidu.com\')
r.encoding=None
result=r.text
bs=BeautifulSoup(result,\'html.parser\')
print(bs.title)
print(bs.title.text)

import requests
from bs4 import BeautifulSoup

#用来解决乱码现象，所以编写爬取信息的代码最好带上（输出出现乱码或者UnicodeEncodeError：\'gbk\'codec can\'t encode character） 
import io   
import sys
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding=\'gb18030\')

#用来防止反爬取，可以了解一下
headers={\"User-Agent\" : \"Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.1.6)\",
　　\"Accept\" : \"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\",
　　\"Accept-Language\" : \"en-us\",
　　\"Connection\" : \"keep-alive\",
　　\"Accept-Charset\" : \"GB2312,utf-8;q=0.7,*;q=0.7\"
}

#获取51job网站的基本信息
r=requests.get(\'https://search.51job.com/list/000000,000000,0000,00,9,99,python,2,1.html?lang=c&stype=&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&providesalary=99&lonlat=0%2C0&radius=-1&ord_field=0&confirmdate=9&fromType=&dibiaoid=0&address=&line=&specialarea=00&from=&welfare=\')
r.encoding=r.apparent_encoding
result=r.text
bs=BeautifulSoup(result,\'html.parser\')
print(bs.prettify())

u1=bs.find_all(\'u1\',attrs={\'class\':\'item_con_list\'})  #这部分代码便是我们爬取的目标，51job网站上关于python职业的薪资
print(len(u1))
li=bs.find_all(\'span\',attrs={\'class\':\'t4\'})
for l in li:
  print(l.text)