Python simple crawler

Crawl Lianjia's second-hand housing information

Import requestsimport refrom bs4 import BeautifulSoupimport csvurl = ['https://cq.lianjia.com/ershoufang/']for i in range (2101): url.append (' https://cq.lianjia.com/ershoufang/pg%s/'%(str(i))) # simulates Google browser headers = {'User-Agent':' Mozilla/5.0 (Windows NT 10.0; Win64) X64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36'} for u in url: r = requests.get (uLTEX headers) soup = BeautifulSoup (r.textlml`) .find_all ('li' Class_='clear LOGCLICKDATA') for i in soup: ns = i.select ('div [class = "positionInfo"]') [0] .get _ text () region = ns.split ('-') [1] .replace (','). Encode ('gbk') rem = ns.split (' -') [0] .replace ('' ''). Encode ('gbk') ns = i.select (' div [class = "houseInfo"]') [0] .get _ text () xiaoqu_name = ns.split ('|) [0] .replace (','). Encode ('gbk') huxing = ns.split (' |) [1] .replace ('') ''). Encode ('gbk') pingfang = ns.split (' |) [2]. Replace ('','). Encode ('gbk') chaoxiang = ns.split (' |) [3]. Replace ('','). Encode ('gbk') zhuangxiu = ns.split (' |) [4]. Replace (','). Encode ('gbk') danjia = re.findall ("\ d +") I.select ('div [class = "unitPrice"]') [0] .string) [0] zongjia = i.select ('div [class = "totalPrice"]') [0] .get _ text () .encode ('gbk') out=open ("/ data/data.csv",' a') csv_write=csv.writer (out) data = [region,xiaoqu_name,rem,huxing,pingfang,chaoxiang,zhuangxiu,danjia Zongjia] csv_write.writerow (data) out.close ()

Data result