How to use python to capture Lianjia net second-hand house data
This article introduces the relevant knowledge of "how to use python to capture Lianjia net second-hand house data". In the operation of actual cases, many people will encounter such a dilemma, so let the editor lead you to learn how to deal with these situations. I hope you can read it carefully and be able to achieve something!
#-*-coding: utf-8
Import urllib2
Import urllib
Import re,os
Import time
# from bs4 import BeautifulSoup
Import sys
Reload (sys)
Sys.setdefaultencoding ('utf-8')
Class HomeLink:
# initialization data
Def _ init__ (self,base_url):
Self.base_url = base_url
Self.page = 1
Self.out_put_file = 'DJV _ apt _ pythonax _ TestUniverse _ house.txt'
Self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
Self.headers = {'User-Agent': self.user_agent}
# get page content
Def get_content (self,url):
Try:
Request = urllib2.Request (url,headers=self.headers)
Response = urllib2.urlopen (request)
Act_url = response.geturl ()
Print 'init url=',url,'act url=',act_url
If url = = act_url:
Content = response.read ()
Return content
Else:
Return None
Except urllib2.URLError, e:
If hasattr (e, "reason"):
Print u "failed to connect to the page, error reason", e.reason
Return None
# get the starting url link address of each zone
Def get_region_url (self):
D_region_url = {}
Content = self.get_content (self.base_url)
Pattern = re.compile ('(. *?)', re.S)
Result = re.findall (pattern,content)
If result:
For x in result:
D_region_url [x [1]] = x [0]
Else:
Pass
Return d_region_url
# get the list of url addresses for all pages in each zone
Def get_region_url_list (self,region_url):
Page_num = self.get_page_num (region_url)
L_url = [region_url+'pg'+str (I) +'/ 'for i in range (2)
Return l_url
# get the total number of pages
Def get_page_num (self,url):
Content = self.get_content (url)
Pattern = re.compile ('{"totalPage": (\ d +), "curPage": 1}', re.S)
Result = re.search (pattern,content)
If result:
Return int (result.group (1). Strip ())
Else:
Return None
# get the house price information of each house
Def get_house_info (self,url,region):
Content = self.get_content (url)
Pattern = re.compile (''+
'(. *?) (. *?).
+'. *? (\ d +) (\ S+)', re.S)
Result = re.findall (pattern,content)
If result:
For x in result:
L = x [1] .split ('|')
Rooms,area,direct,other = l [1], l [2], l [3], l [4]
S_str ='| '.join ([region,x [0], rooms,area,direct,other,x [2], x [3]])
Self.writeStr2File (self.out_put_file,s_str)
Else:
Return None
# start grabbing Lianjia's house price data
Def start_scrapy (self):
D_region_url = self.get_region_url ()
For k in d_region_url:
Region = k
Region_init_url = 'http://bj.lianjia.com' + dumped regionurl [region]
L_region_url = self.get_region_url_list (region_init_url)
For url in l_region_url:
Time.sleep (1)
Url = url.strip ()
Self.get_house_info (url,region)
# write a file
Def writeStr2File (self,out_put_file,str1,append ='a'):
# remove the file and keep the path. For example,'a _
SubPath = out_put_file [: self.out_put_file.rfind ('/')]
# if the folder does not exist in the given path, create
If not os.path.exists (subPath):
Os.makedirs (subPath)
# Open the file and write the str content to the given file
With open (out_put_file, append) as f:
F.write (str1.strip () +'\ n')
Url = 'http://bj.lianjia.com/ershoufang/'
Home = HomeLink (url)
Home.start_scrapy ()
"how to use python to grab Lianjia net second-hand room data" content is introduced here, thank you for reading. If you want to know more about the industry, you can follow the website, the editor will output more high-quality practical articles for you!
