How to grab Baidu Tieba's content with python
This article introduces the relevant knowledge of "how to use python to capture Baidu Tieba's content". In the operation of actual cases, many people will encounter such a dilemma, so let the editor lead you to learn how to deal with these situations. I hope you can read it carefully and be able to achieve something!
#-*-coding: utf-8
Import urllib2
Import urllib
Import re,os
Import time
Import sys
Reload (sys)
Sys.setdefaultencoding ('utf-8')
Class Tiebar:
# initialization data
Def _ init__ (self,base_url,see_lz):
Self.base_url = base_url
Self.see_lz ='? see_lz=' + see_lz
Self.page = 1
Self.user_agent = 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'
Self.headers = {'User-Agent': self.user_agent}
Self.tool = Tool ()
Self.out_put_file ='d _ Vera _ account _ pythonGap _ out. Txt'
# method of getting page content
Def get_cotent (self,page):
Try:
Url = self.base_url + self.see_lz +'& pn=' + str (page)
Request = urllib2.Request (url,headers=self.headers)
Response = urllib2.urlopen (request)
Act_url = response.geturl ()
Print 'init url=',url,'act url=',act_url
If url = = act_url:
Content = response.read ()
Return content
Else:
Return None
Except urllib2.URLError, e:
If hasattr (e, "reason"):
Print u "failed to connect to the paste bar page, error reason", e.reason
Return None
# get the topic of the post
Def get_titile (self):
Content = self.get_cotent (1)
Pattern = re.compile ('(. *)')
Result = re.search (pattern,content)
If result:
Return result.group (1) .strip ()
Else:
Return None
# get the number of pages of a post
Def get_page_num (self):
Content = self.get_cotent (1)
Pattern = re.compile ('
