Python crawler: crawl novels and store them in a database

Crawl the novel from the novel website and save it to the database

The first step: get the content of the novel first

#! / usr/bin/python#-*-coding: UTF-8-*-import urllib2,redomain = 'http://www.quanshu.net'headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64) X64) AppleWebKit/537.36 (KHTML Like Gecko) Chrome/58.0.3029.110 Safari/537.36 "} def getTypeList (pn=1): # function to get the classification list req = urllib2.Request ('http://www.quanshu.net/map/%s.html'% pn) # object to be requested by the instance req.headers = headers # replace all header information # req.add_header () # add a single header information res = urllib2.urlopen ( Req) # start requesting html = res.read () .decode ('gbk') # decode decoding Decode it into Unicode reg = r'(. *)' Reg = re.compile (reg) # increase matching efficiency the type returned by regular matching is List return re.findall (reg,html) def getNovelList (url): # get chapter list function req = urllib2.Request (domain + url) req.headers = headers res = urllib2.urlopen (req) html = res.read (). Decode ('gbk') reg = r' (. *?)' Reg = re.compile (reg) return re.findall (reg,html) def getNovelContent (url): # get chapter content req = urllib2.Request (domain + url) req.headers = headers res = urllib2.urlopen (req) html = res.read (). Decode ('gbk') reg = r'style5\ (\) (. *?) style6\ (\) 'return re.findall (reg,html) [0] if _ _ name__ = =' _ main__': for type in range (1jue 10): for url,title in getTypeList (type): for zurl,ztitle in getNovelList (url): print u' regular crawl -% s'% ztitle content = getNovelContent (url.replace ('index.html') Zurl)) print content break break

The results are as follows:

Step 2: store to the database

1. Design database

1.1 New Library: novel

1.2Design table: novel

1.3Design table: chapter

And set foreign keys

2. Write a script

#! / usr/bin/python#-*-coding: UTF-8-*-import urllib2,reimport MySQLdbclass Sql (object): conn = MySQLdb.connect (host='192.168.19.213',port=3306,user='root',passwd='Admin123',db='novel',charset='utf8') def addnovels (self,sort,novelname): cur = self.conn.cursor () cur.execute ("insert into novel (sort,novelname) values (% s,'% s')"% (sort Novelname) lastrowid = cur.lastrowid cur.close () self.conn.commit () return lastrowid def addchapters (self,novelid,chaptername,content): cur = self.conn.cursor () cur.execute ("insert into chapter (novelid,chaptername,content) values (% s,'% s','% s')"% (novelid,chaptername Content)) cur.close () self.conn.commit () domain = 'http://www.quanshu.net'headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.3) Win64 X64) AppleWebKit/537.36 (KHTML Like Gecko) Chrome/58.0.3029.110 Safari/537.36 "} def getTypeList (pn=1): # function to get the classification list req = urllib2.Request ('http://www.quanshu.net/map/%s.html'% pn) # object to be requested by the instance req.headers = headers # replace all header information # req.add_header () # add a single header information res = urllib2.urlopen ( Req) # start requesting html = res.read () .decode ('gbk') # decode decoding Decode it into Unicode reg = r'(. *)' Reg = re.compile (reg) # increase matching efficiency the type returned by regular matching is List return re.findall (reg,html) def getNovelList (url): # get chapter list function req = urllib2.Request (domain + url) req.headers = headers res = urllib2.urlopen (req) html = res.read (). Decode ('gbk') reg = r' (. *?)' Reg = re.compile (reg) return re.findall (reg,html) def getNovelContent (url): # get chapter content req = urllib2.Request (domain + url) req.headers = headers res = urllib2.urlopen (req) html = res.read (). Decode ('gbk') reg = r'style5\ (\) (. *?) style6\ (\) 'return re.findall (reg,html) [0] mysql = Sql () if _ _ name__ =' _ _ main__': for sort in range (1m 10): for url,title in getTypeList (sort): lastrowid = mysql.addnovels (sort, title) for zurl Ztitle in getNovelList (url): print u' regular crawl -% s'% ztitle content = getNovelContent (url.replace ('index.html',zurl)) print u' is being stored -% s'% ztitle mysql.addchapters (lastrowid,ztitle,content)

3. Execute the script

4. View the database

You can see that it has been successfully stored.

Error report:

_ mysql_exceptions.OperationalError: (1364, "Field 'novelid' doesn't have a default value")

Solution: execute sql statement

SELECT @ @ GLOBAL.sql_mode

SET @ @ GLOBAL.sql_mode= "NO_ENGINE_SUBSTITUTION"

Error reference: http://blog.sina.com.cn/s/blog_6d2b3e4901011j9w.html