The method of logging in and generating Cookie by Selenium crawler
This article mainly explains "the method of Selenium crawler login to generate Cookie", interested friends may wish to have a look. The method introduced in this paper is simple, fast and practical. Next let the editor to take you to learn "Selenium crawler login to generate Cookie" bar!
Selenium crawler login to generate Cookie
To log in to a website, in essence, the server gives the client a credential after the login is completed, which can be stored in cookie or somewhere else. The next time the client requests a web page, the client will bring this credential, and the server will think that the client has logged in. So for crawlers, generating cookies for proxy use has become a must.
Example
Install chrome and corresponding driver
Download the corresponding version of chrome deriver for chrome
Download the corresponding version of driver
Https://chromedriver.chromium.org/downloads
For example:
Generate cookies
We use chrome driver for login and cookie generation
Import os
Import time
Import zipfile
From selenium import webdriver
From selenium.common.exceptions import TimeoutException
From selenium.webdriver.common.by import By
From selenium.webdriver.support import expected_conditions as EC
From selenium.webdriver.support.ui import WebDriverWait
Class GenCookies (object):
USER_AGENT = open ('useragents.txt'). Readlines ()
# 16yun proxy configuration
PROXY_HOST = 't.16yun.cn' # proxy or host Niuniu Agent www.16yun.cn
PROXY_PORT = 31111 # port
PROXY_USER = 'USERNAME' # username
PROXY_PASS = 'PASSWORD' # password
@ classmethod
Def get_chromedriver (cls, use_proxy=False, user_agent=None):
Manifest_json = ""
{
"version": "1.0.0"
"manifest_version": 2
"name": "Chrome Proxy"
"permissions": [
"proxy"
"tabs"
"unlimitedStorage"
"storage"
"
"webRequest"
"webRequestBlocking"
]
"background": {
"scripts": ["background.js"]
}
"minimum_chrome_version": "22.0.0"
}
"
Background_js = ""
Var config = {
Mode: "fixed_servers"
Rules: {
SingleProxy: {
Scheme: "http"
Host: "% s"
Port: parseInt (% s)
}
BypassList: ["localhost"]
}
}
Chrome.proxy.settings.set ({value: config, scope: "regular"}, function () {})
Function callbackFn (details) {
Return {
AuthCredentials: {
Username: "% s"
Password: "% s"
}
}
}
Chrome.webRequest.onAuthRequired.addListener (
CallbackFn
{urls: ["]}
['blocking']
);
""% (cls.PROXY_HOST, cls.PROXY_PORT, cls.PROXY_USER, cls.PROXY_PASS)
Path = os.path.dirname (os.path.abspath (_ _ file__))
Chrome_options = webdriver.ChromeOptions ()
If use_proxy:
Pluginfile = 'proxy_auth_plugin.zip'
With zipfile.ZipFile (pluginfile,'w') as zp:
Zp.writestr ("manifest.json", manifest_json)
Zp.writestr ("background.js", background_js)
Chrome_options.add_extension (pluginfile)
If user_agent:
Chrome_options.add_argument ('--user-agent=%s'% user_agent)
Driver = webdriver.Chrome (
Os.path.join (path, 'chromedriver')
Chrome_options=chrome_options)
Return driver
Def _ _ init__ (self, username, password):
Self.url = 'https://passport.weibo.cn/signin/login?entry=mweibo&r=https://m.weibo.cn/'
Self.browser = self.get_chromedriver (use_proxy=True, user_agent=self.USER_AGENT)
Self.wait = WebDriverWait (self.browser, 20)
Self.username = username
Self.password = password
Def open (self):
"
Open the web page and enter the user name and password and click
: return: None
"
Self.browser.delete_all_cookies ()
Self.browser.get (self.url)
Username = self.wait.until (EC.presence_of_element_located ((By.ID, 'loginName')
Password = self.wait.until (EC.presence_of_element_located ((By.ID, 'loginPassword')
Submit = self.wait.until (EC.element_to_be_clickable ((By.ID, 'loginAction')
Username.send_keys (self.username)
Password.send_keys (self.password)
Time.sleep (1)
Submit.click ()
Def password_error (self):
"
Determine if the password is wrong
: return:
"
Try:
Return WebDriverWait (self.browser, 5). Until
EC.text_to_be_present_in_element ((By.ID, 'errorMsg'),' incorrect username or password'))
Except TimeoutException:
Return False
Def get_cookies (self):
"
Get Cookies
: return:
"
Return self.browser.get_cookies ()
Def main (self):
"
Entrance
: return:
"
Self.open ()
If self.password_error ():
Return {
'status': 2
'content':' wrong username or password'
}
# if you log in successfully without a CAPTCHA
Cookies = self.get_cookies ()
Return {
'status': 1
'content': cookies
}
If _ _ name__ = ='_ _ main__':
Result = GenCookies (
Username='180000000'
Password='16yun'
). Main ()
Print (result)
Import osimport timeimport zipfilefrom selenium import webdriverfrom selenium.common.exceptions import TimeoutExceptionfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.support import expected_conditions as ECfrom selenium.webdriver.support.ui import WebDriverWaitclass GenCookies (object): USER_AGENT = open ('useragents.txt'). Readlines () # 16yun proxy configuration PROXY_HOST =' t.16yun.cn' # proxy or host Eniuyun Agent www.16yun.cn PROXY_PORT = 31111 # port PROXY_USER = 'USERNAME' # username PROXY_PASS = 'PASSWORD' # password @ classmethod def get_chromedriver (cls Use_proxy=False, user_agent=None): manifest_json = "" {"version": "1.0.0", "manifest_version": 2, "name": "Chrome Proxy", "permissions": ["proxy", "tabs", "unlimitedStorage" "storage", "", "webRequest", "webRequestBlocking"], "background": {"scripts": ["background.js"]} "minimum_chrome_version": "22.0.0"} "" background_js = "var config = {mode:" fixed_servers ", rules: {singleProxy: {scheme:" http ", host:"% s " Port: parseInt (% s)}, bypassList: ["localhost"]}} Chrome.proxy.settings.set ({value: config, scope: "regular"}, function () {}); function callbackFn (details) {return {authCredentials: {username: "% s", password: "% s"}} } chrome.webRequest.onAuthRequired.addListener (callbackFn, {urls: [""]}, ['blocking']) ""% (cls.PROXY_HOST, cls.PROXY_PORT, cls.PROXY_USER, cls.PROXY_PASS) path = os.path.dirname (os.path.abspath (_ file__)) chrome_options = webdriver.ChromeOptions () if use_proxy: pluginfile = 'proxy_auth_plugin.zip' with zipfile.ZipFile (pluginfile) 'w') as zp: zp.writestr ("manifest.json", manifest_json) zp.writestr ("background.js", background_js) chrome_options.add_extension (pluginfile) if user_agent: chrome_options.add_argument ('-user-agent=%s'% user_agent) driver = webdriver.Chrome (os.path.join (path) 'chromedriver'), chrome_options=chrome_options) return driver def _ _ init__ (self, username, password): self.url =' https://passport.weibo.cn/signin/login?entry=mweibo&r=https://m.weibo.cn/' self.browser = self.get_chromedriver (use_proxy=True, user_agent=self.USER_AGENT) self.wait = WebDriverWait (self.browser 20) self.username = username self.password = password def open (self): "Open the web page to enter the username and password and click: return: None" self.browser.delete_all_cookies () self.browser.get (self.url) username = self.wait.until ((By.ID) " 'loginName') password = self.wait.until (EC.presence_of_element_located ((By.ID,' loginPassword')) submit = self.wait.until (EC.element_to_be_clickable ((By.ID)) 'loginAction')) username.send_keys (self.username) password.send_keys (self.password) time.sleep (1) submit.click () def password_error (self): "determine whether the password is incorrect: return:" try: return WebDriverWait (self.browser) 5. Until (EC.text_to_be_present_in_element ((By.ID, 'errorMsg')) ) except TimeoutException: return False def get_cookies (self): "get Cookies: return:" return self.browser.get_cookies () def main (self): "" entry: return: "self. Open () if self.password_error (): return {'status': 2 Wrong 'content':' username or password'} # if you login successfully cookies = self.get_cookies () return {'status': 1,' content': cookies} if _ _ name__ ='_ _ main__': result = GenCookies (username='180000000') Password='16yun',) .main () print (result)
The useragents.txt file contains random ua.
Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.16) Gecko/20110319 Firefox/40Mozilla/5.0 (Windows NT 6.2; rv:39.0) Gecko/20100101 Firefox/39.0Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; QQBrowser/8.3.4769.400) Mozilla/5.0 (Windows NT 6.1) Rv:39.0) Gecko/20100101 Firefox/39.0 here, I believe that everyone on the "Selenium crawler login to generate Cookie method" have a deeper understanding, might as well to actual operation it! Here is the website, more related content can enter the relevant channels to inquire, follow us, continue to learn!
