python爬虫实战数据可视化分析的软件_Python爬虫实战数据可视化分析

#-*-coding:utf-8-*-

import json

import time

import multiprocessing

import requests

from lxml import etree

class HandelLaGou(object):

def __init__(self):

self.lagou_session = requests.session()

self.header = {

‘User-Agent’: ‘Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/525.19 (KHTML, like Gecko) Chrome/0.4.154.18 Safari/525.19’

}

self.city_list = “”

def handle_city(self):

city_url = “https://www.lagou.com/jobs/allCity.html”

city_result = self.handle_request(method=”GET”, url=city_url)

list = city_result.xpath(‘//ul[contains(@class, “city_list”)]/li/a/text()’)

self.city_list = [x for x in list]

self.lagou_session.cookies.clear()

def handle_city_job(self, city):

first_request_url = “https://www.lagou.com/jobs/list_pythonmp;px=default&city=%s”%city

first_response = self.handle_request(method=”GET”,url=first_request_url)

try:

total_page = first_response.xpath(‘//span[contains(@class, “span totalNum”)]/text()’)

print(total_page)

except:

return

else:

for i in range(1, int(total_page[0])+1):

data = {

“pn”:i,

“kd”:”python”

}

print(i)

page_url = “https://www.lagou.com/jobs/positionAjax.json=default&city=%s&needAddtionalResult=false”%city

referer_url = “https://www.lagou.com/jobs/list_pythonmp;px=default&city=%s”%city

self.header[‘Referer’] = referer_url.encode(‘utf-8’)

response = self.handle_request(method=”POST”, url=page_url, data=data, info=city)

lagou_data = json.loads(response)

job_list = lagou_data[‘content’][‘positionResult’][‘result’]

for job in job_list:

print(job)

def handle_request(self, method, url , data=None, info=None):

while True:

if method == “GET”:

response = self.lagou_session.get(url=url, headers=self.header)

item = etree.HTML(response.text)

elif method == “POST”:

response = self.lagou_session.post(url=url, headers=self.header, data=data)

item = response.text

if ‘频繁’ in response.text:

self.lagou_session.cookies.clear()

first_request_url = “https://www.lagou.com/jobs/list_pythonmp;px=default&city=%s” %info

self.handle_request(method=”GET”, url=first_request_url)

time.sleep(15)

continue

return item

if __name__ == ‘__main__’:

lagou = HandelLaGou()

lagou.handle_city()

pool = multiprocessing.Pool(2)

for city in lagou.city_list:

pool.apply_async(lagou.handle_city_job, args=(city,))

pool.close()

pool.join()

文章知识点与官方知识档案匹配，可进一步学习相关知识Python入门技能树 络爬虫urllib211399 人正在系统学习中 相关资源：开源的爬虫软件Heritrix3.1.0_开源爬虫-Java工具类资源-CSDN文库