# -*-coding:UTF-8 -*-
import multiprocessing as mp
from multiprocessing import Queue
import BeautifulSoup as bs4
from Queue import Empty
import urllib
import json
'''
getPageData 类似于生产者，获取工作列表。 getDetail 类似消费者，去获取工作的详细信息。queue 中存放工作详情页面的ID
'''
interface = 'http://www.lagou.com/jobs/positionAjax.json?px=default&yx=10k-15k&needAddtiOnalResult=false'
detailUrl = 'http://www.lagou.com/jobs/{0}.html'
def getPageData(task, queue, keyword='python'):
while True:
try:
page = task.get(timeout=1)
except Empty:
break
post_data = {'kd': keyword, 'pn': page, 'first': 'false'}
opener = urllib.urlopen(interface, urllib.urlencode(post_data))
jsOnData= json.loads(opener.read())
results = jsonData['content']['positionResult']['result']
for result in results:
queue.put(result['positionId'])
def getDetail(queue, result):
while True:
try:
positiOnId= queue.get(timeout=1)
except Empty:
print mp.current_process().name + 'exit'
break
url = detailUrl.format(positionId)
print url, mp.current_process().name
opener = urllib.urlopen(url)
html = opener.read()
soup = bs4.BeautifulSoup(html)
cOntent= soup.findAll(attrs={"class": "job_bt"})[0]
result.put('{0}
{1}'.format(detailUrl.format(positionId), content))
def start(keyword='python'):
task = Queue()
queue = Queue()
result = Queue()
post_data = {'kd': keyword, 'pn': 1, 'first': 'true'}
opener = urllib.urlopen(interface, urllib.urlencode(post_data))
jsOnData= json.loads(opener.read())
# 页数
totalCount = jsonData['content']['positionResult']['totalCount']
resultSize = jsonData['content']['positionResult']['resultSize']
pageNums = totalCount / resultSize
if totalCount % resultSize:
pageNums += 1
results = jsonData['content']['positionResult']['result']
for r in results:
queue.put(r['positionId'])
# 调试前三页
pageNums = 3
for i in range(2, pageNums + 1):
task.put(i)
num_cOnsumers= mp.cpu_count()
processes = [mp.Process(target=getDetail, args=(queue, result))
for _ in range(num_consumers)]
processes.append(mp.Process(target=getPageData, args=(task, queue)))
for p in processes:
p.start()
for p in processes:
p.join()
print 'processes over'
with open('jobs', 'w+') as f:
while not result.empty():
a = result.get()
f.write(a)
if __name__ == '__main__':
start()
到此这篇dockerdesktop运行错误（docker desktop is shutting down）的文章就介绍到这了,更多相关 内容请继续浏览下面的相关 推荐文章，希望大家都能在编程的领域有一番成就！版权声明：
本文来自互联网用户投稿，该文观点仅代表作者本人，不代表本站立场。本站仅提供信息存储空间服务，不拥有所有权，不承担相关法律责任。
如若内容造成侵权、违法违规、事实不符，请将相关资料发送至xkadmin@xkablog.com进行投诉反馈，一经查实，立即处理！
转载请注明出处，原文链接：https://www.xkablog.com/rfx/19988.html