python爬虫3——爬取腾讯招聘全部招聘信息

python爬虫2中，已经有了初步的代码，之后做了优化

增加了工作职责、工作要求：

获取的数据有：

代码如下：

#!/usr/bin/env python
# -*- coding:utf-8 -*-

from bs4 import BeautifulSoup
import urllib2
import urllib
import json    # 使用了json格式存储


def tengxun(detail,num):
    url = 'https://hr.tencent.com/'
    # detail = 'position.php?&start=0#a'
    request = urllib2.Request(url + detail)
    response =urllib2.urlopen(request)
    resHtml = response.read()
    soup = BeautifulSoup(resHtml,'html.parser', from_encoding='utf-8')
    result = soup.select(".even")
    result += soup.select(".odd")
    # print len(result)
    # 处理页面
    items = []
    for node in result:
        item = {}
        # 职位名
        zname = node.select('td')[0].get_text()
        # 职位类别
        ztype = node.select('td')[1].get_text()
        # 人数
        znum = node.select('td')[2].get_text()
        # 地点
        zlocal = node.select('td')[3].get_text()
        # 发布时间
        ztime = node.select('td')[4].get_text()
        # 链接
        detailLink = node.select('td a')[0].attrs['href']
        # 获取工作职责、工作要求
        request1 = urllib2.Request(url + detailLink)
        response1 = urllib2.urlopen(request1)
        jobHtml = response1.read()
        soup1 = BeautifulSoup(jobHtml, 'html.parser', from_encoding='utf-8')
        # print len(soup1.select('ul.squareli'))
        # 工作职责
        jobRes = ''
        for li in soup1.select('ul.squareli')[0].select('li') :
            jobRes += li.get_text() + '\n'
        # 工作要求
        jobReq = ''
        for li in soup1.select('ul.squareli')[1].select('li') :
            jobReq += li.get_text() + '\n'
        # print jobReq
        # 将数据存入item中
        item['zname']=zname;
        item['detailLink'] = detailLink;
        item['ztype']=ztype
        item['znum'] = znum
        item['zlocal'] = zlocal
        item['ztime'] = ztime
        item['jobRes'] = jobRes
        item['jobReq'] = jobReq
        # 处理工作职责和工作要求
        items.append(item)

    origin = []
    print(len(items))

    # 以json格式输出到文件中
    # 禁用ascii编码，按utf-8编码
    output = open('tencent.json'+ str(num), 'w')
    for i in origin:
        items.append(i)
    line = json.dumps(items, ensure_ascii=False);
    # print line
    output.write(line.encode('utf-8'))
    output.close()

# print resHtml
for i in range(303):
    print("进行到第" + str(i) + "页")
    url = 'position.php?&start='+ str(i * 10) +'#a'
    tengxun(url, i)

取出来的json数据：