【mysql数据库】通过python 3.7 爬虫获取重庆智博会上重庆本地的参会企业，并记录到mysql数据库中

1、爬虫思路参考这篇博文
https://blog.****.net/qq_39295735/article/details/83539853

2、数据库保存结果如下(篇幅有限，仅部分截图)

3、python代码如下

#!/usr/bin/python3
#-*- coding: utf-8 -*-

import urllib.request
import re
import pymysql

#打开数据库连接
db = pymysql.connect("139.159.205.119","root","[email protected]","enterprise")
cursor = db.cursor()
cursor.execute("select version()")
data = cursor.fetchone()
print("Database version:  %s" % data)

#将企业名字写入到数据表company_list中
def writeCompanyToDB(companyName, source):
    sql="insert into company_list(company, source) values('%s', '%s')" % (companyName, source)
    cursor = db.cursor()
    try:       
        cursor.execute(sql)
        db.commit()
        print("Info: 记录公司 %s 成功" % companyName)
    except Exception as err:
        db.rollback()
        print("Error: 记录公司 %s 失败" % companyName)
        print(err)

#获取智博会的名单
def getCompanysFromWWW():
    companyList = []

    #浏览器伪装池，将爬虫伪装成浏览器，避免被网站屏蔽
    headers=("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/38.0.2125.122 Safari/537.36 SE 2.X MetaSr 1.0")
    opener = urllib.request.build_opener()
    opener.addheaders = [headers]
    urllib.request.install_opener(opener)
    
    #重庆智博会官网商参展商的页面数从1到28
    urlPrefix        = "https://www.smartchina-expo.cn/exhibition/roster-page-"
    urlPostfix       = ".html"
    pageStartIdx = 1
    pageEndIdx   = 28

    #获取每一页的企业名单
    for pageIdx in range(pageStartIdx, pageEndIdx):
        visitURL   = urlPrefix + str(pageIdx) + urlPostfix
        webData  =  urllib.request.urlopen(visitURL).read().decode("utf-8", "ignore")
        matchPat ='<p class="zsmcc-org">(.*?)</p>'
        companys = re.compile(matchPat, re.S).findall(webData)

        #将本页中所有包含'重庆'并且名字完整的公司名添加到列表变量中                                
        for nameIdx in range(0, len(companys)):
            company = companys[nameIdx]
            condition1 = company.find('重庆', 0, len(company))
            condition2 = company.find('公司', 0, len(company))
            condition3 = company.find('...', 0, len(company))
            if condition1 >= 0 and condition2  >= 0 and condition3 < 0:
                companyList.append(companys[nameIdx])

    return companyList

#获取智博会上的公司名称
companyList = getCompanysFromWWW()

#将公司名写入数据库中，注明来源是智博会官网
print(len(companyList))
for idx in range(0, len(companyList)):
    writeCompanyToDB(companyList[idx], '重庆智博会参展商')

#关闭数据库
cursor.close()
db.close()