Python爬虫实战: 通用版豆瓣电影数据及图片的获取与入库,含防呆逻辑

由于最近需要准备一些数据,故开始练习使用胶水语言,经过一番探索终于完成了豆瓣电影信息的爬取,特此分享.

需要说明的是,我这里把电影信息提取之后,缓存了电影封面和演职人员的图片,并对图片信息进行了获取入库

先贴出我两种表结构:

1.电影表:

 其中data是存储电影信息的json数据,如下:
{"mActorRole": [{"name": "奥克塔维亚·斯宾瑟 ", "id": 1154263, "role": "暂无角色信息"},{"name": "约翰·浩克斯 ", "id": 1100849, "role": "暂无角色信息"},{"name": "凯蒂·洛茨 ", "id": 1232918, "role": "暂无角色信息"},{"name": "詹姆斯·拉夫尔提 ", "id": 1022640, "role": "暂无角色信息"},{"name": "小克利夫顿·克林斯 ", "id": 1019033, "role": "暂无角色信息"}], "mCoverId": 2499355591, "mDirector": [{"name": "Eshom Nelms ", "id": 1387484}], "mId": 26802500, "mLength": "92分钟", "mName": "小城犯罪 Small Town Crime", "mShowDate": "2017-03-11(美国)", "mSynopsis": "                                　　奥克塔维亚·斯宾瑟将携约翰·浩克斯出演独立惊悚片《小城犯罪》。电影讲述由浩克斯扮演的酗酒前警察偶然发现一具女尸，并不慎将他的家庭至于危险之中，他不得不一边寻找凶手，一边与恶势力作斗争。该片由内尔姆斯兄弟执导，目前正在拍摄中。                        ", "mType": "惊悚"}

2.图片表:

两个表,结构一致名字不同,id为电影表内演职人员的id.data为图片属性的json字符串,如下:
{"format": "jpg", "height": 383, "size": 20016, "width": 270}

脚本由以下方法调用执行:
需要注意的是 , 下图中的local_url是 豆瓣最新电影 这个页面缓存到本地后的html文件的路径 , 也可以不传 .

# coding=utf-8
import os
import sys

import cv2
import pymysql
import re
import json
import time
import requests
from bs4 import BeautifulSoup


def get_all_url(local_url, extra_url_list):
    """

    :param local_url:    电影列表采取动态解析 , 故需要下载解析完毕的数据 . 可为空
    :param extra_url_list:  有时候新出的电影凑不够一页,所以手动加在数组里传进来
    :return: 单条电影url的list
    """
    if local_url is not None:
        html_file = open(local_url, 'r', encoding='UTF-8')  # NOTE  只读打开 注意编码
        try:
            html_page = html_file.read()
        except IOError:
            print("--------------------本地html读取失败! ------------------------")
            return
        soup = BeautifulSoup(html_page, 'html.parser')
        new_list = soup.findAll('a', {'class': 'item'})
        print('本次预计进行更新的电影数据个数为: %d' % (len(new_list) + len(extra_url_list)))
        for movie in new_list:
            movie_url = movie.get('href')
            extra_url_list.append(movie_url)
    if len(extra_url_list) == 0:
        print("没有收到任何url , 请检查参数 !")
        sys.exit(0)
    command = int(input('本次即将进行更新的电影数据个数为 : %d 个 , 按<1>确认更新,按<0>取消更新.....\n' % len(extra_url_list)))
    i = 0
    while i < 2:
        if command == 1:
            return extra_url_list
        if command == 0:
            print("本次更新已取消 ! ")
            sys.exit(0)
        else:
            i += 1
            command = int(input("您的输入有误 , 还有%d次机会 ! 请再次输入..." % (3 - i)))
    print("检查一下你的输入法 ! 再见 ! ")
    sys.exit(0)


def frisk_image(folder, table):
    if not os.path.exists(folder):
        print("-------------------------- %s 不存在 , 请检查 !-------------------------------" % folder)
        return
    start_time = time.time()
    m_image = pymysql.Connect(
        host="数据库主机", port=3306,
        user="用户名", passwd="密码",
        db="图片表名", charset='utf8')  # NOTE  一定要设置编码
    image_cursor = m_image.cursor()
    count = 0
    for parent, dir_names, file_names in os.walk(folder):
        for file_name in file_names:
            full_name = os.path.join(parent, file_name)  # note  全路径名
            try:
                img = cv2.imread(full_name)
            except Exception as err:
                print("-------------------------------读取<%s>时发生错误 !------------------------------ " % full_name)
                print(err)
                continue
            shape = img.shape
            image_data = {"format": re.findall(r"[^.]+$", file_name)[0], "height": int(shape[0]),
                          "size": int(os.path.getsize(full_name)), "width": int(shape[1])}
            img_id = int(
                file_name.replace(re.findall(r"[^.]+$", file_name)[0], "").replace(".", "").replace("p",
                                                                                                    ""))
            json_data = json.dumps(image_data,
                                   sort_keys=True, ensure_ascii=False).replace("'", '"')  # NOTE  格式化
            image_cursor.execute("select count(*) from %s where id =%d" % (table, img_id))  # note 有记录则跳过
            if not image_cursor.fetchall()[0][0] == 0:
                print("----------------------图片库中已存在id为<%d>的数据 ! 已跳过---------------------" % img_id)
                continue
            sql = "INSERT INTO %s (id,data, create_time) VALUES (%d , '%s', %d)" % (
                table, img_id, json_data, round(time.time() * 1000))
            try:
                image_cursor.execute(sql)
                m_image.commit()
            except Exception as err:
                print(err)
                print("-------------------<%s>在写入数据库过程中出现问题 , 已回滚 !---------------------" % full_name)
                m_image.rollback()  # NOTE 出错则回滚
                continue
            print("恭喜! %s  已成功入库啦!" % file_name)
            count += 1
    m_image.close()  # TODO  别忘了这个!
    print("恭喜您入库完成 ! 本次入库%d条数据,用时%d秒" % (count, time.time() - start_time))


def get_movie_info(info_table, cover_table, avatar_table,
                   cover_folder, avatar_folder,
                   extra_url_list, local_url=None):
    start_time = time.time()
    url_list = get_all_url(local_url, extra_url_list)
    mask_tmp = pymysql.Connect(
       host="数据库主机", port=3306,
        user="用户名", passwd="密码",
        db="电影表名", charset='utf8')  # NOTE  一定要设置编码
    movie_cursor = mask_tmp.cursor()
    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.80 Safari/537.36'
    }
    count = 0
    for url in url_list:
        print(">>>>>>>>>>>>>>>>>>>>>>第" + str(count + 1) + "个开始<<<<<<<<<<<<<<<<<<<<<<<<<")
        data = requests.get(url, headers=headers).content
        row_id = 0
        soup = BeautifulSoup(data, 'html.parser')  # NOTE 实例化一个BeautifulSoup对象
        movie_data = {}
        try:
            m_id = re.findall(r"ect/(.+?)/", url)[0]
            if m_id.isdigit():  # NOTE  校验一下mId是否为数字
                movie_data['mId'] = int(m_id)  # NOTE 设置id为原站id
            else:
                continue
        except Exception as err:
            print(err)
            print( str(url))+">>>>>>>>>>>>>>>>>>>>>>>>>>>>>解析id出错 !"
            continue
        has_movie = movie_cursor.execute(
            "select id from " + str(info_table) + " where data like '%mId\":" + str(m_id) + ",%'")
        if not has_movie == 0:
            row_id = int(movie_cursor.fetchall()[0][0])
            print("发现重复电影>>>>>>>>>>>>>>>>>>mId =<%d>   row_id=<%d> ,  ! " % (int(m_id), row_id))
        try:
            if soup.find('span', {'property': 'v:itemreviewed'}) is None:  # NOTE  这个和下面的一样,如果标签找不到就跳过此项数据(或将此数据置默认值)
                continue
            else:
                movie_data['mName'] = soup.find('span', {'property': 'v:itemreviewed'}).getText()  # NOTE 设置电影名
            if soup.find('span', {'property': 'v:summary'}) is not None:
                movie_data['mSynopsis'] = soup.find('span', {'property': 'v:summary'}).getText().replace("\n",
                                                                                                         "").replace("\t",
                                                                                                                     "").replace(
                    "\v", "").replace("\r", "")  # NOTE 设置简介( 去掉格式符号)
            else:
                movie_data['mSynopsis'] = "暂无电影简介"
            if len(soup.findAll('li', {'class': 'celebrity'})) == 0:
                continue
            else:
                director__actor = soup.findAll('li', {'class': 'celebrity'})  # NOTE 演职人员数据都在一起
                directors = []
                for single_director in director__actor:
                    if single_director.find("span", {"title": "导演"}) is not None:
                        director = {}
                        if single_director.find("a", {"class": "name"}) is not None:
                            a = single_director.find("a", {"class": "name"})
                            director["name"] = a.getText()
                            director["id"] = int(re.findall(r"/celebrity/(.+?)/", a['href'])[0])
                            inner_all_url = re.findall(r"url\((.+?)\)",
                                                       single_director.find("div", {"class": "avatar"})["style"])
                            img_url = inner_all_url[len(inner_all_url) - 1]  # NOTE 图片url (部分人的图片不止一张,最后一张一定人头像)
                            if img_url.find("default") == -1:  # NOTE 初步发现默认图的路径包含 default  如果没有应该就是有图的
                                if not os.path.exists(avatar_folder):
                                    os.makedirs(avatar_folder)
                                try:
                                    director_pic = requests.get(img_url, timeout=15)
                                    file_name = "%s/%s" % (avatar_folder,
                                                           str(director["id"]) + "." + re.findall(r"[^.]+$", img_url)[
                                                               0])  # NOTE  数值id拼接格式,格式为最后一个.后的字符串   图片id为导演id
                                    fp = open(file_name, 'wb')
                                    fp.write(director_pic.content)
                                    fp.close()
                                    print("导演图片 : %s 成功保存到本地 ! " % file_name)
                                except Exception as err:
                                    print(err)
                                    print(" %s>>>>>>>>>>>>>>>>>在处理导演图片时出错!" % url)
                            directors.append(director)
                movie_data["mDirector"] = [','.join(str(i) for i in directors)]  # NOTE 导演

                actors = []
                for single_actor in director__actor:
                    has_role = True
                    li = single_actor.find("span", {"class", "role"})
                    if li is None:  # NOTE  没有角色
                        has_role = False
                    else:
                        if li.getText().find("饰") == -1:  # NOTE title与文本不同  肯定不是演员
                            continue
                    act = {}
                    a = single_actor.find("a", {"class": "name"})
                    if a is None:
                        continue
                    act["name"] = a.getText()
                    act["id"] = int(re.findall(r"/celebrity/(.+?)/", a['href'])[0])
                    act["role"] = single_actor.find("span", {"class", "role"}).getText().replace("饰 ",
                                                                                                 "") if has_role else "暂无角色信息"
                    inner_all_url = re.findall(r"url\((.+?)\)",
                                               single_actor.find("div", {"class": "avatar"})["style"])
                    img_url = inner_all_url[len(inner_all_url) - 1]  # NOTE 图片url (部分人的图片不止一张,最后一张一定人头像)
                    if img_url.find("default") == -1:  # NOTE 初步发现默认图的路径包含 default  如果没有应该就是有图的
                        if not os.path.exists(avatar_folder):
                            os.makedirs(avatar_folder)
                        try:
                            actor_pic = requests.get(img_url, timeout=15)
                            file_name = "%s/%s" % (avatar_folder,
                                                   str(act["id"]) + "." + re.findall(r"[^.]+$", img_url)[
                                                       0])  # NOTE  数值id拼接格式,格式为最后一个.后的字符串  图片id 为演员id
                            fp = open(file_name, 'wb')
                            fp.write(actor_pic.content)
                            fp.close()
                            print("演员图片 : %s 成功保存到本地 ! " % file_name)
                        except Exception as err:
                            print(err)
                            print("<%s>>>>>>>>>>>>>>>>>>>>> 在处理演员图片时出错!" % url)
                    actors.append(act)
                movie_data["mActorRole"] = [','.join(str(i) for i in actors)]  # NOTE 演员
            if len(soup.findAll('span', {'property': 'v:genre'})) == 0:
                continue
            else:
                m_type = soup.findAll('span', {'property': 'v:genre'})
                types = []
                for single_type in m_type:
                    types.append(single_type.getText())
                movie_data["mType"] = ' / '.join(str(i) for i in types)  # NOTE 设置类型 ,  结果格式为 : 爱情 / 动作 / 伦理
            if len(soup.findAll('span', {'property': 'v:initialReleaseDate'})) == 0:
                continue
            else:
                show_date_list = soup.findAll('span', {'property': 'v:initialReleaseDate'})
                show_date = []
                for single_date in show_date_list:
                    show_date.append(single_date.getText())
                movie_data["mShowDate"] = ' / '.join(str(i) for i in show_date)

            movie_data["mLength"] = soup.find('span', {'property': 'v:runtime'}).getText() \
                if soup.find('span', {'property': 'v:runtime'}) is not None else "暂无时长"  # NOTE 电影时长

            img_url = soup.find("img", {"title": "点击看更多海报"})["src"]
            if img_url.find("default") == -1:  # NOTE 初步发现默认图的路径包含 default  如果没有应该就是有图的
                img_name = re.findall(r"lic/.*", img_url)[0].replace("lic/", "")
                if not os.path.exists(cover_folder):
                    os.makedirs(cover_folder)
                try:
                    cover = requests.get(img_url, timeout=15)
                    file_name = "%s/%s" % (cover_folder, img_name)
                    fp = open(file_name, 'wb')
                    fp.write(cover.content)
                    fp.close()
                    print("封面图片 : %s 成功保存到本地 ! " % file_name)
                except Exception as err:
                    print(err)
                    print("url为----> %s 的电影在存储封面时出错!" % url)
                    movie_data["mCoverId"] = 0
                movie_data["mCoverId"] = int(re.findall(r"ic/p(.+?)\.", img_name)[0])
            else:
                movie_data["mCoverId"] = 0  # NOTE 没封面的电影图片指向0
            json_data = json.dumps(movie_data, sort_keys=True, ensure_ascii=False).replace("'", '"').replace('["',
                                                                                                             '[').replace(
                '"]', ']')  # NOTE  格式化输出电影信息
        except Exception as err:
            print(err)
            print("--------------------------------解析网页失败 %s失败 ----------------------------------" % url)
            continue
        if not row_id == 0:
            sql = "update %s set data = '%s' where id = %d " % (info_table, json_data, row_id)
            count -= 1
        else:
            movie_cursor.execute("select id from %s order by id desc limit 1 " % info_table)
            last_id = movie_cursor.fetchall()[0][0]  # NOTE  获取最后一条数据的id
            sql = "INSERT INTO %s (id,data, createTime) VALUES (%d , '%s', %d)" % (
                info_table, last_id + 1, json_data, round(time.time() * 1000))
        try:
            movie_cursor.execute(sql)
            mask_tmp.commit()
        except Exception as err:
            print(err)
            print("---------------------------%s在写入数据库过程中出现问题 , 已回滚 ! -------------------------------" % url)
            mask_tmp.rollback()  # NOTE 出错则回滚
            continue
        print("恭喜! 《%s》已成功入库啦!" % url)
        count += 1
    mask_tmp.close()  # TODO  别忘了这个!
    print("恭喜您入库完成 ! 本次新入库%d条数据,用时%d秒" % (count, time.time() - start_time))
    print("------------------------华丽的分割线-----------------------------")
    print("即将扫描生成的图片.............................................................................................倒计时5")
    print("即将扫描生成的图片.............................................................................................倒计时4")
    print("即将扫描生成的图片.............................................................................................倒计时3")
    print("即将扫描生成的图片.............................................................................................倒计时2")
    print("即将扫描生成的图片.............................................................................................倒计时1")
    print("开始扫描生成的图片!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")
    frisk_image(cover_folder, cover_table)   #调用图片处理
    frisk_image(avatar_folder, avatar_table) #调用图片处理

经实测,效果显著,百试不爽几乎完美适配所有豆瓣电影!!图书与音乐类似,在此不再赘述!感谢阅读!