用requests写的采集百度图片到本地，求改写规范化

Question

今天看requests文档，想试试下载图片的功能。单一张图片，下载成功了，对于新手来说，已经很值得高兴了。后来一琢磨，既然咱学了正则，为啥不用requests和正则多抓几张图片试试？都说学代码要勤动手，那就练习练习试试吧。这是我第一个自己写的完整python代码，用最笨的办法，共计耗时3个小时，完成了我一个自创代码，新鲜刚出炉，通篇没有一个函数（因为不会用。。。。。）求群友给改写优化一下。谢谢

#-*- coding:utf-8 -*-
import requests
from contextlib import closing
import re
word=input('你想搜啥词？').strip()
ye=int(input('你想下载几页？'))
sites=[]
shu=0
offset=0
i=0
start = 'http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word='+word+'&pn='+str(offset)+'&gsm=3c&ct=&ic=0&lm=-1&width=0&height=0'
if ye==1:
    sites.append(start)
elif ye==0:
    print('别闹')
elif ye>1:
    while shu<ye:
        sites.append('http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word='+word+'&pn='+str(offset)+'&gsm=3c&ct=&ic=0&lm=-1&width=0&height=0')
        offset+=20
        shu+=1
print(sites)

while len(sites)>0:
    search_url=sites.pop()
    headers = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
    html=requests.get(search_url).text
    pic_url=re.findall(r'objURL":"(.*?)"',html,re.S)
    print(pic_url)
    for surl in pic_url:
        strings='bdpics\\'+str(i)+'.jpg'
        try:
            with closing(requests.get(surl,headers=headers,stream=True,timeout=20)) as ok:
                with open(strings,'wb') as fd:
                    for chunk in ok.iter_content(128):
                        fd.write(chunk)
        except:
            print('这张图片链接失效')
            continue
        i+=1

Answer 1

代码

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import os
import re

import requests


class Downloader(object):
    @staticmethod
    def down_page(url):
        html = requests.get(url).text
        return html

    @staticmethod
    def down_pic(url):
        headers = {'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
        return requests.get(url, headers=headers, stream=True, timeout=20).content


class UrlManager(object):
    def __init__(self):
        self.page_urls = set()

    def gen_urls(self, word, pages):
        for page in xrange(pages):
            url = self.__gen_url__(word, page)
            self.page_urls.add(url)

    def __gen_url__(self, word, page):
        return 'http://image.baidu.com/search/flip?tn=baiduimage&ie=utf-8&word=%s&pn=%s&gsm=3c&ct=&ic=0&lm=-1&width=0&height=0' % (
            word, page * 20)

    def has_new_url(self):
        return len(self.page_urls) > 0

    def get_craw_url(self):
        return self.page_urls.pop()


class Parser(object):
    def parse(self, html):
        return re.findall(r'objURL":"(.*?)"', html, re.S)


class Output(object):
    def __init__(self):
        self.result_dir = 'bdpics/'
        self.__check_dir__(self.result_dir)

    def save(self, file_name, pic_content):
        with open(os.path.join(self.result_dir, file_name), 'wb') as result_file:
            result_file.write(pic_content)

    def __check_dir__(self, dir):
        if not os.path.exists(dir):
            os.makedirs(dir)

    def show(self):
        print "result dir: %s" % os.path.abspath(self.result_dir)


class Spider_Main(object):
    def __init__(self):
        self.urls = UrlManager()
        self.downloader = Downloader()
        self.parser = Parser()
        self.output = Output()

    def crawl(self, word, pages):
        self.__check_params__(word, pages)
        self.urls.gen_urls(word, pages)

        count = 0
        while self.urls.has_new_url():
            page_url = self.urls.get_craw_url()
            html = self.downloader.down_page(page_url)
            pic_urls = self.parser.parse(html)
            for pic_url in pic_urls:
                try:
                    count += 1
                    file_name = '%s_%s' % (count, pic_url[pic_url.rindex("/") + 1:])

                    pic_content = self.downloader.down_pic(pic_url)
                    self.output.save(file_name, pic_content)
                    print "crawl %s url=%s" % (count, pic_url)
                except Exception, e:
                    print "failed %s url=%s, e=%s" % (count, pic_url, e.message)

        self.output.show()

    @staticmethod
    def __check_params__(word, pages):
        if word is None:
            raise Exception('关键字不能为空')
        if pages <= 0:
            raise Exception('页数必须大于0')


if __name__ == '__main__':
    spider_main = Spider_Main()
    spider_main.crawl('狗', 2)

结果

狗

美女（?）