用 nodejs 再写一个爬虫，但是跑着跑着就报错了

Question

我的所有代码

var http = require('http');
var cheerio = require('cheerio');
var Promise = require('bluebird');
var baseURI = 'http://www.nuomi.com/pcindex/main/changecity';

var zzURL = 'http://zz.nuomi.com/690'


provinceF(baseURI)
    .then(function (allCityList) {
        // start(zzURL);
        return getHtml(allCityList)
    })
    .then(function (allCityArr) {
        return new Promise(function (resolve, reject) {
            Promise.all(allCityArr.map(function (href) {
                return start(href);
            })).then(function (values) {
                resolve(values);
            }).catch(function (err) {
                reject(err);
            });
        });
        // allCityArr.forEach(function(href){
        //     start(href)
        // })
    }).then(function (value) {
        //console.log(value);
    });

function start (url) {
    return new Promise(function (resolve, reject) {
        provinceF(url)
            .then(function (html) {
                return cityF(html);
            })
            .then(function (arr) {
                var hrefArr = arr.cityHrefArr;
                var txtArr = arr.cityTxtArr;
                Promise.all(hrefArr.map(function (href) {
                    return new Promise(function (resolve, reject) {
                        provinceF(href)
                            .then(function (html) {
                                return street(html)
                            }).then(function (value) {
                                resolve(value);
                            }).catch(function (err) {
                                reject(err);
                            });
                    });
                })).then(function (value) {
                    resolve(value);
                }).catch(function (err) {
                    reject(err);
                });
            }).catch(function (err) {
                reject(err);
            });
    });
}
//获取所有城市的街道信息
function street (html) {
    return new Promise(function (resolve, reject) {
        var $ = cheerio.load(html);

        var streetArr = [];
        var con = $('.district-sub-list-ab a');
        con.each(function (i) {
            var txt = $(this).children('span').text();
            streetArr.push(txt);
        });
        console.log(streetArr)
        resolve(streetArr);
    });
}


//获取所有城市的市
function cityF (html) {
    var $ = cheerio.load(html);

    var cityHrefArr = [];
    var cityTxtArr = []
    var con = $('.district-list-ab').eq(0).children('a');
    con.each(function (i) {
        var href = "http:" + $(this).attr('href');
        var txt = $(this).text();
        cityHrefArr.push(href);
        cityTxtArr.push(txt);
    });
    // return cityHrefArr;
    return {
        cityHrefArr: cityHrefArr,
        cityTxtArr: cityTxtArr
    }
}

//获取页面
function provinceF (url) {
    return new Promise(function (resolve, reject) {
        http.get(url, function (res) {
            var html = '';

            res.on('data', function (data) {
                html += data;
            })

            res.on('end', function () {
                resolve(html);
            })
        }).on('error', function (e) {
            reject(e);
        })
    })
}

//获取所有城市链接
function getHtml (html) {
    var $ = cheerio.load(html);
    var provinceArr = [];

    var cityEl = $('#J-city-list > li a');

    cityEl.each(function (i) {
        //console.log(i);
        var href = $(this).attr('href');
        provinceArr[i] = ("http:" + href + "/690");
        //provinceArr.push("http:" + href + "/690");
    });
    return provinceArr;
}

报错信息

Unhandled rejection Error: connect ECONNREFUSED 127.0.0.1:80
debuggability.js:868
    at Object._errnoException (util.js:1019:11)
    at _exceptionWithHostPort (util.js:1041:20)
    at TCPConnectWrap.afterConnect [as oncomplete] (net.js:1175:14)

Answer 1

有一个地方Promise的reject没有catch

Answer 2

我试了下程序是正常的，应该只是网络偶尔不稳定造成的，你用了promise但并没有对应全部catch error，捕获一下异常让程序不挂即可，也可以全局监听下：

process.on('unhandledRejection', (reason, p) => {
    console.log('Unhandled Rejection at:', p, 'reason:', reason);
});

Answer 3

对promise没有完整处理好，爬虫程序一般获取的内容会很大，网络跟解析都会花比较长的时间，建议你分段分批去写