网络爬虫
WebCollector是Java的爬虫框架,比起直接采用HttpClient、JSoup爬取有强大的好处,框架中集成了断点续爬、Url去重、自定义Http请求等。例如Nutch、Heritrix,底层实现都类似。
下面是俩种爬虫的实现:
1、Node爬虫
npm下载模块
var eventproxy = require('./lib/eventproxy');
var ep = new eventproxy();
var superagent = require('superagent');
var cheerio = require('cheerio');
var url = require('url');
var cnodeUrl = 'https://cnodejs.org/';
superagent.get(cnodeUrl).end(function(err,res){
if(err)
return console.error(err);
var topicUrls = [];
var $ = cheerio.load(res.text);
//获取首页所有链接
$('#topic_list .topic_title').each(function(idx,element){
var $element = $(element);
var href = url.resolve(cnodeUrl,$element.attr('href'));
topicUrls.push(href);
});
console.log(topicUrls);
// 命令 ep 重复监听 topicUrls.length 次(在这里也就是 40 次) `topic_html` 事件再行动
ep.after('topic_html', topicUrls.length, function (topics) {
// topics 是个数组,包含了 40 次 ep.emit('topic_html', pair) 中的那 40 个 pair
// 开始行动
topics = topics.map(function (topicPair) {
// 接下来都是 jquery 的用法了
var topicUrl = topicPair[0];
var topicHtml = topicPair[1];
var $ = cheerio.load(topicHtml);
return ({
title: $('.topic_full_title').text().trim(),
href: topicUrl,
comment1: $('.reply_content').eq(0).text().trim(),
});
});
console.log('final:');
console.log(topics);
});
topicUrls.forEach(function (topicUrl) {
superagent.get(topicUrl)
.end(function (err, res) {
console.log('fetch ' + topicUrl + ' successful');
ep.emit('topic_html', [topicUrl, res.text]);
});
});
});
//异步并发
ep.all('data1','data2',function(data1,data2){
console.log(data1+","+data2);
});
superagent.get(cnodeUrl).end(function(err,res){
ep.emit('data1',res.test);
});
superagent.get(cnodeUrl).end(function(err,res){
ep.emit('data2',res.test);
});
2、WebCollector
需要下载的Jar:
WebCollector,解压后将webcollector-2.32-bin中的jar放入项目中。
selenium(用于解析Html)。
下面是爬取新浪微博的代码:
import cn.edu.hfut.dmic.webcollector.model.CrawlDatum;
import cn.edu.hfut.dmic.webcollector.model.CrawlDatums;
import cn.edu.hfut.dmic.webcollector.model.Page;
import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import cn.edu.hfut.dmic.webcollector.net.HttpResponse;
import cn.edu.hfut.dmic.webcollector.plugin.berkeley.BreadthCrawler;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
/**
*
* 爬取微博
* @author Alex
*
*/
public class WeiboCrawler extends BreadthCrawler {
private String cookie;
public WeiboCrawler(String crawlPath, boolean autoParse) throws Exception {
super(crawlPath, autoParse);
cookie = WeiboCN.getSinaCookie("XXXXXXXXXXX", "XXXXXXXXXX");//账号、密码
}
@Override
public HttpResponse getResponse(CrawlDatum crawlDatum) throws Exception {
HttpRequest request = new HttpRequest(crawlDatum);
request.setCookie(cookie);
return request.getResponse();
}
public void visit(Page page, CrawlDatums next) {
int pageNum = Integer.valueOf(page.getMetaData("pageNum"));
Elements weibos = page.select("div.c");//或者Document doc = page.doc();
for (Element weibo : weibos) {
System.out.println("第" + pageNum + "页\t" + weibo.text());
}
}
public static void main(String[] args) throws Exception {
WeiboCrawler crawler = new WeiboCrawler("WeiboCrawler", false);
crawler.setThreads(3);//线程数
for (int i = 1; i <= 5; i++) {//爬取XXX前5页
crawler.addSeed(new CrawlDatum("http://weibo.cn/zhouhongyi?vt=4&page=" + i).putMetaData("pageNum", i + ""));
}
//crawlerNews.setResumable(true);//断点续爬
crawler.start(1);
}
}
import cn.edu.hfut.dmic.webcollector.net.HttpRequest;
import cn.edu.hfut.dmic.webcollector.net.HttpResponse;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.util.Set;
import javax.imageio.ImageIO;
import org.openqa.selenium.By;
import org.openqa.selenium.Cookie;
import org.openqa.selenium.WebElement;
import org.openqa.selenium.htmlunit.HtmlUnitDriver;
import java.awt.BorderLayout;
import java.awt.Container;
import java.awt.Dimension;
import java.awt.Graphics;
import java.awt.Toolkit;
import java.awt.event.ActionEvent;
import java.awt.event.ActionListener;
import java.awt.image.BufferedImage;
import javax.swing.JButton;
import javax.swing.JFrame;
import javax.swing.JPanel;
import javax.swing.JTextField;
/**
*
* @author Alex
*
*/
public class WeiboCN {
public static String getSinaCookie(String username, String password) throws Exception {
HtmlUnitDriver driver = new HtmlUnitDriver();//加载Html解析驱动
driver.setJavascriptEnabled(true);
driver.get("http://login.weibo.cn/login/");
WebElement ele = driver.findElementByCssSelector("img");//selenium选择器
String src = ele.getAttribute("src");
String cookie = concatCookie(driver);
HttpRequest request = new HttpRequest(src);//请求验证码
request.setCookie(cookie);
HttpResponse response = request.getResponse();
ByteArrayInputStream is = new ByteArrayInputStream(response.getContent());
BufferedImage img = ImageIO.read(is);
is.close();
ImageIO.write(img, "png", new File("result.png"));
String userInput = new CaptchaFrame(img).getUserInput();
//模拟表单登录
WebElement mobile = driver.findElementByCssSelector("input[name=mobile]");
mobile.sendKeys(username);
WebElement pass = driver.findElementByCssSelector("input[type=password]");
pass.sendKeys(password);
WebElement code = driver.findElementByCssSelector("input[name=code]");
code.sendKeys(userInput);
WebElement rem = driver.findElementByCssSelector("input[name=remember]");
rem.click();
WebElement submit = driver.findElementByCssSelector("input[name=submit]");
submit.click();
String result = concatCookie(driver);
driver.close();
if (result.contains("gsid_CTandWM")) {
return result;
} else {
throw new Exception("weibo login failed");
}
}
public static String concatCookie(HtmlUnitDriver driver) {
Set<Cookie> cookieSet = driver.manage().getCookies();
StringBuilder sb = new StringBuilder();
for (Cookie cookie : cookieSet) {
sb.append(cookie.getName() + "=" + cookie.getValue() + ";");
}
String result = sb.toString();
return result;
}
//根据图片生成窗体验证码
public static class CaptchaFrame {
JFrame frame;//窗口
JPanel panel;//面板
JTextField input;//输入框
int inputWidth = 100;
BufferedImage img;
String userInput = null;
public CaptchaFrame(BufferedImage img) {
this.img = img;
}
public String getUserInput() {
frame = new JFrame("输入验证码");
final int imgWidth = img.getWidth();
final int imgHeight = img.getHeight();
int width = imgWidth * 2 + inputWidth * 2;
int height = imgHeight * 2+50;
Dimension dim = Toolkit.getDefaultToolkit().getScreenSize();
int startx = (dim.width - width) / 2;
int starty = (dim.height - height) / 2;
frame.setBounds(startx, starty, width, height);
Container container = frame.getContentPane();
container.setLayout(new BorderLayout());
panel = new JPanel() {
@Override
public void paintComponent(Graphics g) {//将图片画在面板上
super.paintComponent(g);
g.drawImage(img, 0, 0, imgWidth * 2, imgHeight * 2, null);
}
};
panel.setLayout(null);
container.add(panel);
input = new JTextField(6);
input.setBounds(imgWidth * 2, 0, inputWidth, imgHeight * 2);
panel.add(input);
JButton btn = new JButton("登录");
btn.addActionListener(new ActionListener() {//注册监听
public void actionPerformed(ActionEvent e) {
userInput = input.getText().trim();
synchronized (CaptchaFrame.this) {//同步窗口释放
CaptchaFrame.this.notify();
}
}
});
btn.setBounds(imgWidth * 2 + inputWidth, 0, inputWidth, imgHeight * 2);
panel.add(btn);
frame.setVisible(true);
synchronized (this) {
try {
this.wait();
} catch (InterruptedException ex) {
ex.printStackTrace();
}
}
frame.dispose();
return userInput;
}
}
}
大家注意password!这个name="password_9384"
其中的数字是动态生成的,每次请求都会变,所以上面代码中的selenium选择器要用input[type=password]
。
先这样吧,不太会写文章,希望大家海涵。
**粗体** _斜体_ [链接](http://example.com) `代码` - 列表 > 引用
。你还可以使用@
来通知其他用户。