web-spider/src/pageProcesser.js

web-spider is a simple and fast web spider written with Nodejs!

/**
 * Created by yuexing on 2016/6/17.
 * 用于分析页面
 * 1.提取链接
 * 2.转换编码
 * 3.分析是否重定向
 */

exports.analysisPage = (url,data)=> {
	this._simpleCache.set(url,data);
	var $ = cheerio.load(data.data.content);
	var list = [];
	$('a').each((k, obj)=>{
		list.push($(obj).attr("href"));
	});
};