public class JDEngine {

	public static void main(String[] args) {
		String url = "https://www.jd.com/allSort.aspx";
		String classpath = "cn.succy.geccospider";
		HttpRequest request = new HttpGetRequest(url);
		request.setCharset("gb2312");
		// 如果pipeline和htmlbean不在同一个包，classpath就要设置到他们的共同父包
		// 本引擎主要是把京东分类的页面手机板块给抓取过来封装成htmlbean
		GeccoEngine.create().classpath(classpath).start(request).interval(2000).run();

		// 本引擎是负责抓取每一个细目对应的页面的第一页的所有商品列表的数据，开启5个线程同时抓取，提升效率
		GeccoEngine.create().classpath(classpath).start(AllSortPipeline.cateRequests).thread(5)
				.interval(2000).run();
	}
}

@Gecco(matchUrl = "https://www.jd.com/allSort.aspx", pipelines = { "allSortPipeline",
		"consolePipeline" })
public class AllSort implements HtmlBean {
    private static final long serialVersionUID = 3422937382621558860L;

	@Request
	private HttpRequest request;

	/**
	 * 抓取手机模块的数据
	 */
	@HtmlField(cssPath = "div.category-items > div:nth-child(1) > div:nth-child(2) > div.mc > div.items > dl")
	private List<Category> cellPhone;

	public HttpRequest getRequest() {
		return request;
	}

	public void setRequest(HttpRequest request) {
		this.request = request;
	}

	public List<Category> getCellPhone() {
		return cellPhone;
	}

	public void setCellPhone(List<Category> cellPhone) {
		this.cellPhone = cellPhone;
	}
}

public class Category implements HtmlBean {

	private static final long serialVersionUID = -1808704248579938878L;

	/**
	 * 对应的是大的分类名字，如手机通讯，运营商，手机配件等
	 */
	@Text
	@HtmlField(cssPath = "dt > a")
	private String typeName;

	/**
	 * 相对于上面的大的分类下的小类目名字
	 */
	@HtmlField(cssPath = "dd > a")
	private List<HrefBean> categories;

	public String getTypeName() {
		return typeName;
	}

	public void setTypeName(String typeName) {
		this.typeName = typeName;
	}

	public List<HrefBean> getCategories() {
		return categories;
	}

	public void setCategories(List<HrefBean> categories) {
		this.categories = categories;
	}
}

@HtmlField(cssPath = "div.category-items > div:nth-child(1) > div:nth-child(2) > div.mc > div.items > dl")
private List<Category> cellPhone;

@PipelineName("allSortPipeline")
public class AllSortPipeline implements Pipeline<AllSort> {

	public static List<HttpRequest> cateRequests = new ArrayList<>();

	@Override
	public void process(AllSort allSort) {
		List<Category> cellPhones = allSort.getCellPhone();
		for (Category category : cellPhones) {
			// 获取mongo的集合
			MongoCollection<Document> collection = MongoUtils.getCollection();
			// 把json转成Document
			Document doc = Document.parse(JSON.toJSONString(category));
			// 向集合里边插入一条文档
			collection.insertOne(doc);
			List<HrefBean> hrefs = category.getCategories();
			// 遍历HrefBean，取出里边保存的url
			for (HrefBean href : hrefs) {
				HttpRequest request = allSort.getRequest();
				// 把url保存起来，方便后面开启一个新的引擎进行多线程抓取数据
				cateRequests.add(request.subRequest(href.getUrl()));
			}
		}
	}
}

@Gecco(matchUrl = "https://list.jd.com/list.html?cat={cat}", pipelines = { "consolePipeline",
		"filePipeline" ,"mongoPipeline"})
public class ProductList implements HtmlBean {

	private static final long serialVersionUID = -6580138290566056728L;

	/**
	 * 获取请求对象，从该对象中可以获取抓取的是哪个url
	 */
	 @Request
	 private HttpRequest request;

	// #plist > ul > li.gl-item > div.j-sku-item
	@HtmlField(cssPath = "#plist > ul > li.gl-item")
	private List<ProductDetail> details;

	 public HttpRequest getRequest() {
	 return request;
	 }
	
	 public void setRequest(HttpRequest request) {
	 this.request = request;
	 }

	public List<ProductDetail> getDetails() {
		return details;
	}

	public void setDetails(List<ProductDetail> details) {
		this.details = details;
	}

}

@PipelineName("mongoPipeline")
public class MongoPipeline implements Pipeline<ProductList> {

	@Override
	public void process(ProductList productList) {
		MongoCollection<Document> collection = MongoUtils.getCollection();
		HttpRequest req = productList.getRequest();
		// 从productList里边获取url，目的是为了从之前存进数据库中找到对应url的小类目
		String url = req.getUrl();
		// 把类目名对应的商品详情的列表获取，例如，手机对应到的页面的60条记录
		List<ProductDetail> details = productList.getDetails();
		// 转成json字符串
		String jsonString = JSON.toJSONString(details);
		// 通过url找到数组里边对应url的类目，然后添加一个字段叫做details，并且把details的值
		// 给添加进去
		collection.updateOne(new Document("categories.url", url),
				Document.parse("{\"$set\":{\"categories.$.details\":" + jsonString + "}}"));
	}
}

public class ProductDetail implements HtmlBean {

	private static final long serialVersionUID = -6362237918542798717L;

	@Attr(value = "data-sku")
	@HtmlField(cssPath = "div.j-sku-item")
	private String pCode;

	@Image({ "data-lazy-img", "src" })
	@HtmlField(cssPath = "div.j-sku-item > div.p-img > a > img")
	private String pImg;

	
	//#plist > ul > li:nth-child(1) > div > div.p-price > strong:nth-child(1) > i
	@Text
	@HtmlField(cssPath = "div.j-sku-item > div.p-price > strong:nth-child(1) > i")
	private String pPrice;

	@Text
	@HtmlField(cssPath = "div.j-sku-item > div.p-name > a > em")
	private String pTitle;

	@Text
	@HtmlField(cssPath = "div.j-sku-item > div.p-comment > strong > a.comment")
	private String pComment;

	@Text
	@HtmlField(cssPath = "div.j-sku-item > div.p-shop > span > a")
	private String pShop;

	@Text
	@HtmlField(cssPath = "div.j-sku-item > div.p-icons > *")
	private List<String> pIcons;

	public String getpCode() {
		return pCode;
	}

	public void setpCode(String pCode) {
		this.pCode = pCode;
	}

	public String getpImg() {
		return pImg;
	}

	public void setpImg(String pImg) {
		this.pImg = pImg;
	}

	public String getpPrice() {
		return pPrice;
	}

	public void setpPrice(String pPrice) {
		this.pPrice = pPrice;
	}

	public String getpTitle() {
		return pTitle;
	}

	public void setpTitle(String pTitle) {
		this.pTitle = pTitle;
	}

	public String getpComment() {
		return pComment;
	}

	public void setpComment(String pComment) {
		this.pComment = pComment;
	}

	public String getpShop() {
		return pShop;
	}

	public void setpShop(String pShop) {
		this.pShop = pShop;
	}

	public List<String> getpIcons() {
		return pIcons;
	}

	public void setpIcons(List<String> pIcons) {
		this.pIcons = pIcons;
	}

}