package xiaohao.datacollection;

import com.xiaohao.mongo.MongoDBUtil;
import org.bson.Document;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.time.Duration;
import java.time.LocalDateTime;

/**
 *  爬取网站 列表页：http://www.zhongyoo.com/name/
 */
public class MedicineCountPageProcessor implements PageProcessor {
    @Override
    public void process(Page page) {

        //爬取列表页
        if (page.getUrl().toString().endsWith("name/")) {
            for (Selectable selectable : page.getHtml().$(".all").nodes()) {
                String name = selectable.xpath("a/text()").toString();
                String href = selectable.xpath("a/@href").toString();
                Document document = new Document();
                document.append("name", name).append("href", href);
                MongoDBUtil.insertDocument(document,new String[]{MongoDBUtil.DATABASE_DEMO,MongoDBUtil.COLLECTION_MEDICINE_TYPE});
                page.addTargetRequest(href);
            }
        } else {
            //爬取详细页
            String[] split = page.getUrl().toString().split("/");
            //第一级分类
            String firstTypeEnglishName = split[split.length - 1];
            String firstTypeName = page.getHtml().$(".listT1").xpath("b/h1/text()").toString();

            //页面数据处理部分
            page.getHtml().$(".jbLetCon").xpath("dl").nodes().forEach(data -> {
                //第二级分类
                String secondClassType = data.xpath("dt/a/@href").toString();
                String secondTypeUrl = secondClassType;
                if ("".equals(secondClassType)) {
                    secondTypeUrl = page.getUrl().toString();
                }
                String secondTypeName = data.xpath("dt/a/text()").toString();
                String finalSecondTypeUrl = secondTypeUrl;

                //中药归类信息
                data.xpath("dd/ul/li").nodes().forEach(li -> {
                    String url = li.xpath("a/@href").toString();
                    String name = li.xpath("a/text()").toString();
                    String[] hrefSplit = finalSecondTypeUrl.split("/");
                    Document document = new Document().append("name", name).append("href", url)
                            .append("firstTypeEnglishName", firstTypeEnglishName).append("firstTypeName",
                                    firstTypeName).append(
                                    "firstTypeUrl",
                                    page.getUrl().toString())
                            .append("secondTypeEnglishName", hrefSplit[hrefSplit.length - 1])
                            .append("secondTypeName", secondTypeName).append(
                                    "secondTypeUrl", finalSecondTypeUrl);
                    MongoDBUtil.insertDocument(document, new String[]{MongoDBUtil.DATABASE_DEMO,firstTypeEnglishName});

                });

            });
        }
    }

    @Override
    public Site getSite() {
        return new Site();
    }

    public static void main(String[] args) {
        LocalDateTime begin = LocalDateTime.now();
        Spider.create(new MedicineCountPageProcessor()).addUrl("http://www.zhongyoo.com/name/").thread(15).run();
        System.out.println("爬取用时："+Duration.between(begin,LocalDateTime.now()).toMinutes()+"分钟");
    }
}