jsoup爬取文章内容



protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    // TODO Auto-generated method stub
    //response.getWriter().append("Served at: ").append(request.getContextPath());
    String agent1 = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36";

    int pageNum=1;
    int pageSize=899;
    //for(pageNum=1;pageNum<101;pageNum++)
    for(pageNum=1;pageNum<2;pageNum++)
    {
        try {
            int page1= 277;
            Map<Integer,String> map1 = ManageMySQL.getNewsLinkInTable(page1,pageSize,"data_szyjglj");
            for(Integer key : map1.keySet())
            {
                System.out.println(key+"  "+map1.get(key));
                String news_link = map1.get(key);
                String context1="";
                String source1="";
                //String context1 = getContentByURL(news_link).replace(" ", "");

                Document documentRoot = Jsoup.connect(news_link).userAgent(agent1).get();
                Elements elements1 = documentRoot.select("div.source span");
                if(elements1.size()==2)
                {
                    Element span_ele = elements1.get(0);
                    source1 = span_ele.text();
                }

                Elements elements2 = documentRoot.select("div.view_box");
                if(elements2.size()==1)
                {
                    Element div_ele = elements2.get(0);
                    context1 = div_ele.text();
                }

                ManageMySQL.updateContextAndPublishDate(key, context1.replace("'", "").replace("\"", ""),source1,"data_szyjglj");
            }
        } catch (Exception e) {
            // TODO Auto-generated catch block
            e.printStackTrace();
        }

    }
}