spring boot 爬虫 需要使用框架吗

一、新建springboot项目

 二、设置es版本跟本地一样

<properties>
        <java.version>1.8</java.version>
        <elasticsearch.version>7.12.1</elasticsearch.version>
    </properties>

引入fastjson

<dependency>
            <groupId>com.alibaba</groupId>
            <artifactId>fastjson</artifactId>
            <version>1.2.62</version>
        </dependency>

删除多余的文件 右键-delete

三、添加狂神的前端资料并设置application.properties

四、创建访问controller

package com.xx.es.jdreptile.controller;

import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;

@Controller
public class IndexController {
    @GetMapping({"/","/index"})
    public String get(){
        return "index";
    }
}

五、启动项目并访问

六、爬虫问题

数据问题：数据库中获取，消息队列中获取，都可以成为数据源，爬虫

七、怎么爬取数据？（获取请求返回的页面信息，筛选出我们想要的数据就可以了！）

Java中使用jsoup包就可以实现

首先引入jsoup的包

<!-- 解析网页 jsoup tika包可以pa电影或者音乐-->
        <dependency>
            <groupId>org.jsoup</groupId>
            <artifactId>jsoup</artifactId>
            <version>1.13.1</version>
        </dependency>

然后创建util

import com.luoyu.es.jdreptile.pojo.Content;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Component;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
@Component
public class HtmlParseUtil {

    public static void main(String[] args) throws Exception {
        //中文可以在链接后面加&enc=utf-8
//        new HtmlParseUtil().parseJd("心理学").stream().forEach(System.out::println);
        //获取请求
        //前提，需要联网， ajax不能获取到，模拟浏览器才能获取到
        String url = "https://search.jd.com/Search?keyword=java";
        /**
         * 解析网页(Jsoup返回Document就是浏览器Document对象)
         * 这里最开始得到的是跳转登录的页面，后来我在网页上登陆了，还是不行，看了弹幕，把
         * jsoup升级到最新版本（1.31.1）就好了
         */
        Document document = Jsoup.parse(new URL(url), 30000);
        //所有你在js中使用的方法这里都能用
        Element element = document.getElementById("J_goodsList");
        System.out.println(element.html());
        //获取所有的li元素
        Elements elements = element.getElementsByTag("li");
        //获取元素中的内容，这里el就是每一个li标签了
        for (Element el:elements){
            //关于这种图片，特别多的网站都是延迟加载的
            String img = el.getElementsByTag("img").eq(0).attr("data-lazy-img");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();
            System.out.println("======================================");
            System.out.println(img);
            System.out.println(price);
            System.out.println(title);
        }
    }

    public List<Content> parseJd(String keyword) throws Exception {
        String url = "https://search.jd.com/Search?keyword="+keyword;
        Document document = Jsoup.parse(new URL(url), 30000);
        Element element = document.getElementById("J_goodsList");
        System.out.println(element.html());
        Elements elements = element.getElementsByTag("li");
        ArrayList<Content> contents = new ArrayList<>();
        for (Element el:elements){
            //关于这种图片，特别多的网站都是延迟加载的
            String img = el.getElementsByTag("img").eq(0).attr("data-lazy-img");
            String price = el.getElementsByClass("p-price").eq(0).text();
            String title = el.getElementsByClass("p-name").eq(0).text();

            contents.add(Content.builder().img(img).price(price).title(title).build());
        }
        return contents;
    }
}

构建service

import java.io.IOException;
import java.util.List;
import java.util.Map;

//业务编写
public interface ContentService {
    //1.解析数据放入es索引中
    Boolean parseContent(String keywords) throws Exception;

    //获取这些数据实现搜索功能
    List<Map<String,Object>> searchPage(String keyword, Integer pageNo, Integer pageSize) throws IOException;

    //获取这些数据实现搜索高亮功能
    List<Map<String,Object>> searchPagehighlight(String keyword, Integer pageNo, Integer pageSize) throws IOException;
}

service实现类

import com.alibaba.fastjson.JSON;
import com.luoyu.es.jdreptile.pojo.Content;
import com.luoyu.es.jdreptile.utils.HtmlParseUtil;
import org.apache.lucene.util.QueryBuilder;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.FuzzyQueryBuilder;
import org.elasticsearch.index.query.MatchPhraseQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightField;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.AnnotationConfigApplicationContext;
import org.springframework.stereotype.Service;

import javax.swing.text.Highlighter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;

@Service
public class ContentServiceImpl implements ContentService {
    @Autowired
    private RestHighLevelClient restHighLevelClient;

    public static void main(String[] args) throws Exception {
        AnnotationConfigApplicationContext annotationConfigApplicationContext = new AnnotationConfigApplicationContext("com.luoyu.es.jdreptile");
        ContentServiceImpl bean = (ContentServiceImpl)annotationConfigApplicationContext.getBean("contentServiceImpl");
        Boolean result = bean.parseContent("java");
        annotationConfigApplicationContext.destroy();
        System.out.println(result);
    }

    @Override
    public Boolean parseContent(String keywords) throws Exception {
        List<Content> contents = HtmlParseUtil.parseJd(keywords);
        //把查询数据放入es中
        BulkRequest bulkRequest = new BulkRequest();
        bulkRequest.timeout("2m");
        for (int i = 0; i < contents.size(); i++) {
            bulkRequest.add(new IndexRequest("jd_goods").source(JSON.toJSONString(contents.get(i)), XContentType.JSON));
        }
        BulkResponse bulk = restHighLevelClient.bulk(bulkRequest, RequestOptions.DEFAULT);
        return !bulk.hasFailures();
    }

    @Override
    public List<Map<String, Object>> searchPage(String keyword, Integer pageNo, Integer pageSize) throws IOException {
        if(pageNo<1)
            pageNo=1;
        //条件搜索
        SearchRequest searchRequest = new SearchRequest("jd_goods");
        SearchSourceBuilder sourceBuilder = getSourceBuilder(keyword, pageNo, pageSize);
        searchRequest.source(sourceBuilder);
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        ArrayList<Map<String,Object>> list = new ArrayList<>();
        for (SearchHit documentFields : searchResponse.getHits().getHits()) {
             list.add(documentFields.getSourceAsMap());
        }
        return list;
    }

    private SearchSourceBuilder getSourceBuilder(String keyword, Integer pageNo, Integer pageSize) {
        SearchSourceBuilder sourceBuilder =new SearchSourceBuilder();
        //使用TermQueryBuilder没查出来数据，换成MatchPhraseQueryBuilder就好了
        //精准匹配
        MatchPhraseQueryBuilder termQueryBuilder= QueryBuilders.matchPhraseQuery("title",keyword);
        sourceBuilder.query(termQueryBuilder);
        sourceBuilder.timeout(new TimeValue(60, TimeUnit.SECONDS));
        //分页
        sourceBuilder.from(pageNo);
        sourceBuilder.size(pageSize);
        return sourceBuilder;
    }

    @Override
    public List<Map<String, Object>> searchPagehighlight(String keyword, Integer pageNo, Integer pageSize) throws IOException {
        if(pageNo<1)
            pageNo=1;
        //条件搜索
        SearchRequest searchRequest = new SearchRequest("jd_goods");
        SearchSourceBuilder sourceBuilder = getSourceBuilder(keyword, pageNo, pageSize);

        //高亮
        HighlightBuilder highlightBuilder = new HighlightBuilder();
        highlightBuilder.field("title");
        highlightBuilder.preTags("<span style='color:red'>");
        highlightBuilder.postTags("</span>");
//        highlightBuilder.requireFieldMatch(false);
        sourceBuilder.highlighter(highlightBuilder);

        //执行搜索
        searchRequest.source(sourceBuilder);
        SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        //解析结果
        ArrayList<Map<String,Object>> list = new ArrayList<>();
        for (SearchHit documentFields : searchResponse.getHits().getHits()) {
            //解析高亮的字段
            Map<String, HighlightField> highlightFields = documentFields.getHighlightFields();
            HighlightField title = highlightFields.get("title");
            Map<String, Object> sourceAsMap = documentFields.getSourceAsMap(); //原来的结果
            if(title!=null){
                Text[] fragments = title.getFragments();
                StringBuilder n_title=new StringBuilder();
                for (Text text : fragments) {
                    n_title.append(text);
                }
                sourceAsMap.put("title",n_title.toString());
            }
            list.add(sourceAsMap);
        }
        return list;
    }
}

构建controller

import com.luoyu.es.jdreptile.service.ContentService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RestController;

import java.util.List;
import java.util.Map;

@RestController
public class ContentController {
    @Autowired
    private ContentService contentService;
    @GetMapping("/parse/{keyword}")
    public Boolean parse(@PathVariable String keyword) throws Exception {
        return contentService.parseContent(keyword);
    }
    @GetMapping("/search/{keyword}/{pageNo}/{pageSize}")
    public List<Map<String, Object>> search(@PathVariable String keyword, @PathVariable Integer pageNo, @PathVariable Integer pageSize) throws Exception {
        return contentService.searchPagehighlight(keyword,pageNo,pageSize);
    }
}

index

import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.GetMapping;

@Controller
public class IndexController {
    @GetMapping({"/","/index"})
    public String get(){
        return "index";
    }
}

application.properties

server.port=8090
#关闭thymeleaf缓存
spring.thymeleaf.cache=false

index.html

<!DOCTYPE html>
<html xmlns:th="http://www.thymeleaf.org">

<head>
    <meta charset="utf-8"/>
    <title>狂神说Java-ES仿京东实战</title>
    <link rel="stylesheet" th:href="@{/css/style.css}"/>
</head>

<body class="pg">
<div class="page" id="app">
    <div id="mallPage" class=" mallist tmall- page-not-market ">

        <!-- 头部搜索 -->
        <div id="header" class=" header-list-app">
            <div class="headerLayout">
                <div class="headerCon ">
                    <!-- Logo-->
                    <h1 id="mallLogo">
                        <img th:src="@{/images/jdlogo.png}" alt="">
                    </h1>

                    <div class="header-extra">

                        <!--搜索-->
                        <div id="mallSearch" class="mall-search">
                            <form name="searchTop" class="mallSearch-form clearfix">
                                <fieldset>
                                    <legend>天猫搜索</legend>
                                    <div class="mallSearch-input clearfix">
                                        <div class="s-combobox" id="s-combobox-685">
                                            <div class="s-combobox-input-wrap">
                                                <input v-model="keyword" type="text" autocomplete="off" value="dd" id="mq"
                                                       class="s-combobox-input" aria-haspopup="true">
                                            </div>
                                        </div>
                                        <button type="submit" @click.prevent="searchKey" id="searchbtn">搜索</button>
                                    </div>
                                </fieldset>
                            </form>
                            <ul class="relKeyTop">
                                <li><a>狂神说Java</a></li>
                                <li><a>狂神说前端</a></li>
                                <li><a>狂神说Linux</a></li>
                                <li><a>狂神说大数据</a></li>
                                <li><a>狂神聊理财</a></li>
                            </ul>
                        </div>
                    </div>
                </div>
            </div>
        </div>

        <!-- 商品详情页面 -->
        <div id="content">
            <div class="main">
                <!-- 品牌分类 -->
                <form class="navAttrsForm">
                    <div class="attrs j_NavAttrs" style="display:block">
                        <div class="brandAttr j_nav_brand">
                            <div class="j_Brand attr">
                                <div class="attrKey">
                                    品牌
                                </div>
                                <div class="attrValues">
                                    <ul class="av-collapse row-2">
                                        <li><a href="#"> 狂神说 </a></li>
                                        <li><a href="#"> Java </a></li>
                                    </ul>
                                </div>
                            </div>
                        </div>
                    </div>
                </form>

                <!-- 排序规则 -->
                <div class="filter clearfix">
                    <a class="fSort fSort-cur">综合<i class="f-ico-arrow-d"></i></a>
                    <a class="fSort">人气<i class="f-ico-arrow-d"></i></a>
                    <a class="fSort">新品<i class="f-ico-arrow-d"></i></a>
                    <a class="fSort">销量<i class="f-ico-arrow-d"></i></a>
                    <a class="fSort">价格<i class="f-ico-triangle-mt"></i><i class="f-ico-triangle-mb"></i></a>
                </div>

                <!-- 商品详情 -->
                <div class="view grid-nosku">

                    <div class="product" v-for="result in results">
                        <div class="product-iWrap">
                            <!--商品封面-->
                            <div class="productImg-wrap">
                                <a class="productImg">
                                    <img :src="result.img">
                                </a>
                            </div>
                            <!--价格-->
                            <p class="productPrice">
                                <em>{{result.price}}</em>
                            </p>
                            <!--标题-->
                            <p class="productTitle">
                                <a v-html="result.title"> </a>
                            </p>
                            <!-- 店铺名 -->
                            <div class="productShop">
                                <span>店铺： 狂神说Java </span>
                            </div>
                            <!-- 成交信息 -->
                            <p class="productStatus">
                                <span>月成交<em>999笔</em></span>
                                <span>评价 <a>3</a></span>
                            </p>
                        </div>
                    </div>
                </div>
            </div>
        </div>
    </div>
</div>

<!--前端使用vue实现前后端分离 -->
<script th:src="@{/js/axios.min.js}"></script>
<script th:src="@{/js/vue.min.js}"></script>
<script>
    new Vue({
        el:"#app",
        data:{
            keyword:'',//搜索的关键字
            results:[] //搜索的结果
        },
        methods:{
            searchKey(){
                var keyword = this.keyword;
                console.log(keyword);
                axios.get('search/'+keyword+"/1/10").then(response=>{
                    console.log(response);
                    this.results=response.data; //绑定数据
                })
            }
        }
    })
</script>

</body>
</html>

引入的前端依赖

其中这个axio.min.js和vue.min.js是通过 下面三个命令安装的，先创建一个vue的文件夹，然后输入npm install vue,好了以后输入npm init,按照步骤一路enter，好了以后输入npm install axios，然后进入对应的文件夹就可以找到上面需要的文件了。

npm install vue
npm init
npm install axios