安装elasticsearch 7.7.0
1.使用docker安装elasticsearch 7.7.0
docker pull docker.elastic.co/elasticsearch/elasticsearch:7.7.02.运行elasticsearch
docker run -d -p 9200:9200 -p 9300:9300 --name search elasticsearch:7.7.03.修改elasticsearch配置
docker exec -it elasticsearch /bin/bash
vi /usr/share/elasticsearch/config/elasticsearch.yml
修改为以下配置
cluster.name: "docker-cluster"
network.host: 0.0.0.0
action.auto_create_index: true
http.cors.enabled: true
http.cors.allow-origin: "*"安装ik分词器
先安装wget
yum install wget
再进入elasitcsearch的plugin目录下
cd /usr/share/elasticsearch/plugins
创建ik文件夹
mkdir ik
进入ik文件夹
cd ik
下载ik分词器，必须与elasticsearch版本相同
wget https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v7.7.0/elasticsearch-analysis-ik-7.7.0.zip
将压缩包解压在当前文件夹，然后删除安装包，重启elasticsearch即可
unzip elasticsearch-analysis-ik-7.7.0.zip
rm -rf elasticsearch-analysis-ik-7.7.0.zip安装Logstash(自动同步mysql到elasticsearch)
配置 mysql.conf
cd /usr/local/
wget https://artifacts.elastic.co/downloads/logstash/logstash-7.7.0.zip
unzip logstash-7.7.0.zip
cd logstash-7.7.0/config/
创建mysql.conf
touch mysql.conf
vi mysql.conf
修改配置文件如下
input {
stdin {
}
jdbc {
#jdbc sql server 驱动存放位置,各个数据库都有对应的驱动，需自己下载
jdbc_driver_library => "/opt/jar/mysql-connector-java-8.0.13.jar"
#jdbc class 不同数据库有不同的 class 配置
jdbc_driver_class => "com.mysql.cj.jdbc.Driver"
#配置数据库连接 ip 和端口，以及数据库
jdbc_connection_string => "jdbc:mysql://localhost:3306/shirotest?useSSL=false&useUnicode=true&characterEncoding=utf8&serverTimezone=GMT%2B8"
jdbc_user => "数据库账号"
jdbc_password => "数据库密码"
# schedule => 分 时 天 月 年
# schedule => * 22 * * * 表示每天22点执行一次
schedule => "* * * * *"
#是否清除 last_run_metadata_path 的记录,如果为真那么每次都相当于从头开始查询所有的数据库记录
clean_run => false
#标志目前logstash同步的位置信息（类似offset）。比如id、updatetime。logstash通过这个标志，可以判断目前同步到哪一条数据。
statement => "SELECT ID,USERNAME,EMAIL,PHONE,AGE,NICKNAME,AVATAR_URL,LAST_LOGIN_TIME,BAN,IS_DELETED,CREATE_TIME,UPDATE_TIME FROM user_info WHERE UPDATE_TIME >= :sql_last_value"
#是否需要记录某个column 的值,如果 record_last_run 为真,可以自定义我们需要表的字段名称，
#此时该参数就要为 true. 否则默认 track 的是 timestamp 的值.
use_column_value => true
#如果列是时间字段（比如updateTime），一定要指定这个类型为timestamp。
tracking_column_type => "timestamp"
#表示表中哪一列用于判断logstash同步的位置信息。与sql_last_value比较判断是否需要同步这条数据。
tracking_column => "UPDATE_TIME"
#我们只需要在 SQL 语句中 WHERE MY_ID > :last_sql_value 即可. 其中 :last_sql_value 取得就是该文件中的值
last_run_metadata_path => "syncpoint_table"
}
}
output {
elasticsearch {
# ES集群地址（如果是单机的填一个就行）
hosts => ["localhost:9200"]
# 索引名称
# 需要关联的数据库中有有一个id字段，对应doc中的id
index => "user"
document_id => "%{id}"
}
# 这里输出调试，正式运行时可以注释掉
stdout {
# JSON格式输出
codec => json_lines
}
}运行logstash
cd /usr/local/logstash-7.7.0/
nohup ./bin/logstash -f ./config/mysql.confSpringboot整合ElasticSearch
分片(shard)与副本(replica)
1)分片（shard）
Elasticsearch集群允许系统存储的数据量超过单机容量，实现这一目标引入分片策略shard。在一个索引index中，数据（document）被分片处理（sharding）到多个分片上。Elasticsearch屏蔽了管理分片的复杂性，使得多个分片呈现出一个大索引的样子。
2）副本（replica）
为了提升访问压力过大是单机无法处理所有请求的问题，Elasticsearch集群引入了副本策略replica。副本策略对index中的每个分片创建冗余的副本，处理查询时可以把这些副本当做主分片来对待（primary shard），此外副本策略提供了高可用和数据安全的保障，当分片所在的机器宕机，Elasticsearch可以使用其副本进行恢复，从而避免数据丢失。
yml中添加连接参数
#ElasticSearch配置参数
elasticsearch:
host: Your Ip
port: 9200添加elasticsearch配置
package com.demo.Config;
import lombok.Data;
import org.apache.http.HttpHost;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.elasticsearch.config.AbstractElasticsearchConfiguration;
/**
* @author zqf
* @date 2021/6/14 18:35
* @description
*/
@Configuration
//读取yml中的elasticsearch自动赋值给host和port
@ConfigurationProperties(prefix = "elasticsearch")
@Data
public class ElasticSearchConfig extends AbstractElasticsearchConfiguration {
private String host;
private Integer port;
@Override
public RestHighLevelClient elasticsearchClient() {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, port));
RestHighLevelClient restHighLevelClient = new
RestHighLevelClient(builder);
return restHighLevelClient;
}
}
创建实体类
package com.demo.Entity.ElasticSearch;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.ToString;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
/**
* @author zqf
* @date 2021/6/14 18:33
* @description
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
@ToString
//index为elasticsearch索引名,shards为分片数，replicas为副本
@Document(indexName = "user", shards = 3, replicas = 1)
public class User {
@Id
private String id;
//analyzer = "ik_max_word"使用分词器
@Field(type = FieldType.Text,analyzer = "ik_max_word")
private String username;
@Field(type = FieldType.Text)
private String nickname;
@Field(type = FieldType.Keyword)
private String phone;
@Field(type = FieldType.Text)
private String address;
//type=FieldType.Keyword即不能进行模糊查询
@Field(type = FieldType.Keyword)
private Integer age;
//index=false即不能通过该字段查询
@Field(type = FieldType.Keyword,index = false)
private String avatarUrl;
}
创建Mapper并集成ElasticsearchRepository<User,String>
package com.demo.Mapper.ElkMapper;
import com.demo.Entity.ElasticSearch.User;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.annotations.Highlight;
import org.springframework.data.elasticsearch.annotations.HighlightField;
import org.springframework.data.elasticsearch.annotations.HighlightParameters;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
import org.springframework.stereotype.Repository;
import java.util.List;
/**
* @author zqf
* @date 2021/6/14 18:43
* @description
*/
@Repository
//User为实体类，String为注解类型
public interface UserElkMapper extends ElasticsearchRepository<User,String> {
//该注解使nickname，username，phone，address字段检索结果中对关键字进行高亮处理
@Highlight(
fields = {@HighlightField(name = "nickname"),@HighlightField(name = "username"),
@HighlightField(name = "phone"),@HighlightField(name = "address"),
},
parameters = @HighlightParameters(preTags = {"<span style='color:red'>"}, postTags = {"</span>"}, numberOfFragments = 0)
)
//因为elasticsearch的查询是按照方法名来进行处理的，下面方法名的意思为查找只要该四个字段存在查找的关键字就视为查询成功
//传入了Pageable条件即视为分页查询
List<SearchHit<User>> findByUsernameOrNicknameOrPhoneOrAddress(String word1,String word2,String word3,String word4,Pageable pageable);
}
创建控制器
package com.demo.Controller;
import com.alibaba.fastjson.JSON;
import com.demo.Common.lang.ResultBody;
import com.demo.Entity.ElasticSearch.User;
import com.demo.Mapper.ElkMapper.UserElkMapper;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.data.domain.Pageable;
import org.springframework.data.elasticsearch.core.SearchHit;
import org.springframework.data.elasticsearch.core.SearchHits;
import org.springframework.web.bind.annotation.*;
import java.util.*;
/**
* @author zqf
* @date 2021/6/14 19:14
* @description
*/
@RestController
public class ElkTestController {
@Autowired
private UserElkMapper userElkMapper;
//分页+高亮
@GetMapping("/getPage")
public ResultBody getPage(@RequestParam("page")Integer page,@RequestParam("size")Integer size,@RequestParam("word")String word) {
Pageable pageable = PageRequest.of(page,size);
List<SearchHit<User>> list = userElkMapper.findByUsernameOrNicknameOrPhoneOrAddress(word,word,word,word,pageable);
return mergeSearchHitHighlight(list);
}
//合并List<SearchHit<T>>
public <T> ResultBody mergeSearchHitHighlight(List<SearchHit<T>> searchHit) {
Map map;
List<Map> resMap = new ArrayList<>();
for (SearchHit<T> userSearchHit : searchHit) {
Set<String> strings = userSearchHit.getHighlightFields().keySet();
List<String> highlightFields = new ArrayList<>(strings);
T content = userSearchHit.getContent();
map = JSON.parseObject(JSON.toJSONString(content), Map.class);
for (String highlightField : highlightFields) {
String s = userSearchHit.getHighlightFields().get(highlightField).toString();
map.put(highlightField, s.substring(1, s.length() - 1));
}
resMap.add(map);
}
return ResultBody.ok().data("res", resMap);
}
//合并SearchHits
public <T> ResultBody mergeSearchHitHighlight(SearchHits<T> searchHit) {
Map map;
List<Map> resMap = new ArrayList<>();
List<SearchHit<T>> searchHits = searchHit.getSearchHits();
for (SearchHit<T> userSearchHit : searchHits) {
Set<String> strings = userSearchHit.getHighlightFields().keySet();
List<String> highlightFields = new ArrayList<>(strings);
T content = userSearchHit.getContent();
map = JSON.parseObject(JSON.toJSONString(content), Map.class);
for (String highlightField : highlightFields) {
String s = userSearchHit.getHighlightFields().get(highlightField).toString();
map.put(highlightField, s.substring(1, s.length() - 1));
}
resMap.add(map);
}
return ResultBody.ok().data("res", resMap);
}
}
使用POSTMAN测试
http://localhost:8082/getPage?page=0&size=2&word=这诗,方块状查询结果
{
"success": true,
"code": 200,
"message": "成功",
"data": {
"res": [
{
"phone": "<span style='color:red'>这</span>首<span style='color:red'>诗</span>格式整饬，前六句呈<span style='color:red'>方</span><span style='color:red'>块</span><span style='color:red'>状</span>WQFQWFWF",
"nickname": "<span style='color:red'>这</span>首<span style='color:red'>诗</span>格式整饬，前六句呈<span style='color:red'>方</span><span style='color:red'>块</span><span style='color:red'>状</span>WQFQWFWF",
"id": "12344",
"age": 1,
"username": "AWFAWF<span style='color:red'>这</span>AW首<span style='color:red'>诗</span>格式整饬，前六句呈<span style='color:red'>方</span><span style='color:red'>块</span><span style='color:red'>状</span>WQFQWFWFWAFF"
}
]
},
"expound": {}
}