/**
* @author: 官昌洪
* @Description:
* @Date: 2020/2/1 10:31
* @Param:
* @return:
*/
@Service
public class BwbdTypeServiceImpl implements BwbdTypeService {
@Autowired
BwbdTypeDao bwbdTypeDao;
@Autowired
RestHighLevelClient client;
@Autowired
IFgTestR3Service iFgTestR3Service;
@Autowired
IWdTestR3Service iWdTestR3Service;
@Autowired
IAlTestR3Service iAlTestR3Service;
@Override
@Transactional(rollbackFor = Exception.
class)
public void syncFg() {
List<FgTestR3> r3s = iFgTestR3Service.list((
new QueryWrapper<FgTestR3>
())
.select(
"webGuid").groupBy(
"webGuid"));
List<String> webGuids = r3s.stream().filter(fgTestR3 ->
StringUtils.isNotEmpty(fgTestR3
.getWebGuid())).map(fgTestR3 ->
fgTestR3.getWebGuid()).collect(Collectors.toList());
for (String webGuid : webGuids) {
List<FgTestR3> fgTestR3s = iFgTestR3Service.list((
new QueryWrapper<FgTestR3>
())
.eq("webGuid", webGuid));
List<BwbdType> bwbdTypes =
new ArrayList<>
();
for (FgTestR3 fgTestR3 : fgTestR3s) {
BwbdType bwbdType =
new BwbdType();
BeanUtil.copyProperties(fgTestR3, bwbdType, CopyOptions.create().ignoreNullValue());
bwbdType.setId(BwbdType.DATA_TYPE_FG + BwbdType.ID_SPLIT +
fgTestR3.getAutoid());
bwbdType.setDataType(BwbdType.DATA_TYPE_FG);
bwbdTypes.add(bwbdType);
}
bwbdTypeDao.saveAll(bwbdTypes);
}
}
@Override
@Transactional(rollbackFor = Exception.
class)
public void syncWd() {
List<WdTestR3> r3s = iWdTestR3Service.list((
new QueryWrapper<WdTestR3>
())
.select(
"webGuid").groupBy(
"webGuid"));
List<String> webGuids = r3s.stream().filter(wdTestR3 ->
StringUtils.isNotEmpty(wdTestR3
.getWebGuid())).map(wdTestR3 ->
wdTestR3.getWebGuid()).collect(Collectors.toList());
for (String webGuid : webGuids) {
List<WdTestR3> wdTestR3s = iWdTestR3Service.list((
new QueryWrapper<WdTestR3>
())
.eq("webGuid", webGuid));
List<BwbdType> bwbdTypes =
new ArrayList<>
();
for (WdTestR3 wdTestR3 : wdTestR3s) {
BwbdType bwbdType =
new BwbdType();
BeanUtil.copyProperties(wdTestR3, bwbdType, CopyOptions.create().ignoreNullValue());
bwbdType.setContents(wdTestR3.getRequestContent() +
wdTestR3.getContents()
+
wdTestR3.getResponseContent());
bwbdType.setId(BwbdType.DATA_TYPE_WD + BwbdType.ID_SPLIT +
wdTestR3.getAutoid());
bwbdType.setDataType(BwbdType.DATA_TYPE_WD);
bwbdTypes.add(bwbdType);
}
bwbdTypeDao.saveAll(bwbdTypes);
}
}
@Override
@Transactional(rollbackFor = Exception.
class)
public void syncAl() {
List<AlTestR3> r3s = iAlTestR3Service.list((
new QueryWrapper<AlTestR3>
())
.select(
"webGuid").groupBy(
"webGuid"));
List<String> webGuids = r3s.stream().filter(fgTestR3 ->
StringUtils.isNotEmpty(fgTestR3
.getWebGuid())).map(fgTestR3 ->
fgTestR3.getWebGuid()).collect(Collectors.toList());
for (String webGuid : webGuids) {
List<AlTestR3> alTestR3s = iAlTestR3Service.list((
new QueryWrapper<AlTestR3>
())
.eq("webGuid", webGuid));
List<BwbdType> bwbdTypes =
new ArrayList<>
();
for (AlTestR3 alTestR3 : alTestR3s) {
BwbdType bwbdType =
new BwbdType();
BeanUtil.copyProperties(alTestR3, bwbdType, CopyOptions.create().ignoreNullValue());
bwbdType.setId(BwbdType.DATA_TYPE_AL + BwbdType.ID_SPLIT +
alTestR3.getAutoid());
bwbdType.setDataType(BwbdType.DATA_TYPE_AL);
bwbdType.setNumbers(alTestR3.getCaseNumber());
bwbdTypes.add(bwbdType);
}
if (bwbdTypes.size() >
20000) {
splitSave(bwbdTypes);
} else {
bwbdTypeDao.saveAll(bwbdTypes);
}
}
}
/**
* @author: 官昌洪
* @Description: 直接保存会内存溢出，切片保存数据
* @Date: 2020/2/5 11:43
* @Param:
* @return:
*/
private void splitSave(List<BwbdType>
bwbdTypes) {
int size =
bwbdTypes.size();
int step =
2000;
int fromIndex =
0;
int toIndex =
step;
while (toIndex <
size) {
List<BwbdType> optList =
bwbdTypes.subList(fromIndex, toIndex);
bwbdTypeDao.saveAll(optList);
fromIndex =
toIndex;
toIndex = toIndex +
step;
}
if (fromIndex <
size) {
toIndex =
size;
List<BwbdType> optList =
bwbdTypes.subList(fromIndex, toIndex);
bwbdTypeDao.saveAll(optList);
}
}
@Override
public List<BwbdType>
improveSearch(SearchDto searchDto) {
String text =
searchDto.getTerm();
text = QueryParser.escape(text);
// 主要就是这一句把特殊字符都转义,那么lucene就可以识别
// 搜索请求对象
SearchRequest searchRequest =
new SearchRequest(BwbdType.ES_INDEX);
// 指定类型
searchRequest.types(BwbdType.ES_TYPE);
// 搜索源构建对象
SearchSourceBuilder searchSourceBuilder =
new SearchSourceBuilder();
// 搜索方式
// 首先构造多关键字查询条件
MultiMatchQueryBuilder matchQueryBuilder =
QueryBuilders
.multiMatchQuery(text, BwbdType.PROPERTY_NUMBERS
, BwbdType.PROPERTY_TITLES, BwbdType.PROPERTY_CONTENTS)
.field(BwbdType.PROPERTY_NUMBERS, 100)
.field(BwbdType.PROPERTY_TITLES, 10)
.field(BwbdType.PROPERTY_CONTENTS, 1).minimumShouldMatch(BwbdType.MATCH_LEVEL_THREE);
// 添加条件到布尔查询
BoolQueryBuilder boolQueryBuilder =
QueryBuilders.boolQuery();
boolQueryBuilder.must(matchQueryBuilder);
// // 通过布尔查询来构造过滤查询
// boolQueryBuilder.filter(QueryBuilders.matchQuery("economics","L"));
// 将查询条件封装给查询对象
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.size(searchDto.getSize());
searchSourceBuilder.from(searchDto.getPage() -
1);
// ***********************
// 高亮查询
HighlightBuilder highlightBuilder =
new HighlightBuilder();
highlightBuilder.preTags(CommonConstraint.LIGHT_TAG_START); // 高亮前缀
highlightBuilder.postTags(CommonConstraint.LIGHT_TAG_END);
// 高亮后缀
List<HighlightBuilder.Field> fields =
highlightBuilder.fields();
fields.add(new HighlightBuilder
.Field(BwbdType.PROPERTY_NUMBERS)); // 高亮字段
fields.add(
new HighlightBuilder
.Field(BwbdType.PROPERTY_TITLES)); // 高亮字段
fields.add(
new HighlightBuilder
.Field(BwbdType.PROPERTY_CONTENTS)); // 高亮字段
// 添加高亮查询条件到搜索源
searchSourceBuilder.highlighter(highlightBuilder);
// ***********************
// // 设置源字段过虑,第一个参数结果集包括哪些字段，第二个参数表示结果集不包括哪些字段
// searchSourceBuilder.fetchSource(new String[]{"name","studymodel","price","timestamp"},new String[]{});
// 向搜索请求对象中设置搜索源
searchRequest.source(searchSourceBuilder);
// 执行搜索,向ES发起http请求
SearchResponse searchResponse =
null;
try {
searchResponse =
client.search(searchRequest, RequestOptions.DEFAULT);
} catch (IOException e) {
e.printStackTrace();
}
List<BwbdType> bwbdTypes =
obtainFgType(searchResponse);
return bwbdTypes;
}
private List<BwbdType>
obtainFgType(SearchResponse searchResponse) {
// 搜索结果
SearchHits hits =
searchResponse.getHits();
// 匹配到的总记录数
long totalHits =
hits.getTotalHits();
// 得到匹配度高的文档
SearchHit[] searchHits =
hits.getHits();
List<BwbdType> bwbdTypes =
new ArrayList<>
();
for (SearchHit hit : searchHits) {
String content = hit.getSourceAsString();
//使用ES的java接口将实体类对应的内容转换为json字符串
BwbdType bwbdType = JSONObject.parseObject(content,BwbdType.
class);
//生成pojo对象
// 获取高亮查询的内容。如果存在，则替换原来的name
Map<String, HighlightField> highlightFields =
hit.getHighlightFields();
if( highlightFields !=
null ){
HighlightField nameField = highlightFields.
get(bwbdType.PROPERTY_NUMBERS);
if(nameField!=
null){
Text[] fragments =
nameField.getFragments();
StringBuffer stringBuffer =
new StringBuffer();
for (Text str : fragments) {
stringBuffer.append(str.string());
}
String numbers =
stringBuffer.toString();
bwbdType.setNumbers(numbers);
}
HighlightField titlesField = highlightFields.
get(bwbdType.PROPERTY_TITLES);
if(titlesField!=
null){
Text[] fragments =
titlesField.getFragments();
StringBuffer stringBuffer =
new StringBuffer();
for (Text str : fragments) {
stringBuffer.append(str.string());
}
String titles =
stringBuffer.toString();
bwbdType.setTitles(titles);
}
HighlightField contentsField = highlightFields.
get(bwbdType.PROPERTY_CONTENTS);
if(contentsField!=
null){
Text[] fragments =
contentsField.getFragments();
StringBuffer stringBuffer =
new StringBuffer();
for (Text str : fragments) {
stringBuffer.append(str.string());
}
String contents =
stringBuffer.toString();
bwbdType.setContents(contents);
}
}
bwbdTypes.add(bwbdType);
}
return bwbdTypes;
}
}
SpringBoot迁移数据库数据到ES+操纵ES进行高级检索源码
标签：过滤 使用 转义 builder @param rest json date factor
