FT/foton-slrs-system-rest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'develop_sync_es' into 'develop_master'

Browse Source 
Develop sync es

See merge request LiuChao/foton-slrs-system-rest!458
This commit is contained in:
super_liu
2022-04-28 19:39:21 +08:00
parent cd1494e16f 875e8670fe
commit 0d7091ff16
7 changed files with 204 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
adc-da-main/src/main/resources/application-dev.properties
+1 -1
View File
@@ -11,7 +11,7 @@ spring.datasource.password = Fting&8g35g#geg2
#==============================================
# 应用设置
spring.application.name=FotonLAWSSystem
spring.application.name=FotonLAWSSystem-REST-SEARCH
application.code=20200101
application.center=1
#==============================================
adc-da-main/src/main/resources/application.properties
+1 -1
View File
@@ -9,7 +9,7 @@ spring.profiles.active=dev
server.compression.enabled=true
server.compression.mime-types=application/json,application/xml,text/html,text/plain,text/css,application/x-javascript
# 端口号设置
server.port=4202
server.port=10086
#主服务session超时
server.servlet.session.timeout =600
adc-da-main/src/main/resources/logback-spring.xml
+1 -1
View File
@@ -8,7 +8,7 @@
<!-- 项目名称 -->
<property name="PROJECT_NAME" value="adc-da" />
<!-- 定义日志文件的存储地址,勿在 LogBack的配置中使用相对路径 -->
<property name="LOG_HOME" value="/tmp/applog/pcms-rest-system" />
<property name="LOG_HOME" value="/data/slrs/rest-system-search/logs" />
<!-- <property name="LOG_HOME" value="../logs/pcms-rest" />-->
<!-- 定义系统日志文件的存储地址,勿在 LogBack的配置中使用相对路径 -->
<property name="LOG_HOME_SYSTEM" value="system" />
adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java
+5
View File
@@ -118,6 +118,11 @@ public interface ElasticsearchService {
*/
Map<String, Object> searchDataById(String index, String type, String id, String fields);
/**
* 使用索引查询所有
*/
List<Map<String,Object>> searchAll(String index);
/**
* 使用分词查询
adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java
+55
View File
@@ -8,6 +8,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest;
import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse;
import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -24,11 +25,16 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.BucketOrder;
import org.elasticsearch.search.aggregations.bucket.range.Range;
@@ -36,6 +42,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms;
import org.elasticsearch.search.aggregations.metrics.Avg;
import org.elasticsearch.search.aggregations.metrics.Max;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -273,6 +280,54 @@ public class ElasticsearchServiceImpl implements ElasticsearchService {
return getResponse.getSource();
}
/**
* 使用索引查询所有
*/
@Override
public List<Map<String, Object>> searchAll(String index) {
List<Map<String, Object>> sourceList = new ArrayList<Map<String, Object>>();
//1、指定es集群 cluster.name 是固定的key值，my-application是ES集群的名称
// Settings settings = Settings.builder().put("cluster.name", "my-application").build();
QueryBuilder qBuilder = QueryBuilders.matchAllQuery();
SearchResponse sResponse = client.prepareSearch(index)
.setQuery(qBuilder).setTrackTotalHits(true)
.get();
SearchHits hits = sResponse.getHits();
if(hits.getTotalHits().value > 0){
SearchResponse scrollResp = search(index,qBuilder, 1,(int) hits.getTotalHits().value);
for (SearchHit hit : scrollResp.getHits().getHits()) {
sourceList.add(hit.getSourceAsMap());
}
}
return sourceList;
}
public SearchResponse search(String index, QueryBuilder query,int page, int size) {
updateIndex(index, page,size);
SearchResponse searchResponse = client.prepareSearch(index)
.setScroll(new TimeValue(360000))
.setQuery(query).setTrackTotalHits(true).setSize(size)
.get();
return searchResponse;
}
//更新索引的max_result_window参数
private boolean updateIndex(String indices, int from,int size) {
int records = from * size + size;
if (records <= 10000) return true;
AcknowledgedResponse indexResponse = client.admin().indices()
.prepareUpdateSettings(indices)
.setSettings(Settings.builder()
.put("index.max_result_window", records)
.build()
).get();
return indexResponse.isAcknowledged();
}
/**
* 使用分词查询
adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java
+70 -7
View File
@@ -18,6 +18,7 @@ import com.adc.da.util.http.ResponseMessage;
import com.adc.da.util.http.Result;
import com.adc.da.utils.util.DateUtil;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
@@ -91,6 +92,55 @@ public class ResetSearchCenterService {
return Result.success(res);
}
@Async
public ResponseMessage syncResetALLSearchCenter(SearchCenter searchCenter) throws Exception{
Boolean getFulltextserchIndex = elasticsearchService.isIndexExist("fulltextserch");
SearchCenter standSearch = new SearchCenter();
QueryWrapper qw = new QueryWrapper();
qw.eq("VALID_FLAG","0");
int count = iSarStandardsInfoService.count(qw);
standSearch.setExecType(searchCenter.getExecType());
standSearch.setCountStand(count);
SarStandardsInfoEOPage page = new SarStandardsInfoEOPage();
page.setSyncParam("sync");
ResponseMessage stand = resetStandSearchCenter(page,standSearch);
List<String> r = new ArrayList<>();
if(stand.isOk() && StringUtils.isNotBlank(stand.getData().toString())){
r.add(stand.getData().toString());
}
SearchCenter lawsSearch = new SearchCenter();
QueryWrapper qw2 = new QueryWrapper();
qw2.eq("VALID_FLAG","0");
int count2 = iSarLawsStandInfoService.count(qw2);
lawsSearch.setExecType(searchCenter.getExecType());
lawsSearch.setCountStand(count2);
SarLawsStandInfoPage lawsPage = new SarLawsStandInfoPage();
page.setSyncParam("sync");
ResponseMessage laws = resetLawsSearchCenter(lawsPage,lawsSearch);
if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){
r.add(laws.getData().toString());
}
SearchCenter bussSearch = new SearchCenter();
QueryWrapper qw3 = new QueryWrapper();
qw3.eq("VALID_FLAG","0");
int count3 = iSarBussionessStandService.count(qw3);
bussSearch.setExecType(searchCenter.getExecType());
bussSearch.setCountStand(count3);
SarBussionessStandEOPage bussPage = new SarBussionessStandEOPage();
page.setSyncParam("sync");
ResponseMessage buss = resetBussStandSearchCenter(bussPage,bussSearch);
if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){
r.add(buss.getData().toString());
}
String res = String.join(", ", r);
logger.info(res);
return Result.success(res);
}
//重建国内外标准
@Async
public ResponseMessage resetStandSearchCenter(SarStandardsInfoEOPage page,SearchCenter searchCenter) throws Exception{
@@ -100,12 +150,15 @@ public class ResetSearchCenterService {
Boolean getStandIndex = elasticsearchService.isIndexExist("stand");
List<Map<String, Object>> searchListData = elasticsearchService.searchListData("stand","","","");
if(!getStandIndex){
logger.info("ES 不存在索引:stand");
return Result.error("ES 不存在索引:stand");
}
List<Map<String, Object>> searchListData = elasticsearchService.searchAll("stand");
List<String> esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList());
logger.info(searchListData.toString());
logger.info(esIdList.toString());
page.setValidFlag("0");
if(searchCenter.getIdList() != null && !searchCenter.getIdList().isEmpty()){
String[] result = searchCenter.getIdList().toArray(new String[0]);
@@ -170,15 +223,20 @@ public class ResetSearchCenterService {
elasticsearchService.deleteBatchId(delList,"stand");
elasticsearchService.deleteBatchId(delList,"fulltextserch");
}
logger.info("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）删除-共"+delList.size()+"条数据");
return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）删除-共"+delList.size()+"条数据");
logger.info("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+0+"条）删除-共"+delList.size()+"条数据");
return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+0+"条）删除-共"+delList.size()+"条数据");
}
@Async
public ResponseMessage resetLawsSearchCenter(SarLawsStandInfoPage sarLawsInfoEOPage,SearchCenter searchCenter) throws Exception{
Boolean getLawsIndex = elasticsearchService.isIndexExist("laws");
List<Map<String, Object>> searchListData = elasticsearchService.searchListData("laws","","","");
if(!getLawsIndex){
logger.info("ES 不存在索引:laws");
return Result.error("ES 不存在索引:laws");
}
List<Map<String, Object>> searchListData = elasticsearchService.searchAll("laws");
List<String> esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList());
@@ -262,7 +320,12 @@ public class ResetSearchCenterService {
public ResponseMessage resetBussStandSearchCenter(SarBussionessStandEOPage sarBussionessStandEOPage,SearchCenter searchCenter) throws Exception{
Boolean getbussstandIndex = elasticsearchService.isIndexExist("bussstand");
List<Map<String, Object>> searchListData = elasticsearchService.searchListData("bussstand","","","");
if(!getbussstandIndex){
logger.info("ES 不存在索引:bussstand");
return Result.error("ES 不存在索引:bussstand");
}
List<Map<String, Object>> searchListData = elasticsearchService.searchAll("bussstand");
List<String> esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList());
adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java
+71
View File
@@ -0,0 +1,71 @@
package com.adc.da.search.sync;
import com.adc.da.search.bean.SearchCenter;
import com.adc.da.search.server.ResetSearchCenterService;
import com.adc.da.slrs.sarBussionessStand.dao.SarBussionessStandDao;
import com.adc.da.slrs.sarBussionessStand.entity.SarBussionessStand;
import com.adc.da.slrs.sarBussionessStandState.entity.SarBussionessStandState;
import com.adc.da.slrs.sarBussionessStandState.service.ISarBussionessStandStateService;
import com.adc.da.utils.util.DateUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import lombok.extern.slf4j.Slf4j;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Date;
import java.util.List;
import java.util.stream.Collectors;
/**
* @Description: TODO
* @author: super_liu
* @date: 2022年01月25日 3:34
*/
@EnableScheduling
@Component
@Slf4j
public class restSearchSync {
Logger logger = LoggerFactory.getLogger(restSearchSync.class);
@Autowired
private ResetSearchCenterService resetSearchCenterService;
/**
* 根据配置文件设置是否开启定时器
*/
@Value("${isNotScheduled}")
private boolean isNotScheduled; //是否开启定时器
// 每天0点1分执行 重置ES 自动更新、新增、删除
// @Scheduled(cron="0 0 1 1 * ?")
@Scheduled(cron = "0 1 0 * * ?")
@Async
public void StandScheduledJobMonthBegin(){
if(isNotScheduled){
try{
Thread.sleep(2000);
logger.info("每天0点1分执行 重置ES 自动更新、新增、删除:"+Thread.currentThread().getName() + " cron=0 1 0 * * ? --- " + new Date()+"---START-01");
SearchCenter searchCenter = new SearchCenter();
// ALL 执行全部
searchCenter.setExecType("ALL");
resetSearchCenterService.syncResetALLSearchCenter(searchCenter);
logger.info("每天0点1分执行 重置ES 自动更新、新增、删除:"+Thread.currentThread().getName() + " cron=0 1 0 * * ? --- " + new Date()+"---End-01");
}catch(Exception e){
logger.info(e.getMessage());
}
}
}
}