From 8e7c6b19d37fdc3b1e5dce332a30f1ad23322294 Mon Sep 17 00:00:00 2001 From: super_liu <396572590@qq.com> Date: Fri, 8 Apr 2022 11:48:46 +0800 Subject: [PATCH 1/6] feat: There is no way the, rest es conf --- adc-da-main/src/main/resources/application-dev.properties | 2 +- adc-da-main/src/main/resources/application.properties | 2 +- adc-da-main/src/main/resources/logback-spring.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/adc-da-main/src/main/resources/application-dev.properties b/adc-da-main/src/main/resources/application-dev.properties index a4aa4373..1e3d58ff 100644 --- a/adc-da-main/src/main/resources/application-dev.properties +++ b/adc-da-main/src/main/resources/application-dev.properties @@ -11,7 +11,7 @@ spring.datasource.password = Fting&8g35g#geg2 #============================================== # 应用设置 -spring.application.name=FotonLAWSSystem +spring.application.name=FotonLAWSSystem-REST-SEARCH application.code=20200101 application.center=1 #============================================== diff --git a/adc-da-main/src/main/resources/application.properties b/adc-da-main/src/main/resources/application.properties index 7a7db19f..2bd567fb 100644 --- a/adc-da-main/src/main/resources/application.properties +++ b/adc-da-main/src/main/resources/application.properties @@ -9,7 +9,7 @@ spring.profiles.active=dev server.compression.enabled=true server.compression.mime-types=application/json,application/xml,text/html,text/plain,text/css,application/x-javascript # 端口号设置 -server.port=4202 +server.port=10086 #主服务session超时 server.servlet.session.timeout =600 diff --git a/adc-da-main/src/main/resources/logback-spring.xml b/adc-da-main/src/main/resources/logback-spring.xml index 39f90eab..841d37a2 100644 --- a/adc-da-main/src/main/resources/logback-spring.xml +++ b/adc-da-main/src/main/resources/logback-spring.xml @@ -8,7 +8,7 @@ - + From d7bdf83678cce512bef38c5363899b80b65e171e Mon Sep 17 00:00:00 2001 From: super_liu <396572590@qq.com> Date: Fri, 8 Apr 2022 15:20:07 +0800 Subject: [PATCH 2/6] feat: There is no way the, rest es conf --- .../search/service/ElasticsearchService.java | 5 ++ .../impl/ElasticsearchServiceImpl.java | 55 +++++++++++++++++++ .../server/ResetSearchCenterService.java | 12 ++-- 3 files changed, 65 insertions(+), 7 deletions(-) diff --git a/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java b/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java index baedb9e1..41563bb6 100644 --- a/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java +++ b/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java @@ -118,6 +118,11 @@ public interface ElasticsearchService { */ Map searchDataById(String index, String type, String id, String fields); + /** + * 使用索引查询所有 + */ + + List> searchAll(String index); /** * 使用分词查询 diff --git a/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java b/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java index 0b86db69..76f1f3c5 100644 --- a/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java +++ b/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -24,11 +25,16 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.range.Range; @@ -36,6 +42,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.Avg; import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,6 +280,54 @@ public class ElasticsearchServiceImpl implements ElasticsearchService { return getResponse.getSource(); } + /** + * 使用索引查询所有 + */ + @Override + public List> searchAll(String index) { + List> sourceList = new ArrayList>(); + + //1、指定es集群 cluster.name 是固定的key值，my-application是ES集群的名称 +// Settings settings = Settings.builder().put("cluster.name", "my-application").build(); + QueryBuilder qBuilder = QueryBuilders.matchAllQuery(); + SearchResponse sResponse = client.prepareSearch(index) + .setQuery(qBuilder).setTrackTotalHits(true) + .get(); + SearchHits hits = sResponse.getHits(); + if(hits.getTotalHits().value > 0){ + SearchResponse scrollResp = search(index,qBuilder, 1,(int) hits.getTotalHits().value); + for (SearchHit hit : scrollResp.getHits().getHits()) { + sourceList.add(hit.getSourceAsMap()); + } + } + return sourceList; + } + + + public SearchResponse search(String index, QueryBuilder query,int page, int size) { + + updateIndex(index, page,size); + + SearchResponse searchResponse = client.prepareSearch(index) + .setScroll(new TimeValue(360000)) + .setQuery(query).setTrackTotalHits(true).setSize(size) + .get(); + return searchResponse; + } + + //更新索引的max_result_window参数 + private boolean updateIndex(String indices, int from,int size) { + int records = from * size + size; + if (records <= 10000) return true; + AcknowledgedResponse indexResponse = client.admin().indices() + .prepareUpdateSettings(indices) + .setSettings(Settings.builder() + .put("index.max_result_window", records) + .build() + ).get(); + return indexResponse.isAcknowledged(); + } + /** * 使用分词查询 diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java index 2c26cdc9..5c91de56 100644 --- a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java +++ b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java @@ -100,12 +100,10 @@ public class ResetSearchCenterService { Boolean getStandIndex = elasticsearchService.isIndexExist("stand"); - List> searchListData = elasticsearchService.searchListData("stand","","",""); + List> searchListData = elasticsearchService.searchAll("stand"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); - logger.info(searchListData.toString()); - logger.info(esIdList.toString()); page.setValidFlag("0"); if(searchCenter.getIdList() != null && !searchCenter.getIdList().isEmpty()){ String[] result = searchCenter.getIdList().toArray(new String[0]); @@ -170,15 +168,15 @@ public class ResetSearchCenterService { elasticsearchService.deleteBatchId(delList,"stand"); elasticsearchService.deleteBatchId(delList,"fulltextserch"); } - logger.info("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）删除-共"+delList.size()+"条数据"); - return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）删除-共"+delList.size()+"条数据"); + logger.info("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+0+"条）删除-共"+delList.size()+"条数据"); + return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+0+"条）删除-共"+delList.size()+"条数据"); } @Async public ResponseMessage resetLawsSearchCenter(SarLawsStandInfoPage sarLawsInfoEOPage,SearchCenter searchCenter) throws Exception{ Boolean getLawsIndex = elasticsearchService.isIndexExist("laws"); - List> searchListData = elasticsearchService.searchListData("laws","","",""); + List> searchListData = elasticsearchService.searchAll("laws"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); @@ -262,7 +260,7 @@ public class ResetSearchCenterService { public ResponseMessage resetBussStandSearchCenter(SarBussionessStandEOPage sarBussionessStandEOPage,SearchCenter searchCenter) throws Exception{ Boolean getbussstandIndex = elasticsearchService.isIndexExist("bussstand"); - List> searchListData = elasticsearchService.searchListData("bussstand","","",""); + List> searchListData = elasticsearchService.searchAll("bussstand"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); From 178c30336e81680ae097ad2d847f6a120ef16838 Mon Sep 17 00:00:00 2001 From: super_liu <396572590@qq.com> Date: Fri, 8 Apr 2022 15:26:37 +0800 Subject: [PATCH 3/6] feat: There is no way the, rest es conf --- adc-da-main/src/main/resources/logback-spring.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/adc-da-main/src/main/resources/logback-spring.xml b/adc-da-main/src/main/resources/logback-spring.xml index 841d37a2..b1319cf0 100644 --- a/adc-da-main/src/main/resources/logback-spring.xml +++ b/adc-da-main/src/main/resources/logback-spring.xml @@ -8,7 +8,7 @@ - + From 003ef482791b8715bff2841da3aeb1bc5a9e404a Mon Sep 17 00:00:00 2001 From: super_liu <396572590@qq.com> Date: Fri, 8 Apr 2022 15:47:34 +0800 Subject: [PATCH 4/6] feat: There is no way the, rest es conf --- .../search/server/ResetSearchCenterService.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java index 5c91de56..eb408bc7 100644 --- a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java +++ b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java @@ -100,6 +100,11 @@ public class ResetSearchCenterService { Boolean getStandIndex = elasticsearchService.isIndexExist("stand"); + if(!getStandIndex){ + logger.info("ES 不存在索引:stand"); + return Result.error("ES 不存在索引:stand"); + } + List> searchListData = elasticsearchService.searchAll("stand"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); @@ -176,6 +181,11 @@ public class ResetSearchCenterService { public ResponseMessage resetLawsSearchCenter(SarLawsStandInfoPage sarLawsInfoEOPage,SearchCenter searchCenter) throws Exception{ Boolean getLawsIndex = elasticsearchService.isIndexExist("laws"); + if(!getLawsIndex){ + logger.info("ES 不存在索引:laws"); + return Result.error("ES 不存在索引:laws"); + } + List> searchListData = elasticsearchService.searchAll("laws"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); @@ -260,6 +270,11 @@ public class ResetSearchCenterService { public ResponseMessage resetBussStandSearchCenter(SarBussionessStandEOPage sarBussionessStandEOPage,SearchCenter searchCenter) throws Exception{ Boolean getbussstandIndex = elasticsearchService.isIndexExist("bussstand"); + if(!getbussstandIndex){ + logger.info("ES 不存在索引:bussstand"); + return Result.error("ES 不存在索引:bussstand"); + } + List> searchListData = elasticsearchService.searchAll("bussstand"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); From 1fd0861d648d48ee98e650df2a3d4538ac1ed061 Mon Sep 17 00:00:00 2001 From: super_liu <396572590@qq.com> Date: Wed, 27 Apr 2022 20:21:01 +0800 Subject: [PATCH 5/6] feat: There is no way the, rest es conf --- .../server/ResetSearchCenterService.java | 44 ++++++++++++ .../adc/da/search/sync/restSearchSync.java | 71 +++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java index eb408bc7..aaacb4cb 100644 --- a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java +++ b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java @@ -18,6 +18,7 @@ import com.adc.da.util.http.ResponseMessage; import com.adc.da.util.http.Result; import com.adc.da.utils.util.DateUtil; import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.gson.JsonObject; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.client.transport.TransportClient; @@ -91,6 +92,49 @@ public class ResetSearchCenterService { return Result.success(res); } + @Async + public ResponseMessage syncResetALLSearchCenter(SearchCenter searchCenter) throws Exception{ + Boolean getFulltextserchIndex = elasticsearchService.isIndexExist("fulltextserch"); + SearchCenter standSearch = new SearchCenter(); + QueryWrapper qw = new QueryWrapper(); + qw.eq("VALID_FLAG","0"); + int count = iSarStandardsInfoService.count(qw); + standSearch.setExecType(searchCenter.getExecType()); + standSearch.setCountStand(count); + ResponseMessage stand = resetStandSearchCenter(new SarStandardsInfoEOPage(),standSearch); + List r = new ArrayList<>(); + if(stand.isOk() && StringUtils.isNotBlank(stand.getData().toString())){ + r.add(stand.getData().toString()); + } + + SearchCenter lawsSearch = new SearchCenter(); + QueryWrapper qw2 = new QueryWrapper(); + qw2.eq("VALID_FLAG","0"); + int count2 = iSarLawsStandInfoService.count(qw2); + lawsSearch.setExecType(searchCenter.getExecType()); + lawsSearch.setCountStand(count2); + ResponseMessage laws = resetLawsSearchCenter(new SarLawsStandInfoPage(),lawsSearch); + if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){ + r.add(laws.getData().toString()); + } + + SearchCenter bussSearch = new SearchCenter(); + QueryWrapper qw3 = new QueryWrapper(); + qw3.eq("VALID_FLAG","0"); + int count3 = iSarBussionessStandService.count(qw3); + bussSearch.setExecType(searchCenter.getExecType()); + bussSearch.setCountStand(count3); + ResponseMessage buss = resetBussStandSearchCenter(new SarBussionessStandEOPage(),bussSearch); + if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){ + r.add(buss.getData().toString()); + } + + String res = String.join(", ", r); + logger.info(res); + + return Result.success(res); + } + //重建国内外标准 @Async public ResponseMessage resetStandSearchCenter(SarStandardsInfoEOPage page,SearchCenter searchCenter) throws Exception{ diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java b/adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java new file mode 100644 index 00000000..beeef499 --- /dev/null +++ b/adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java @@ -0,0 +1,71 @@ +package com.adc.da.search.sync; + +import com.adc.da.search.bean.SearchCenter; +import com.adc.da.search.server.ResetSearchCenterService; +import com.adc.da.slrs.sarBussionessStand.dao.SarBussionessStandDao; +import com.adc.da.slrs.sarBussionessStand.entity.SarBussionessStand; +import com.adc.da.slrs.sarBussionessStandState.entity.SarBussionessStandState; +import com.adc.da.slrs.sarBussionessStandState.service.ISarBussionessStandStateService; +import com.adc.da.utils.util.DateUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @Description: TODO + * @author: super_liu + * @date: 2022年01月25日 3:34 + */ +@EnableScheduling +@Component +@Slf4j +public class restSearchSync { + Logger logger = LoggerFactory.getLogger(restSearchSync.class); + + @Autowired + private ResetSearchCenterService resetSearchCenterService; + + /** + * 根据配置文件设置是否开启定时器 + */ + + @Value("${isNotScheduled}") + private boolean isNotScheduled; //是否开启定时器 + + + // 每天0点1分执行 重置ES 自动更新、新增、删除 +// @Scheduled(cron="0 0 1 1 * ?") + @Scheduled(cron = "0 1 0 * * ?") + @Async + public void StandScheduledJobMonthBegin(){ + if(isNotScheduled){ + try{ + Thread.sleep(2000); + logger.info("每天0点1分执行 重置ES 自动更新、新增、删除:"+Thread.currentThread().getName() + " cron=0 1 0 * * ? --- " + new Date()+"---START-01"); + + SearchCenter searchCenter = new SearchCenter(); + // ALL 执行全部 + searchCenter.setExecType("ALL"); + resetSearchCenterService.syncResetALLSearchCenter(searchCenter); + + logger.info("每天0点1分执行 重置ES 自动更新、新增、删除:"+Thread.currentThread().getName() + " cron=0 1 0 * * ? --- " + new Date()+"---End-01"); + }catch(Exception e){ + logger.info(e.getMessage()); + } + } + } + +} From 875e8670fe1cec08c4de729739ef8dcc7df001e6 Mon Sep 17 00:00:00 2001 From: super_liu <396572590@qq.com> Date: Thu, 28 Apr 2022 19:38:37 +0800 Subject: [PATCH 6/6] feat: There is no way the, rest es conf --- .../da/search/server/ResetSearchCenterService.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java index aaacb4cb..2f76d925 100644 --- a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java +++ b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java @@ -101,7 +101,9 @@ public class ResetSearchCenterService { int count = iSarStandardsInfoService.count(qw); standSearch.setExecType(searchCenter.getExecType()); standSearch.setCountStand(count); - ResponseMessage stand = resetStandSearchCenter(new SarStandardsInfoEOPage(),standSearch); + SarStandardsInfoEOPage page = new SarStandardsInfoEOPage(); + page.setSyncParam("sync"); + ResponseMessage stand = resetStandSearchCenter(page,standSearch); List r = new ArrayList<>(); if(stand.isOk() && StringUtils.isNotBlank(stand.getData().toString())){ r.add(stand.getData().toString()); @@ -113,7 +115,9 @@ public class ResetSearchCenterService { int count2 = iSarLawsStandInfoService.count(qw2); lawsSearch.setExecType(searchCenter.getExecType()); lawsSearch.setCountStand(count2); - ResponseMessage laws = resetLawsSearchCenter(new SarLawsStandInfoPage(),lawsSearch); + SarLawsStandInfoPage lawsPage = new SarLawsStandInfoPage(); + page.setSyncParam("sync"); + ResponseMessage laws = resetLawsSearchCenter(lawsPage,lawsSearch); if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){ r.add(laws.getData().toString()); } @@ -124,7 +128,9 @@ public class ResetSearchCenterService { int count3 = iSarBussionessStandService.count(qw3); bussSearch.setExecType(searchCenter.getExecType()); bussSearch.setCountStand(count3); - ResponseMessage buss = resetBussStandSearchCenter(new SarBussionessStandEOPage(),bussSearch); + SarBussionessStandEOPage bussPage = new SarBussionessStandEOPage(); + page.setSyncParam("sync"); + ResponseMessage buss = resetBussStandSearchCenter(bussPage,bussSearch); if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){ r.add(buss.getData().toString()); }