diff --git a/adc-da-main/src/main/resources/application-dev.properties b/adc-da-main/src/main/resources/application-dev.properties index a4aa4373..1e3d58ff 100644 --- a/adc-da-main/src/main/resources/application-dev.properties +++ b/adc-da-main/src/main/resources/application-dev.properties @@ -11,7 +11,7 @@ spring.datasource.password = Fting&8g35g#geg2 #============================================== # 应用设置 -spring.application.name=FotonLAWSSystem +spring.application.name=FotonLAWSSystem-REST-SEARCH application.code=20200101 application.center=1 #============================================== diff --git a/adc-da-main/src/main/resources/application.properties b/adc-da-main/src/main/resources/application.properties index 7a7db19f..2bd567fb 100644 --- a/adc-da-main/src/main/resources/application.properties +++ b/adc-da-main/src/main/resources/application.properties @@ -9,7 +9,7 @@ spring.profiles.active=dev server.compression.enabled=true server.compression.mime-types=application/json,application/xml,text/html,text/plain,text/css,application/x-javascript # 端口号设置 -server.port=4202 +server.port=10086 #主服务session超时 server.servlet.session.timeout =600 diff --git a/adc-da-main/src/main/resources/logback-spring.xml b/adc-da-main/src/main/resources/logback-spring.xml index 39f90eab..b1319cf0 100644 --- a/adc-da-main/src/main/resources/logback-spring.xml +++ b/adc-da-main/src/main/resources/logback-spring.xml @@ -8,7 +8,7 @@ - + diff --git a/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java b/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java index baedb9e1..41563bb6 100644 --- a/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java +++ b/adc-da-search/src/main/java/com/adc/da/search/service/ElasticsearchService.java @@ -118,6 +118,11 @@ public interface ElasticsearchService { */ Map searchDataById(String index, String type, String id, String fields); + /** + * 使用索引查询所有 + */ + + List> searchAll(String index); /** * 使用分词查询 diff --git a/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java b/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java index 0b86db69..76f1f3c5 100644 --- a/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java +++ b/adc-da-search/src/main/java/com/adc/da/search/service/impl/ElasticsearchServiceImpl.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsRequest; import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; +import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsRequestBuilder; import org.elasticsearch.action.bulk.BulkRequest; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; @@ -24,11 +25,16 @@ import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; import org.elasticsearch.client.transport.TransportClient; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.search.Scroll; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.AggregationBuilders; import org.elasticsearch.search.aggregations.BucketOrder; import org.elasticsearch.search.aggregations.bucket.range.Range; @@ -36,6 +42,7 @@ import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.Avg; import org.elasticsearch.search.aggregations.metrics.Max; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.SortOrder; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -273,6 +280,54 @@ public class ElasticsearchServiceImpl implements ElasticsearchService { return getResponse.getSource(); } + /** + * 使用索引查询所有 + */ + @Override + public List> searchAll(String index) { + List> sourceList = new ArrayList>(); + + //1、指定es集群 cluster.name 是固定的key值，my-application是ES集群的名称 +// Settings settings = Settings.builder().put("cluster.name", "my-application").build(); + QueryBuilder qBuilder = QueryBuilders.matchAllQuery(); + SearchResponse sResponse = client.prepareSearch(index) + .setQuery(qBuilder).setTrackTotalHits(true) + .get(); + SearchHits hits = sResponse.getHits(); + if(hits.getTotalHits().value > 0){ + SearchResponse scrollResp = search(index,qBuilder, 1,(int) hits.getTotalHits().value); + for (SearchHit hit : scrollResp.getHits().getHits()) { + sourceList.add(hit.getSourceAsMap()); + } + } + return sourceList; + } + + + public SearchResponse search(String index, QueryBuilder query,int page, int size) { + + updateIndex(index, page,size); + + SearchResponse searchResponse = client.prepareSearch(index) + .setScroll(new TimeValue(360000)) + .setQuery(query).setTrackTotalHits(true).setSize(size) + .get(); + return searchResponse; + } + + //更新索引的max_result_window参数 + private boolean updateIndex(String indices, int from,int size) { + int records = from * size + size; + if (records <= 10000) return true; + AcknowledgedResponse indexResponse = client.admin().indices() + .prepareUpdateSettings(indices) + .setSettings(Settings.builder() + .put("index.max_result_window", records) + .build() + ).get(); + return indexResponse.isAcknowledged(); + } + /** * 使用分词查询 diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java index 2c26cdc9..2f76d925 100644 --- a/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java +++ b/adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java @@ -18,6 +18,7 @@ import com.adc.da.util.http.ResponseMessage; import com.adc.da.util.http.Result; import com.adc.da.utils.util.DateUtil; import com.alibaba.fastjson.JSONObject; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; import com.google.gson.JsonObject; import org.apache.commons.lang3.StringUtils; import org.elasticsearch.client.transport.TransportClient; @@ -91,6 +92,55 @@ public class ResetSearchCenterService { return Result.success(res); } + @Async + public ResponseMessage syncResetALLSearchCenter(SearchCenter searchCenter) throws Exception{ + Boolean getFulltextserchIndex = elasticsearchService.isIndexExist("fulltextserch"); + SearchCenter standSearch = new SearchCenter(); + QueryWrapper qw = new QueryWrapper(); + qw.eq("VALID_FLAG","0"); + int count = iSarStandardsInfoService.count(qw); + standSearch.setExecType(searchCenter.getExecType()); + standSearch.setCountStand(count); + SarStandardsInfoEOPage page = new SarStandardsInfoEOPage(); + page.setSyncParam("sync"); + ResponseMessage stand = resetStandSearchCenter(page,standSearch); + List r = new ArrayList<>(); + if(stand.isOk() && StringUtils.isNotBlank(stand.getData().toString())){ + r.add(stand.getData().toString()); + } + + SearchCenter lawsSearch = new SearchCenter(); + QueryWrapper qw2 = new QueryWrapper(); + qw2.eq("VALID_FLAG","0"); + int count2 = iSarLawsStandInfoService.count(qw2); + lawsSearch.setExecType(searchCenter.getExecType()); + lawsSearch.setCountStand(count2); + SarLawsStandInfoPage lawsPage = new SarLawsStandInfoPage(); + page.setSyncParam("sync"); + ResponseMessage laws = resetLawsSearchCenter(lawsPage,lawsSearch); + if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){ + r.add(laws.getData().toString()); + } + + SearchCenter bussSearch = new SearchCenter(); + QueryWrapper qw3 = new QueryWrapper(); + qw3.eq("VALID_FLAG","0"); + int count3 = iSarBussionessStandService.count(qw3); + bussSearch.setExecType(searchCenter.getExecType()); + bussSearch.setCountStand(count3); + SarBussionessStandEOPage bussPage = new SarBussionessStandEOPage(); + page.setSyncParam("sync"); + ResponseMessage buss = resetBussStandSearchCenter(bussPage,bussSearch); + if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){ + r.add(buss.getData().toString()); + } + + String res = String.join(", ", r); + logger.info(res); + + return Result.success(res); + } + //重建国内外标准 @Async public ResponseMessage resetStandSearchCenter(SarStandardsInfoEOPage page,SearchCenter searchCenter) throws Exception{ @@ -100,12 +150,15 @@ public class ResetSearchCenterService { Boolean getStandIndex = elasticsearchService.isIndexExist("stand"); - List> searchListData = elasticsearchService.searchListData("stand","","",""); + if(!getStandIndex){ + logger.info("ES 不存在索引:stand"); + return Result.error("ES 不存在索引:stand"); + } + + List> searchListData = elasticsearchService.searchAll("stand"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); - logger.info(searchListData.toString()); - logger.info(esIdList.toString()); page.setValidFlag("0"); if(searchCenter.getIdList() != null && !searchCenter.getIdList().isEmpty()){ String[] result = searchCenter.getIdList().toArray(new String[0]); @@ -170,15 +223,20 @@ public class ResetSearchCenterService { elasticsearchService.deleteBatchId(delList,"stand"); elasticsearchService.deleteBatchId(delList,"fulltextserch"); } - logger.info("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）删除-共"+delList.size()+"条数据"); - return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）删除-共"+delList.size()+"条数据"); + logger.info("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+0+"条）删除-共"+delList.size()+"条数据"); + return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+0+"条）删除-共"+delList.size()+"条数据"); } @Async public ResponseMessage resetLawsSearchCenter(SarLawsStandInfoPage sarLawsInfoEOPage,SearchCenter searchCenter) throws Exception{ Boolean getLawsIndex = elasticsearchService.isIndexExist("laws"); - List> searchListData = elasticsearchService.searchListData("laws","","",""); + if(!getLawsIndex){ + logger.info("ES 不存在索引:laws"); + return Result.error("ES 不存在索引:laws"); + } + + List> searchListData = elasticsearchService.searchAll("laws"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); @@ -262,7 +320,12 @@ public class ResetSearchCenterService { public ResponseMessage resetBussStandSearchCenter(SarBussionessStandEOPage sarBussionessStandEOPage,SearchCenter searchCenter) throws Exception{ Boolean getbussstandIndex = elasticsearchService.isIndexExist("bussstand"); - List> searchListData = elasticsearchService.searchListData("bussstand","","",""); + if(!getbussstandIndex){ + logger.info("ES 不存在索引:bussstand"); + return Result.error("ES 不存在索引:bussstand"); + } + + List> searchListData = elasticsearchService.searchAll("bussstand"); List esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList()); diff --git a/adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java b/adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java new file mode 100644 index 00000000..beeef499 --- /dev/null +++ b/adc-da-slrs/src/main/java/com/adc/da/search/sync/restSearchSync.java @@ -0,0 +1,71 @@ +package com.adc.da.search.sync; + +import com.adc.da.search.bean.SearchCenter; +import com.adc.da.search.server.ResetSearchCenterService; +import com.adc.da.slrs.sarBussionessStand.dao.SarBussionessStandDao; +import com.adc.da.slrs.sarBussionessStand.entity.SarBussionessStand; +import com.adc.da.slrs.sarBussionessStandState.entity.SarBussionessStandState; +import com.adc.da.slrs.sarBussionessStandState.service.ISarBussionessStandStateService; +import com.adc.da.utils.util.DateUtil; +import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.scheduling.annotation.Async; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.stream.Collectors; + +/** + * @Description: TODO + * @author: super_liu + * @date: 2022年01月25日 3:34 + */ +@EnableScheduling +@Component +@Slf4j +public class restSearchSync { + Logger logger = LoggerFactory.getLogger(restSearchSync.class); + + @Autowired + private ResetSearchCenterService resetSearchCenterService; + + /** + * 根据配置文件设置是否开启定时器 + */ + + @Value("${isNotScheduled}") + private boolean isNotScheduled; //是否开启定时器 + + + // 每天0点1分执行 重置ES 自动更新、新增、删除 +// @Scheduled(cron="0 0 1 1 * ?") + @Scheduled(cron = "0 1 0 * * ?") + @Async + public void StandScheduledJobMonthBegin(){ + if(isNotScheduled){ + try{ + Thread.sleep(2000); + logger.info("每天0点1分执行 重置ES 自动更新、新增、删除:"+Thread.currentThread().getName() + " cron=0 1 0 * * ? --- " + new Date()+"---START-01"); + + SearchCenter searchCenter = new SearchCenter(); + // ALL 执行全部 + searchCenter.setExecType("ALL"); + resetSearchCenterService.syncResetALLSearchCenter(searchCenter); + + logger.info("每天0点1分执行 重置ES 自动更新、新增、删除:"+Thread.currentThread().getName() + " cron=0 1 0 * * ? --- " + new Date()+"---End-01"); + }catch(Exception e){ + logger.info(e.getMessage()); + } + } + } + +}