FT/foton-slrs-system-rest
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: There is no way the, 跑跑跑 跑数据 重置es

Browse Source
This commit is contained in:
super_liu
2022-03-08 16:33:33 +08:00
parent f1ac00a8dc
commit c1eeea3f1f
19 changed files with 2190 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
adc-da-slrs/src/main/java/com/adc/da/search/ResetSearchCenterController.java
+39 -11
View File
@@ -1,19 +1,20 @@
package com.adc.da.search;
import com.adc.da.base.web.BaseController;
import com.adc.da.search.bean.SearchCenter;
import com.adc.da.search.server.ResetSearchCenterService;
import com.adc.da.slrs.sarLawsInfo.page.SarLawsInfoEOPage;
import com.adc.da.slrs.sarLawsStandInfo.entity.SarLawsStandInfoPage;
import com.adc.da.slrs.sarStandardsInfo.entity.SarBussionessStandEOPage;
import com.adc.da.slrs.sarStandardsInfo.entity.SarStandardsInfoEOPage;
import com.adc.da.util.http.ResponseMessage;
import com.adc.da.util.http.Result;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.bind.annotation.*;
import java.util.List;
import java.util.Map;
@RestController
@@ -25,28 +26,55 @@ public class ResetSearchCenterController extends BaseController<Map<String, Obje
private ResetSearchCenterService resetSearchCenterService;
@ApiOperation(value = "|SearchCenter|重新创建ALL标准数据索引")
@PostMapping(value="/resetALLSearchCenter")
public ResponseMessage resetALLSearchCenter() throws Exception{
return resetSearchCenterService.resetALLSearchCenter();
public ResponseMessage resetALLSearchCenter(@RequestBody SearchCenter searchCenter) throws Exception{
ResponseMessage x = verifySearchCenter(searchCenter);
if (x != null){
return x;
}
return resetSearchCenterService.resetALLSearchCenter(searchCenter);
}
private ResponseMessage verifySearchCenter(SearchCenter searchCenter) {
if(searchCenter.getExecType() == null || "".equals(searchCenter.getExecType())){
return Result.error("执行参数 ALL 存在更新 不存在新增 ,UPD 只 更新存在 es 里的标准 ,ADD 只新增不存在 es 的标准");
}
if(searchCenter.getCountStand() == null || searchCenter.getCountStand() == 0){
return Result.error("分页总数不可为空和0");
}
return null;
}
@ApiOperation(value = "|SearchCenter|重新创建国内外标准数据索引")
@PostMapping(value="/resetAStandSearchCenter")
public ResponseMessage resetStandSearchCenter() throws Exception{
public ResponseMessage resetStandSearchCenter(@RequestBody SearchCenter searchCenter) throws Exception{
ResponseMessage x = verifySearchCenter(searchCenter);
if (x != null){
return x;
}
SarStandardsInfoEOPage sarStandardsInfoEOPage = new SarStandardsInfoEOPage();
return resetSearchCenterService.resetStandSearchCenter(sarStandardsInfoEOPage);
return resetSearchCenterService.resetStandSearchCenter(sarStandardsInfoEOPage,searchCenter);
}
@ApiOperation(value = "|SearchCenter|重新创建国内外政策数据索引")
@PostMapping(value="/resetBLawsSearchCenter")
public ResponseMessage resetLawsSearchCenter() throws Exception{
public ResponseMessage resetLawsSearchCenter(@RequestBody SearchCenter searchCenter) throws Exception{
ResponseMessage x = verifySearchCenter(searchCenter);
if (x != null){
return x;
}
SarLawsStandInfoPage sarLawsInfoEOPage = new SarLawsStandInfoPage();
return resetSearchCenterService.resetLawsSearchCenter(sarLawsInfoEOPage);
return resetSearchCenterService.resetLawsSearchCenter(sarLawsInfoEOPage,searchCenter);
}
@ApiOperation(value = "|SearchCenter|重新创建企业标准数据索引")
@PostMapping(value="/resetCBussStandSearchCenter")
public ResponseMessage resetBussStandSearchCenter() throws Exception{
public ResponseMessage resetBussStandSearchCenter(@RequestBody SearchCenter searchCenter) throws Exception{
ResponseMessage x = verifySearchCenter(searchCenter);
if (x != null){
return x;
}
SarBussionessStandEOPage sarBussionessStandEOPage = new SarBussionessStandEOPage();
return resetSearchCenterService.resetBussStandSearchCenter(sarBussionessStandEOPage);
return resetSearchCenterService.resetBussStandSearchCenter(sarBussionessStandEOPage,searchCenter);
}
}
adc-da-slrs/src/main/java/com/adc/da/search/bean/SearchCenter.java
+29
View File
@@ -0,0 +1,29 @@
package com.adc.da.search.bean;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;
import javax.validation.constraints.NotEmpty;
import javax.validation.constraints.NotNull;
import java.util.List;
/**
* @Description: TODO
* @author: super_liu
* @date: 2022年03月08日 14:21
*/
@Data
public class SearchCenter {
@NotNull
@ApiModelProperty(value = "分页查询总数")
private Integer countStand;
@NotNull
@ApiModelProperty(value = "执行参数 ALL 存在更新 不存在新增 ,UPD 只 更新存在 es 里的标准 ,ADD 只新增不存在 es 的标准")
private String execType;
@NotNull
@ApiModelProperty(value = "指定可执行的标准ID集合")
private List<String> idList;
}
adc-da-slrs/src/main/java/com/adc/da/search/server/ResetSearchCenterService.java
+189 -78
View File
@@ -1,6 +1,7 @@
package com.adc.da.search.server;
import com.adc.da.mq.CreateStandMQService;
import com.adc.da.search.bean.SearchCenter;
import com.adc.da.search.service.ElasticsearchService;
import com.adc.da.slrs.sarBussionessStand.entity.SarBussionessStand;
import com.adc.da.slrs.sarBussionessStand.service.ISarBussionessStandService;
@@ -15,6 +16,7 @@ import com.adc.da.slrs.sarStandardsInfo.entity.SarStandardsInfoEOPage;
import com.adc.da.slrs.sarStandardsInfo.service.ISarStandardsInfoService;
import com.adc.da.util.http.ResponseMessage;
import com.adc.da.util.http.Result;
import com.adc.da.utils.util.DateUtil;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
@@ -27,9 +29,7 @@ import org.springframework.transaction.annotation.Propagation;
import org.springframework.transaction.annotation.Transactional;
import javax.annotation.PostConstruct;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;
@Service("resetSearchCenterService")
@@ -65,26 +65,24 @@ public class ResetSearchCenterService {
}
@Async
public ResponseMessage resetALLSearchCenter() throws Exception{
public ResponseMessage resetALLSearchCenter(SearchCenter searchCenter) throws Exception{
Boolean getFulltextserchIndex = elasticsearchService.isIndexExist("fulltextserch");
if(getFulltextserchIndex){
Boolean deleteFulltextserchIndex = elasticsearchService.deleteIndex("fulltextserch");
}
ResponseMessage stand = resetStandSearchCenter(new SarStandardsInfoEOPage());
ResponseMessage stand = resetStandSearchCenter(new SarStandardsInfoEOPage(),searchCenter);
List<String> r = new ArrayList<>();
if(stand.isOk() && StringUtils.isNotBlank(stand.getData().toString())){
r.add(stand.getData().toString());
}
ResponseMessage laws = resetLawsSearchCenter(new SarLawsStandInfoPage());
if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){
r.add(laws.getData().toString());
}
ResponseMessage buss = resetBussStandSearchCenter(new SarBussionessStandEOPage());
if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){
r.add(buss.getData().toString());
}
// ResponseMessage laws = resetLawsSearchCenter(new SarLawsStandInfoPage(),searchCenter);
// if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){
// r.add(laws.getData().toString());
// }
//
// ResponseMessage buss = resetBussStandSearchCenter(new SarBussionessStandEOPage(),searchCenter);
// if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){
// r.add(buss.getData().toString());
// }
String res = String.join(", ", r);
@@ -92,23 +90,65 @@ public class ResetSearchCenterService {
}
//重建国内外标准
@Async
public ResponseMessage resetStandSearchCenter(SarStandardsInfoEOPage page) throws Exception{
public ResponseMessage resetStandSearchCenter(SarStandardsInfoEOPage page,SearchCenter searchCenter) throws Exception{
int countUpdateSuccess = 0;
int countAddSuccess = 0;
Boolean getStandIndex = elasticsearchService.isIndexExist("stand");
if(getStandIndex){
Boolean deleteStandIndex = elasticsearchService.deleteIndex("stand");
}
List<Map<String, Object>> searchListData = elasticsearchService.searchListData("stand","","","");
List<String> esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList());
logger.info(searchListData.toString());
logger.info(esIdList.toString());
page.setValidFlag("0");
// int countStand = sarStandardsInfoEODao.getSarStandardsInfoCount(page);
int countStand = 10000;
int countSuccess = 0;
if(countStand>0){
page.setPageSize(countStand);
page.setMenuId("nomenu");
page.setStandType("ALL");
List<SarStandardsInfo> rowsStand = iSarStandardsInfoService.getSarStandardsInfoPage(page);
for(SarStandardsInfo sarStandardsInfoEO : rowsStand){
countSuccess++;
if(searchCenter.getIdList() != null && !searchCenter.getIdList().isEmpty()){
String[] result = searchCenter.getIdList().toArray(new String[0]);
page.setIdlist(result);
}
page.setPageSize(searchCenter.getCountStand());
page.setMenuId("nomenu");
page.setStandType("ALL");
List<SarStandardsInfo> rowsStand = iSarStandardsInfoService.getSarStandardsInfoPage(page);
List<SarStandardsInfo> insList = new ArrayList<>();
List<SarStandardsInfo> updList = new ArrayList<>();
if(!esIdList.isEmpty()){
// 差集 (list1 - list2) 新增
List<SarStandardsInfo> distinctByUniqueList = rowsStand.stream()
.filter(item -> !new ArrayList<>(esIdList)
.contains(item.getId()))
.collect(Collectors.toList());
insList.addAll(distinctByUniqueList);
// 交集 更新
List<SarStandardsInfo> intersection = rowsStand.stream()
.filter(item -> new ArrayList<>(esIdList)
.contains(item.getId()))
.collect(Collectors.toList());
updList.addAll(intersection);
}else {
// 新增
insList.addAll(rowsStand);
}
if(!updList.isEmpty() && ("ALL".equals(searchCenter.getExecType()) || "UPD".equals(searchCenter.getExecType()))){
for(SarStandardsInfo sarStandardsInfoEO : updList){
countUpdateSuccess++;
if (sarStandardsInfoEO.getAttrInfoMap() != null) {
sarStandardsInfoEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarStandardsInfoEO.getAttrInfoMap()));
}
createStandMQService.sendStandMQ(sarStandardsInfoEO,"update");
}
}
if(!insList.isEmpty() && ("ALL".equals(searchCenter.getExecType()) || "ADD".equals(searchCenter.getExecType()))){
for(SarStandardsInfo sarStandardsInfoEO : insList){
countAddSuccess++;
if (sarStandardsInfoEO.getAttrInfoMap() != null) {
sarStandardsInfoEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarStandardsInfoEO.getAttrInfoMap()));
}
@@ -116,25 +156,62 @@ public class ResetSearchCenterService {
}
}
return Result.success("重置国内外："+ countSuccess + "条");
return Result.success("重置国内外标准：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外标准共："+rowsStand.size()+"条）");
}
@Async
public ResponseMessage resetLawsSearchCenter(SarLawsStandInfoPage sarLawsInfoEOPage) throws Exception{
public ResponseMessage resetLawsSearchCenter(SarLawsStandInfoPage sarLawsInfoEOPage,SearchCenter searchCenter) throws Exception{
Boolean getLawsIndex = elasticsearchService.isIndexExist("laws");
if(getLawsIndex){
Boolean deleteLawsIndex = elasticsearchService.deleteIndex("laws");
}
List<Map<String, Object>> searchListData = elasticsearchService.searchListData("stand","","","");
List<String> esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList());
sarLawsInfoEOPage.setLawsType("FOREIGN");
// sarLawsInfoEOPage.setMenuId("6bc1632eb6354d4fb725");
// int countLaws = sarLawsInfoEODao.queryByCount(sarLawsInfoEOPage);
int countLaws = 10000;
int countSuccess = 0;
if (countLaws>0) {
sarLawsInfoEOPage.setPageSize(countLaws);
List<SarLawsStandInfo> rowsLaws = iSarLawsStandInfoService.getSarStandardsInfoPage(sarLawsInfoEOPage);
for(SarLawsStandInfo sarLawsInfoEO : rowsLaws){
countSuccess++;
// page idlist 赋值
idListFunc(sarLawsInfoEOPage, searchCenter);
int countUpdateSuccess = 0;
int countAddSuccess = 0;
sarLawsInfoEOPage.setPageSize(searchCenter.getCountStand());
List<SarLawsStandInfo> rowsStand = iSarLawsStandInfoService.getSarStandardsInfoPage(sarLawsInfoEOPage);
List<SarLawsStandInfo> insList = new ArrayList<>();
List<SarLawsStandInfo> updList = new ArrayList<>();
if(!esIdList.isEmpty()){
// 差集 (list1 - list2) 新增
List<SarLawsStandInfo> distinctByUniqueList = rowsStand.stream()
.filter(item -> !new ArrayList<>(esIdList)
.contains(item.getId()))
.collect(Collectors.toList());
insList.addAll(distinctByUniqueList);
// 交集 更新
List<SarLawsStandInfo> intersection = rowsStand.stream()
.filter(item -> new ArrayList<>(esIdList)
.contains(item.getId()))
.collect(Collectors.toList());
updList.addAll(intersection);
}else {
// 新增
insList.addAll(rowsStand);
}
if(!updList.isEmpty() && ("ALL".equals(searchCenter.getExecType()) || "UPD".equals(searchCenter.getExecType()))){
for(SarLawsStandInfo sarLawsInfoEO : updList){
countUpdateSuccess++;
if (sarLawsInfoEO.getAttrInfoMap() != null) {
sarLawsInfoEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarLawsInfoEO.getAttrInfoMap()));
}
createStandMQService.sendLawsMQ(sarLawsInfoEO,"update");
}
}
if(!insList.isEmpty() && ("ALL".equals(searchCenter.getExecType()) || "ADD".equals(searchCenter.getExecType()))){
for(SarLawsStandInfo sarLawsInfoEO : insList){
countAddSuccess++;
if (sarLawsInfoEO.getAttrInfoMap() != null) {
sarLawsInfoEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarLawsInfoEO.getAttrInfoMap()));
}
@@ -142,47 +219,81 @@ public class ResetSearchCenterService {
}
}
return Result.success("重置国内外政策："+ countSuccess + "条");
return Result.success("重置国内外政策：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（国内外政策共："+rowsStand.size()+"条）");
}
private void idListFunc(SarLawsStandInfoPage sarLawsInfoEOPage, SearchCenter searchCenter) {
if(searchCenter.getIdList() != null && !searchCenter.getIdList().isEmpty()){
String[] result = searchCenter.getIdList().toArray(new String[0]);
sarLawsInfoEOPage.setIdlist(result);
}
}
@Async
public ResponseMessage resetBussStandSearchCenter(SarBussionessStandEOPage sarBussionessStandEOPage) throws Exception{
public ResponseMessage resetBussStandSearchCenter(SarBussionessStandEOPage sarBussionessStandEOPage,SearchCenter searchCenter) throws Exception{
Boolean getbussstandIndex = elasticsearchService.isIndexExist("bussstand");
int number = 0;
if(getbussstandIndex){
Boolean deletebussstandIndex = elasticsearchService.deleteIndex("bussstand");
}
List<SarBussionessStand> getAllList = new ArrayList<>();
List<Map<String, Object>> searchListData = elasticsearchService.searchListData("stand","","","");
List<String> esIdList = searchListData.stream().map(stringObjectMap -> stringObjectMap.get("id").toString()).collect(Collectors.toList());
sarBussionessStandEOPage.setValidFlag("0");
sarBussionessStandEOPage.setOrderBy("SAR_BUSSIONESS_STAND.issue_time is null,SAR_BUSSIONESS_STAND.issue_time desc,SAR_BUSSIONESS_STAND.id");
sarBussionessStandEOPage.setMenuRoleList(null);
sarBussionessStandEOPage.setMenuId("nomenu");
int countBuss = 10000;
int countSuccess = 0;
if(countBuss>0){
sarBussionessStandEOPage.setMenuId("0");
sarBussionessStandEOPage.setPageSize(countBuss);
getAllList = iSarBussionessStandService.getSarBussionStandPage(sarBussionessStandEOPage);
number = getAllList.size();
if(searchCenter.getIdList() != null && !searchCenter.getIdList().isEmpty()){
String[] result = searchCenter.getIdList().toArray(new String[0]);
sarBussionessStandEOPage.setIdlist(result);
}
if (getAllList != null && !getAllList.isEmpty()) {
sendESForBussStand(getAllList,countSuccess);
}
return Result.success("重置企标："+ number + "条");
}
@Async
public void sendESForBussStand(List<SarBussionessStand> getAllList,int countSuccess) throws Exception {
for(SarBussionessStand sarBussionessStandEO : getAllList){
countSuccess++;
Map<String,Object> attrInfoMap = sarBussionessStandEO.getAttrInfoMap();
if (null != attrInfoMap && attrInfoMap.containsKey("QBMJ") && null != attrInfoMap.get("QBMJ") && String.valueOf(attrInfoMap.get("QBMJ")).contains("ACF675Z88W")){
countSuccess--;
}
if (sarBussionessStandEO.getAttrInfoMap() != null) {
sarBussionessStandEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarBussionessStandEO.getAttrInfoMap()));
}
createStandMQService.sendBussStandMQ(sarBussionessStandEO,"add");
int countUpdateSuccess = 0;
int countAddSuccess = 0;
sarBussionessStandEOPage.setMenuId("0");
sarBussionessStandEOPage.setPageSize(searchCenter.getCountStand());
List<SarBussionessStand> rowsStand = iSarBussionessStandService.getSarBussionStandPage(sarBussionessStandEOPage);
List<SarBussionessStand> insList = new ArrayList<>();
List<SarBussionessStand> updList = new ArrayList<>();
if(!esIdList.isEmpty()){
// 差集 (list1 - list2) 新增
List<SarBussionessStand> distinctByUniqueList = rowsStand.stream()
.filter(item -> !new ArrayList<>(esIdList)
.contains(item.getId()))
.collect(Collectors.toList());
insList.addAll(distinctByUniqueList);
// 交集 更新
List<SarBussionessStand> intersection = rowsStand.stream()
.filter(item -> new ArrayList<>(esIdList)
.contains(item.getId()))
.collect(Collectors.toList());
updList.addAll(intersection);
}else {
// 新增
insList.addAll(rowsStand);
}
if(!updList.isEmpty() && ("ALL".equals(searchCenter.getExecType()) || "UPD".equals(searchCenter.getExecType()))){
for(SarBussionessStand sarBussionessStandEO : updList){
countUpdateSuccess++;
if (sarBussionessStandEO.getAttrInfoMap() != null) {
sarBussionessStandEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarBussionessStandEO.getAttrInfoMap()));
}
createStandMQService.sendBussStandMQ(sarBussionessStandEO,"update");
}
}
if(!insList.isEmpty() && ("ALL".equals(searchCenter.getExecType()) || "ADD".equals(searchCenter.getExecType()))){
for(SarBussionessStand sarBussionessStandEO : insList){
countAddSuccess++;
if (sarBussionessStandEO.getAttrInfoMap() != null) {
sarBussionessStandEO.setSarStandAttrEOStr(JSONObject.toJSONString(sarBussionessStandEO.getAttrInfoMap()));
}
createStandMQService.sendBussStandMQ(sarBussionessStandEO,"add");
}
}
return Result.success("重置企标：新增-"+ countAddSuccess + "条 更新-"+countUpdateSuccess+"条（企标共："+rowsStand.size()+"条）");
}
}