Merge remote-tracking branch 'origin/develop_master' into develop_master

This commit is contained in:
zj
2022-05-04 17:27:26 +08:00
6 changed files with 161 additions and 90 deletions
@@ -121,6 +121,7 @@ public class WebMvcConfig implements WebMvcConfigurer {
// 重置ES
addInterceptor.excludePathPatterns("/api/search/resetSearchCenter/resetALLSearchCenter");
addInterceptor.excludePathPatterns("/api/search/resetSearchCenter/manualResetALLSearchCenter");
addInterceptor.excludePathPatterns("/api/search/resetSearchCenter/resetAStandSearchCenter");
addInterceptor.excludePathPatterns("/api/search/resetSearchCenter/resetBLawsSearchCenter");
addInterceptor.excludePathPatterns("/api/search/resetSearchCenter/resetCBussStandSearchCenter");
@@ -80,11 +80,11 @@ public class SearchCenterServiceImpl implements SearchCenterService {
BoolQueryBuilder boolQueryShould = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryMust = QueryBuilders.boolQuery();
updateIndex(searchInfoEO.getSelectIndex());
searchRequestBuilder = client.prepareSearch(searchInfoEO.getSelectIndex())
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setFrom((searchInfoEO.getPage()-1)*searchInfoEO.getPageSize())
.addSort("issueTime", SortOrder.DESC)
.addSort("issue_time_data", SortOrder.DESC)
.setSize(searchInfoEO.getPageSize());
if(searchInfoEO.getSelectIndex().equals("stand")) {
@@ -207,11 +207,18 @@ public class SearchCenterServiceImpl implements SearchCenterService {
BoolQueryBuilder boolQueryShould = QueryBuilders.boolQuery();
BoolQueryBuilder boolQueryMust = QueryBuilders.boolQuery();
updateIndex(searchInfoEO.getSelectIndex());
String sortField = "";
if("laws".equals(searchInfoEO.getSelectIndex())){
sortField = "lawsYear";
}else {
sortField = "standYear";
}
searchRequestBuilder = client.prepareSearch(searchInfoEO.getSelectIndex())
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.setFrom((searchInfoEO.getPage()-1)*searchInfoEO.getPageSize())
.addSort("issueTime", SortOrder.DESC)
.addSort(sortField, SortOrder.DESC)
.addSort("issue_time_data", SortOrder.DESC)
.setSize(searchInfoEO.getPageSize());
if(searchInfoEO.getSelectIndex().equals("stand")) {
@@ -322,7 +329,34 @@ public class SearchCenterServiceImpl implements SearchCenterService {
//在结果中检索
if (StringUtils.isNotEmpty(searchInfoEO.getSelectValue()) && !isRqFormat(searchInfoEO.getSelectValue())) {
QueryBuilder multiQuery = queryInResult(searchInfoEO);
boolQueryBuilder.must(multiQuery);
BoolQueryBuilder boolSelectValue = new BoolQueryBuilder();
boolSelectValue.should(multiQuery);
switch (searchInfoEO.getSelectIndex()){
case "stand":
boolSelectValue
.should(QueryBuilders.wildcardQuery("standSort.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1005f))
.should(QueryBuilders.wildcardQuery("standNumber.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1003f))
.should(QueryBuilders.wildcardQuery("standYear.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1002f))
.should(QueryBuilders.wildcardQuery("stand_name.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1000f));
break;
case "laws":
boolSelectValue
.should(QueryBuilders.wildcardQuery("lawsType.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1005f))
.should(QueryBuilders.wildcardQuery("lawsNumber.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1003f))
.should(QueryBuilders.wildcardQuery("lawsYear.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1002f))
.should(QueryBuilders.wildcardQuery("lawsName.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1001f));
break;
case "bussstand":
boolSelectValue
.should(QueryBuilders.wildcardQuery("standSort.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1005f))
.should(QueryBuilders.wildcardQuery("standNumber.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1003f))
.should(QueryBuilders.wildcardQuery("standYear.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1002f))
.should(QueryBuilders.wildcardQuery("stand_name.keyword", "*" + searchInfoEO.getSelectValue().toUpperCase() + "*").boost(1000f));
break;
default: break;
}
boolQueryBuilder.must(boolSelectValue);
}
searchRequestBuilder.setQuery(boolQueryBuilder).setTrackTotalHits(true);
@@ -764,10 +798,10 @@ public class SearchCenterServiceImpl implements SearchCenterService {
BoolQueryBuilder boolQueryBuilder = new BoolQueryBuilder();
List<String> fieldList = getStringField(searchInfoEO);
List<String> fieldNameList = new ArrayList<>();
String[] standStr = {"content","standSort","standNumber","standYear","stand_name","standEnName","standNatureName",
String[] standStr = {"content","standEnName","standNatureName",
"textStatusName","isRelateAccessName","numbershow","standSystem","NYLXCLASS","CLLX"};
String[] lawsStr = {"content","lawsType","lawsNumber","lawsName","lawsEnName","lawsNo","issueCompany","lawsTextState","lawsSyqy","lawsSycx","isRelateAccess","lawsYear","lawsNotisyncNum","lawsBulletin","lawsLabel","lawsRemark"};
String[] bussstandStr = {"content","standSort","stand_code","standYear","stand_name","standEnName","nameshow","standstateshow","numbershow","textStatusBuss","textStatusBussName","NYLXCLASS","CLLX"};
String[] lawsStr = {"content","lawsEnName","lawsNo","issueCompany","lawsTextState","lawsSyqy","lawsSycx","isRelateAccess","lawsNotisyncNum","lawsBulletin","lawsLabel","lawsRemark"};
String[] bussstandStr = {"content","stand_code","standEnName","nameshow","standstateshow","numbershow","textStatusBuss","textStatusBussName","NYLXCLASS","CLLX"};
for (String field : fieldList) {
// 关键字查询属性字段
String keyWordField2 = field + "Name";
@@ -779,7 +813,7 @@ public class SearchCenterServiceImpl implements SearchCenterService {
listStrings.addAll(fieldNameList);
String[] fieldArr = listStrings.stream().toArray(String[]::new);
multiQueryBuilder = QueryBuilders.multiMatchQuery(searchInfoEO.getSelectValue(),fieldArr
).minimumShouldMatch("100%");
).minimumShouldMatch("100%").boost(1L);
// boolQueryBuilder.should();
// for (String field : fieldList) {
// // 关键字查询属性字段
@@ -792,14 +826,14 @@ public class SearchCenterServiceImpl implements SearchCenterService {
listStringLaws.addAll(fieldNameList);
String[] fieldLawsArr = listStringLaws.stream().toArray(String[]::new);
multiQueryBuilder = QueryBuilders.multiMatchQuery(searchInfoEO.getSelectValue(),fieldLawsArr
).minimumShouldMatch("100%");
).minimumShouldMatch("100%").boost(1L);
break;
case "bussstand":
List<String> listStringBussstand = Stream.of(bussstandStr).collect(Collectors.toList());
listStringBussstand.addAll(fieldNameList);
String[] fieldBussstandArr = listStringBussstand.stream().toArray(String[]::new);
multiQueryBuilder = QueryBuilders.multiMatchQuery(searchInfoEO.getSelectValue(),fieldBussstandArr
).minimumShouldMatch("100%");
).minimumShouldMatch("100%").boost(1L);
break;
default: break;
}
@@ -1221,10 +1255,11 @@ public class SearchCenterServiceImpl implements SearchCenterService {
hiBuilder.field(highlightDate);
hiBuilder.preTags("<span class='highlight-class'>");
hiBuilder.postTags("</span>");
updateIndex("fulltextserch",(searchInfoEO.getPage()-1)*searchInfoEO.getPageSize(),searchInfoEO.getPageSize());
updateIndex("fulltextserch");
searchRequestBuilder = client.prepareSearch("fulltextserch")
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.highlighter(hiBuilder)
.addSort("issue_time_order",SortOrder.DESC)
.setFrom((searchInfoEO.getPage()-1)*searchInfoEO.getPageSize())
.setSize(searchInfoEO.getPageSize());
@@ -1515,7 +1550,7 @@ public class SearchCenterServiceImpl implements SearchCenterService {
hiBuilder.preTags("<span class='highlight-class'>");
hiBuilder.postTags("</span>");
updateIndex("fulltextserch",(searchInfoEO.getPage()-1)*searchInfoEO.getPageSize(),searchInfoEO.getPageSize());
updateIndex("fulltextserch");
searchRequestBuilder = client.prepareSearch("fulltextserch")
.setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
.highlighter(hiBuilder)
@@ -1580,14 +1615,13 @@ public class SearchCenterServiceImpl implements SearchCenterService {
}
//更新索引的max_result_window参数
private boolean updateIndex(String indices, int from,int size) {
int records = from * size + size;
if (records <= 10000) return true;
private boolean updateIndex(String indices) {
AcknowledgedResponse indexResponse = client.admin().indices()
.prepareUpdateSettings(indices)
.setSettings(Settings.builder()
.put("index.max_result_window", records)
.put("highlight.max_analyzed_offset", records)
.put("index.max_result_window", 100000000)
.put("highlight.max_analyzed_offset", 100000000)
.put("index.highlight.max_analyzed_offset", 100000000)
.build()
).get();
return indexResponse.isAcknowledged();
@@ -80,7 +80,10 @@ public class SendStandMQService {
}
}
String sarStandAttrEOStr = infoEO.getSarStandAttrEOStr();
Map<String,Object> attrInfoMap = JSONObject.fromObject(sarStandAttrEOStr);
Map<String,Object> attrInfoMap = new TreeMap<>();
if(StringUtils.isNotBlank(sarStandAttrEOStr)){
attrInfoMap = JSONObject.fromObject(sarStandAttrEOStr);
}
Map<String,Object> infoMap = infoEO.getAttrInfoMap();
Map<String,String> infoEOMapItems = infoEO.getMapItems();
@@ -178,15 +181,25 @@ public class SendStandMQService {
// 动态存储属性字段
for (String field : fieldInfoList) {
saveMap.put(field,attrInfoMap.getOrDefault(field,""));
String fieldValue = "";
if(infoMap.get(field) != null && StringUtils.isNotBlank(infoMap.get(field).toString())){
Object json= new JSONTokener(infoMap.get(field).toString()).nextValue();
if(!json.toString().equals("null")){
fieldValue = infoMap.get(field).toString();
String fieldValue = "";
try {
saveMap.put(field,attrInfoMap.getOrDefault(field,""));
if(infoMap.get(field) != null && StringUtils.isNotBlank(infoMap.get(field).toString())){
Object json= new JSONTokener(infoMap.get(field).toString()).nextValue();
if(json instanceof org.json.JSONArray){
fieldValue = String.join(",", com.alibaba.fastjson.JSONObject.parseArray(standMap.get(field).toString(),String.class));
}else {
if(!("null").equals(json)){
fieldValue = standMap.get(field).toString();
}
}
}
saveMap.put(field+"Name",fieldValue);
}catch (Exception e){
logger.info("国内外标准 消息队列: "+addOrUpdate+", 标准ID:"+infoEO.getId()+",问题字段:"+field);
saveMap.put(field+"Name",fieldValue);
}
saveMap.put(field+"Name",fieldValue);
}
if(infoEOMapItems != null && !infoEOMapItems.isEmpty()){
@@ -39,6 +39,17 @@ public class ResetSearchCenterController extends BaseController<Map<String, Obje
return resetSearchCenterService.resetALLSearchCenter(searchCenter);
}
@ApiOperation(value = "|SearchCenter|手动重新创建(---7天内---)ALL标准数据索引(凌晨有自动程序会跑)")
@PostMapping(value="/manualResetALLSearchCenter")
public ResponseMessage manualResetALLSearchCenter(@RequestBody SearchCenter searchCenter) throws Exception{
ResponseMessage x = verifySearchCenter(searchCenter);
if (x != null){
return x;
}
return resetSearchCenterService.syncResetALLSearchCenter(searchCenter);
}
private ResponseMessage verifySearchCenter(SearchCenter searchCenter) {
if(searchCenter.getExecType() == null || "".equals(searchCenter.getExecType())){
return Result.error("执行参数 ALL 存在更新 不存在新增 ,UPD 只 更新存在 es 里的标准 ,ADD 只新增不存在 es 的标准, DEL 只删除不存在标准库的数据");
@@ -33,7 +33,6 @@ import com.adc.da.utils.util.FieldConvertUtil;
import com.adc.da.utils.util.InitStandAttrUtil;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.google.gson.JsonObject;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.transport.TransportClient;
import org.json.JSONTokener;
@@ -165,7 +164,7 @@ public class ResetSearchCenterService {
SearchCenter lawsSearch = new SearchCenter();
lawsSearch.setExecType(searchCenter.getExecType());
SarLawsStandInfoPage lawsPage = new SarLawsStandInfoPage();
page.setSyncParam("sync");
lawsPage.setSyncParam("sync");
ResponseMessage laws = resetLawsSearchCenter(lawsPage,lawsSearch);
if(laws.isOk() && StringUtils.isNotBlank(laws.getData().toString())){
r.add(laws.getData().toString());
@@ -174,7 +173,7 @@ public class ResetSearchCenterService {
SearchCenter bussSearch = new SearchCenter();
bussSearch.setExecType(searchCenter.getExecType());
SarBussionessStandEOPage bussPage = new SarBussionessStandEOPage();
page.setSyncParam("sync");
bussPage.setSyncParam("sync");
ResponseMessage buss = resetBussStandSearchCenter(bussPage,bussSearch);
if(buss.isOk() && StringUtils.isNotBlank(buss.getData().toString())){
r.add(buss.getData().toString());
@@ -3012,47 +3012,50 @@ public class SarStandardsInfoServiceImpl extends ServiceImpl<SarStandardsInfoDao
String[] attidlist = String.valueOf(attrInfoMap.get("content")).split(" , ");
StringBuilder content = new StringBuilder("");
for (int i = 0; i < attidlist.length; i++) {
if (org.apache.commons.lang.StringUtils.isNotEmpty(attidlist[i])) {
if (StringUtils.isNotEmpty(attidlist[i]) && !"null".equals(attidlist[i]) && attidlist[i].contains("att_file")) {
AttFileEO attFileEO = attFileEOService.getFileInfo(attidlist[i]);
String readFilePath = filePath + "/" + attFileEO.getFilePath() + attFileEO.getFileName();
// String readFilePath = "E:\\上汽企标流程总览.pdf";
try {
// 判断文件是docx还是PDF
if (null != attFileEO.getFileSuffix() && (attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("doc") || attFileEO.getFileSuffix().equals("DOC") || attFileEO.getFileSuffix().equals("DOCX") || attFileEO.getFileSuffix().equals("ppt") || attFileEO.getFileSuffix().equals("pptx"))) {
if (attFileEO.getFileSuffix().equals("doc") || attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("DOC") || attFileEO.getFileSuffix().equals("DOCX")) {
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(readFilePath)));
if (FileMagic.valueOf(bis) == FileMagic.OOXML) {
XWPFDocument doc = new XWPFDocument(bis);
XWPFWordExtractor extractor = new XWPFWordExtractor(doc);
File existsFile = new File(readFilePath);
if(existsFile.exists()){
// 判断文件是docx还是PDF
if (null != attFileEO.getFileSuffix() && (attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("doc") || attFileEO.getFileSuffix().equals("DOC") || attFileEO.getFileSuffix().equals("DOCX") || attFileEO.getFileSuffix().equals("ppt") || attFileEO.getFileSuffix().equals("pptx"))) {
if (attFileEO.getFileSuffix().equals("doc") || attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("DOC") || attFileEO.getFileSuffix().equals("DOCX")) {
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(readFilePath)));
if (FileMagic.valueOf(bis) == FileMagic.OOXML) {
XWPFDocument doc = new XWPFDocument(bis);
XWPFWordExtractor extractor = new XWPFWordExtractor(doc);
String txt = extractor.getText();
content.append(txt);
extractor.close();
}else {
WordExtractor wordExtractor = new WordExtractor(bis);
String txt = wordExtractor.getText();
content.append(txt);
wordExtractor.close();
}
bis.close();
} else {
OPCPackage opcPackage = XWPFDocument.openPackage(readFilePath);
XWPFWordExtractor extractor = new XWPFWordExtractor(opcPackage);
String txt = extractor.getText();
content.append(txt);
extractor.close();
}else {
WordExtractor wordExtractor = new WordExtractor(bis);
String txt = wordExtractor.getText();
content.append(txt);
wordExtractor.close();
opcPackage.close();
}
bis.close();
} else {
OPCPackage opcPackage = XWPFDocument.openPackage(readFilePath);
XWPFWordExtractor extractor = new XWPFWordExtractor(opcPackage);
String txt = extractor.getText();
content.append(txt);
opcPackage.close();
PDDocument document = PDDocument.load(new File(readFilePath));
document.getClass();
if (!document.isEncrypted()) {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
// String[] lines = pdfFileInText.split("\\r?\\n");
content.append(pdfFileInText);
}
document.close();
}
} else {
PDDocument document = PDDocument.load(new File(readFilePath));
document.getClass();
if (!document.isEncrypted()) {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
// String[] lines = pdfFileInText.split("\\r?\\n");
content.append(pdfFileInText);
}
document.close();
}
} catch (Exception e) {
logger.error("搜索中心新增数据读取文档内容时失败");
@@ -3083,43 +3086,53 @@ public class SarStandardsInfoServiceImpl extends ServiceImpl<SarStandardsInfoDao
String[] attidlist = String.valueOf(attrInfoMap.get("content")).split(",");
StringBuilder content = new StringBuilder("");
for (int i = 0; i < attidlist.length; i++) {
if (StringUtils.isNotEmpty(attidlist[i]) && !"null".equals(attidlist[i])) {
if (StringUtils.isNotEmpty(attidlist[i]) && !"null".equals(attidlist[i]) && attidlist[i].contains("att_file")) {
AttFileEO attFileEO = attFileEOService.getFileInfo(attidlist[i]);
if (null == attFileEO) {
continue;
}
String readFilePath = filePath + "/" + attFileEO.getFilePath() + attFileEO.getFileName();
try {
// 判断文件是docx还是PDF
if (attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("doc")) {
if (attFileEO.getFileSuffix().equals("doc")) {
FileInputStream fis = new FileInputStream(readFilePath);
WordExtractor wordExtractor = new WordExtractor(fis);
String txt = wordExtractor.getText();
content.append(txt);
File existsFile = new File(readFilePath);
if(existsFile.exists()){
// 判断文件是docx还是PDF
if (null != attFileEO.getFileSuffix() && (attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("doc") || attFileEO.getFileSuffix().equals("DOC") || attFileEO.getFileSuffix().equals("DOCX") || attFileEO.getFileSuffix().equals("ppt") || attFileEO.getFileSuffix().equals("pptx"))) {
if (attFileEO.getFileSuffix().equals("doc") || attFileEO.getFileSuffix().equals("docx") || attFileEO.getFileSuffix().equals("DOC") || attFileEO.getFileSuffix().equals("DOCX")) {
BufferedInputStream bis = new BufferedInputStream(new FileInputStream(new File(readFilePath)));
if (FileMagic.valueOf(bis) == FileMagic.OOXML) {
XWPFDocument doc = new XWPFDocument(bis);
XWPFWordExtractor extractor = new XWPFWordExtractor(doc);
String txt = extractor.getText();
content.append(txt);
extractor.close();
}else {
WordExtractor wordExtractor = new WordExtractor(bis);
String txt = wordExtractor.getText();
content.append(txt);
wordExtractor.close();
}
bis.close();
} else {
OPCPackage opcPackage = XWPFDocument.openPackage(readFilePath);
XWPFWordExtractor extractor = new XWPFWordExtractor(opcPackage);
String txt = extractor.getText();
content.append(txt);
opcPackage.close();
}
} else {
OPCPackage opcPackage = XWPFDocument.openPackage(readFilePath);
XWPFWordExtractor extractor = new XWPFWordExtractor(opcPackage);
String txt = extractor.getText();
content.append(txt);
}
} else if (attFileEO.getFileSuffix().equals("pdf")) {
PDDocument document = PDDocument.load(new File(readFilePath));
document.getClass();
if (!document.isEncrypted()) {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
// String[] lines = pdfFileInText.split("\\r?\\n");
content.append(pdfFileInText);
PDDocument document = PDDocument.load(new File(readFilePath));
document.getClass();
if (!document.isEncrypted()) {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
// String[] lines = pdfFileInText.split("\\r?\\n");
content.append(pdfFileInText);
}
document.close();
}
}
// String readFilePath = "E:\\上汽企标流程总览.pdf";
// FileInputStream fis = new FileInputStream(readFilePath);
// WordExtractor wordExtractor = new WordExtractor(fis);
// String txt = wordExtractor.getText();
// content.append(txt);
} catch (Exception e) {
logger.error("搜索中心修改数据读取文档内容时失败");
logger.error(e.getMessage(), e);