WL/laws_weilai
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

文档库--搜索(文件内容)

Browse Source
This commit is contained in:
zhn
2022-03-22 10:21:42 +08:00
parent 5ef53dfc82
commit 9befc4e57e
5 changed files with 279 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
jero-boot/jero-boot-modules/pom.xml
+5
View File
@@ -32,6 +32,11 @@
</repositories>
<dependencies>
<dependency>
<groupId>org.apache.pdfbox</groupId>
<artifactId>pdfbox</artifactId>
<version>2.0.6</version>
</dependency>
<dependency>
<groupId>com.jero.boot</groupId>
<artifactId>jero-boot-module-system</artifactId>
jero-boot/jero-boot-modules/src/main/java/com/jero/modules/document/service/impl/BussDocumentLibraryEOServiceImpl.java
+59 -2
View File
@@ -29,6 +29,8 @@ import com.jero.modules.document.enums.FieldTypeEnum;
import com.jero.modules.document.enums.SearchEnum;
import com.jero.modules.document.mapper.BussDocumentLibraryEOMapper;
import com.jero.modules.document.service.IBussDocumentLibraryEOService;
import com.jero.modules.document.utils.ReadPdfUtil;
import com.jero.modules.document.utils.ReadWordUtil;
import com.jero.modules.domain.service.DomainUserRelService;
import com.jero.modules.feishu.service.IFeishuService;
import com.jero.modules.log.entity.BussLogEO;
@@ -926,8 +928,9 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
mapFullText.put("module_type", "文档库");
mapFullText.put("serial_number", map.get("serial_number"));
mapFullText.put("title", map.get("title"));
mapFullText.put("file_text", map.get("文件内容"));
StringBuilder sb = new StringBuilder();
List<List<String>> fileTextList = new ArrayList<>();
try {
for (Map.Entry<String, Object> entry : map.entrySet()) {
String key = entry.getKey();
@@ -961,6 +964,13 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
ossFile.setConnectId(valueTemp);
oSSFileList.add(ossFile);
}
if(oSSFileList.size() != 0){
//读取word或PDF文件内容
List<String> list = readFileContent(oSSFileList);
if(list.size() != 0){
fileTextList.add(list) ;
}
}
iOSSFileService.updateFileInfo(oSSFileList);
valuesBuilder.append("," + "'" + valueTemp + "'");
} else {
@@ -972,6 +982,7 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
} catch (Exception e) {
e.printStackTrace();
}
mapFullText.put("file_text", fileTextList);
mapFullText.put("content", sb.toString());
String insertField = mustFields + fieldsBuilder.toString();
String insertValue = mustValues + valuesBuilder.toString();
@@ -999,6 +1010,33 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
}
}
/**
* //读取word或PDF文件内容
* @param oSSFileList
*/
public List<String> readFileContent(List<OSSFile> oSSFileList){
//查询文件详细信息
List<String> fileIdList = oSSFileList.stream().map(OSSFile::getId).collect(Collectors.toList());
oSSFileList = iOSSFileService.getFileInfos(StringUtils.join(fileIdList, ","));
List<String> result = new ArrayList<>();
for (OSSFile ossFile : oSSFileList) {
if(ossFile.getFileName().endsWith(".doc")
|| ossFile.getFileName().endsWith(".docx")
|| ossFile.getFileName().endsWith(".DOC")
|| ossFile.getFileName().endsWith(".DOCX")){
String wordContent = ReadWordUtil.readWord(ossFile.getUrl());
if(StringUtils.isNotBlank(wordContent)){
result.add(wordContent);
}
}else if(ossFile.getFileName().endsWith(".pdf")||ossFile.getFileName().endsWith(".PDF")){
String pdfContent = ReadPdfUtil.readPdf(ossFile.getUrl());
if(StringUtils.isNotBlank(pdfContent)){
result.add(pdfContent);
}
}
}
return result;
}
private void esFullText(String cut, List<OnlCgformField> fieldList, List<SysCategory> categoryList, List<SysDictItem> dictItemList, List<Map<String, Object>> mapList, StringBuilder sb, Map.Entry<String, Object> entry) {
for (OnlCgformField onlCgformField : fieldList) {
@@ -1215,10 +1253,11 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
StringBuilder valuesBuilder = new StringBuilder();
StringBuilder sb = new StringBuilder();
Map<String,Object> mapFullText = new HashMap<>();
mapFullText.put("id", id);
mapFullText.put("module_type", "文档库");
mapFullText.put("serial_number", map.get("serial_number"));
mapFullText.put("title", map.get("title"));
mapFullText.put("file_text", "文件内容-->废止");
List<List<String>> fileTextList = new ArrayList<>();
for (Map.Entry<String, Object> entry : map.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
@@ -1255,9 +1294,25 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
ossFile.setConnectId(valueTemp);
oSSFileList.add(ossFile);
}
if(oSSFileList.size() != 0){
//读取word或PDF文件内容
List<String> list = readFileContent(oSSFileList);
if(list.size() != 0){
fileTextList.add(list) ;
}
}
iOSSFileService.updateFileInfo(oSSFileList);
valuesBuilder.append("'" + valueTemp + "'" + ",");
} else {
List<OSSFile> oSSFileList = new ArrayList<>();
OSSFile ossFile = new OSSFile();
ossFile.setId((String) value);
oSSFileList.add(ossFile);
//读取word或PDF文件内容
List<String> list = readFileContent(oSSFileList);
if (list.size() != 0) {
fileTextList.add(list);
}
valuesBuilder.append("'" + value + "'" + ",");
}
// //判断文件是否修改
@@ -1289,7 +1344,9 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
fieldsBuilder.append(key + ",");
}
}
mapFullText.put("file_text", fileTextList);
mapFullText.put("content", sb.toString());
String insertField = fieldsBuilder.substring(0, fieldsBuilder.length() - 1);
String insertValue = valuesBuilder.substring(0, valuesBuilder.length() - 1);
jero-boot/jero-boot-modules/src/main/java/com/jero/modules/document/utils/ReadPdfUtil.java
+40
View File
@@ -0,0 +1,40 @@
package com.jero.modules.document.utils;
import com.jero.common.exception.JeroBootException;
import org.apache.pdfbox.pdmodel.PDDocument;
import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException;
import org.apache.pdfbox.text.PDFTextStripper;
import org.apache.pdfbox.text.PDFTextStripperByArea;
import java.io.File;
import java.io.IOException;
/**
* @description
* @date 2022/3/21 17:04
* @auth zhn
*/
public class ReadPdfUtil {
public static String readPdf(String path) {
// path="E:\\DeskTop\\4444.pdf";
try (PDDocument document = PDDocument.load(new File(path))) {
document.getClass();
// if(!document.isEncrypted()) {
PDFTextStripperByArea stripper = new PDFTextStripperByArea();
stripper.setSortByPosition(true);
PDFTextStripper tStripper = new PDFTextStripper();
String pdfFileInText = tStripper.getText(document);
return pdfFileInText;
// String[] lines = pdfFileInText.split("\\r?\\n");
// for(String line : lines) {
// System.out.println(line);
// }
// }
}catch (IOException e) {
throw new JeroBootException("文件内容读取失败");
}
}
}
jero-boot/jero-boot-modules/src/main/java/com/jero/modules/document/utils/ReadWordUtil.java
+45
View File
@@ -0,0 +1,45 @@
package com.jero.modules.document.utils;
import com.jero.common.exception.JeroBootException;
import org.apache.poi.POIXMLDocument;
import org.apache.poi.POIXMLTextExtractor;
import org.apache.poi.hwpf.extractor.WordExtractor;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
/**
* @description
* @date 2022/3/21 17:04
* @auth zhn
*/
public class ReadWordUtil {
public static String readWord(String path) {
// String path = "E:\\DeskTop\\1111.doc";
// path = "E:\\DeskTop\\22222.docx";
// String path = "E:\\DeskTop\\标准所ISO-用户手册.doc";
String str = "";
try {
if (path.endsWith(".doc")) {
InputStream is = new FileInputStream(new File(path));
WordExtractor ex = new WordExtractor(is);
str = ex.getText();
ex.close();
} else if (path.endsWith("docx")) {
OPCPackage opcPackage = POIXMLDocument.openPackage(path);
POIXMLTextExtractor extractor = new XWPFWordExtractor(opcPackage);
str = extractor.getText();
extractor.close();
} else {
throw new JeroBootException("此文件不是word文件");
}
return str;
} catch (Exception e) {
throw new JeroBootException("文件内容读取失败");
}
}
}
jero-boot/jero-boot-modules/src/main/java/com/jero/modules/searchcenter/service/impl/DocumentSearchServiceImpl.java
+130 -5
View File
@@ -71,6 +71,10 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
@Override
public IPage getInfoList(Map<String, Object> map) {
boolean flagTemp = jeroElasticsearchTemplate.indexExists(SearchEnum.INDEX_NAME_DOCUMENT.getValue());
if(!flagTemp){
return new Page(Integer.parseInt(map.get("pageNo").toString()), Integer.parseInt(map.get("pageSize").toString()));
}
//文档库所有的字段
List<OnlCgformField> fieldList = onlCgformFieldService.getFieldList(ModuleEnum.DOCUMENT_LIBRARY.getValue());
//需要查询的列表字段
@@ -275,6 +279,12 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
// map2.put("content",selectValue);
// queryMap.put("match", map2);
// }
//判断索引是否存在
boolean flagTemp = jeroElasticsearchTemplate.indexExists(SearchEnum.FULL_TEXT_SEARCH.getValue());
if(!flagTemp){
return new Page(searchVO.getPage(), searchVO.getPageSize());
}
String selectValue = searchVO.getSelectValue();
String selectValueTwo = searchVO.getSelectValueTwo();
//测试数据
@@ -313,10 +323,14 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
searchVO.getPageSize());
// JSONObject queryObject11 = jeroElasticsearchTemplate.buildQuery(null, jsonObject11,highlightMap,null, Integer.parseInt(map.get("pageNo").toString()), Integer.parseInt(map.get("pageSize").toString()));
//2. 数据查询
JSONObject search = jeroElasticsearchTemplate.search("fulltextsearch", "fulltextsearch", queryObject);
JSONObject search = jeroElasticsearchTemplate.search(SearchEnum.FULL_TEXT_SEARCH.getValue(),
SearchEnum.FULL_TEXT_SEARCH.getValue(),
queryObject);
//全部的数据(调试接口用--最后删除即可)
JSONObject searchAll = jeroElasticsearchTemplate.search("fulltextsearch", "fulltextsearch", new JSONObject());
JSONObject searchAll = jeroElasticsearchTemplate.search(SearchEnum.FULL_TEXT_SEARCH.getValue(),
SearchEnum.FULL_TEXT_SEARCH.getValue(),
new JSONObject());
List<Map<String, Object>> list = (List<Map<String, Object>>) (((Map) search.get("hits")).get("hits"));
@@ -331,6 +345,7 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
list = list.stream().filter(e -> searchVO.getIds().contains((String) e.get("id"))).collect(Collectors.toList());
}
List<Map<String, Object>> listNew = new ArrayList<>();
for (Map<String, Object> stringObjectMap : list) {
Map<String, Object> mapSource = (Map<String, Object>) stringObjectMap.get("_source");
if (StringUtils.isNotBlank(selectValueTwo)) {
@@ -338,7 +353,12 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
int count = 0;
for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
String key = entry.getKey();
String value = (String) entry.getValue();
String value = null;
try {
value = (String) entry.getValue();
} catch (Exception e) {
listNew.add(stringObjectMap);
}
//判断value中是否包含二次搜索中的字符
boolean flag = judgeContains(value, selectValueTwo);
if (flag) {
@@ -376,9 +396,18 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
// }
//-----------------------------------------------------------------------------
//处理高亮module_type,title,serial_number,file_text
for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
String key = entry.getKey();
String value = (String) entry.getValue();
String value = null;
try {
//文件内容为字符串格式
value = (String) entry.getValue();
} catch (Exception e) {
//文件内容为集合格式
listNew.add(stringObjectMap);
e.printStackTrace();
}
StringBuilder sb = new StringBuilder();
if (StringUtils.isNotBlank(value)) {
for (int i = 0; i < value.length(); i++) {
@@ -390,10 +419,105 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
sb.append(String.valueOf(chStr));
}
}
entry.setValue(sb.toString());
if(ObjectUtils.isNotEmpty(sb)){
entry.setValue(sb.toString());
}
}
}
}
//若一条标准含有多条发布稿或报批稿等文件,需要对每个附件进行搜索并多行展示,
//例如:GB 7258中含有两个发布稿,并且两个发布稿都有搜索内容,则GB 7258在列表中应该展示两行,
//展示不同发布稿的正文内容
List<Map<String, Object>> result = new ArrayList<>();
List<String> idList = new ArrayList<>();
for (Map<String, Object> map : listNew) {
idList.add((String) map.get("_id"));
Map<String, Object> mapSource = (Map<String, Object>) map.get("_source");
String moduleType = (String) mapSource.get("module_type");
String serialNumber = (String) mapSource.get("serial_number");
String title = (String) mapSource.get("title");
String content = (String) mapSource.get("content");
String id = (String) mapSource.get("id");
int count = 0;
if(StringUtils.isNotBlank(selectValueTwo)){
//二次查询
for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
String key = entry.getKey();
if("file_text".equals(key)){
for (List<String> valueList : (List<List<String>>) entry.getValue()) {
for (String s : valueList) {
StringBuilder sb = new StringBuilder();
if (StringUtils.isNotBlank(s)) {
//判断value中是否包含二次搜索中的字符
boolean flag = judgeContains(s, selectValueTwo);
if(flag){
for (int i = 0; i < s.length(); i++) {
String chStr = String.valueOf(s.charAt(i));
if (selectValue.contains(String.valueOf(chStr))) {
chStr = "<span class='highlight-class'>" + chStr + "</span>";
count ++;
}
sb.append(String.valueOf(chStr));
}
}
}
if(count > 0){
Map<String,Object> mapTemp = new HashMap<>();
mapTemp.put("id",id);
mapTemp.put("module_type",moduleType);
mapTemp.put("serial_number",serialNumber);
mapTemp.put("title",title);
mapTemp.put("content",content);
mapTemp.put("file_text",sb.toString());
result.add(mapTemp);
}
}
}
}
}
}else{
//首次查询
for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
String key = entry.getKey();
if("file_text".equals(key)){
for (List<String> valueList : (List<List<String>>) entry.getValue()) {
for (String s : valueList) {
StringBuilder sb = new StringBuilder();
if (StringUtils.isNotBlank(s)) {
for (int i = 0; i < s.length(); i++) {
String chStr = String.valueOf(s.charAt(i));
if (selectValue.contains(String.valueOf(chStr))) {
chStr = "<span class='highlight-class'>" + chStr + "</span>";
count ++;
}
sb.append(String.valueOf(chStr));
}
}
if(count > 0){
Map<String,Object> mapTemp = new HashMap<>();
mapTemp.put("id",id);
mapTemp.put("module_type",moduleType);
mapTemp.put("serial_number",serialNumber);
mapTemp.put("title",title);
mapTemp.put("content",content);
mapTemp.put("file_text",sb.toString());
result.add(mapTemp);
}
}
}
}
}
}
}
int a=0;
//在原始数据中删除idList,再将result填入到原始数据中
if(result.size() != 0){
mapList.removeIf(e->idList.contains(e.get("id")));
mapList.addAll(result);
}
//处理分页
IPage page = new Page(searchVO.getPage(), searchVO.getPageSize());
if (StringUtils.isNotBlank(selectValueTwo)) {
@@ -449,6 +573,7 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
@Override
public void deleteAll() {
jeroElasticsearchTemplate.removeIndex(SearchEnum.INDEX_NAME_DOCUMENT.getValue());
jeroElasticsearchTemplate.removeIndex(SearchEnum.FULL_TEXT_SEARCH.getValue());
}