文档库--搜索(文件内容)

jero-boot/jero-boot-modules/pom.xml

@@ -32,6 +32,11 @@
     </repositories>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.pdfbox</groupId>
+            <artifactId>pdfbox</artifactId>
+            <version>2.0.6</version>
+        </dependency>
         <dependency>
             <groupId>com.jero.boot</groupId>
             <artifactId>jero-boot-module-system</artifactId>

jero-boot/jero-boot-modules/src/main/java/com/jero/modules/document/service/impl/BussDocumentLibraryEOServiceImpl.java

@@ -29,6 +29,8 @@ import com.jero.modules.document.enums.FieldTypeEnum;
 import com.jero.modules.document.enums.SearchEnum;
 import com.jero.modules.document.mapper.BussDocumentLibraryEOMapper;
 import com.jero.modules.document.service.IBussDocumentLibraryEOService;
+import com.jero.modules.document.utils.ReadPdfUtil;
+import com.jero.modules.document.utils.ReadWordUtil;
 import com.jero.modules.domain.service.DomainUserRelService;
 import com.jero.modules.feishu.service.IFeishuService;
 import com.jero.modules.log.entity.BussLogEO;
@@ -926,8 +928,9 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
         mapFullText.put("module_type", "文档库");
         mapFullText.put("serial_number", map.get("serial_number"));
         mapFullText.put("title", map.get("title"));
-        mapFullText.put("file_text", map.get("文件内容"));
+
         StringBuilder sb = new StringBuilder();
+        List<List<String>> fileTextList = new ArrayList<>();
         try {
             for (Map.Entry<String, Object> entry : map.entrySet()) {
                 String key = entry.getKey();
@@ -961,6 +964,13 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
                             ossFile.setConnectId(valueTemp);
                             oSSFileList.add(ossFile);
                         }
+                        if(oSSFileList.size() != 0){
+                            //读取word或PDF文件内容
+                            List<String> list = readFileContent(oSSFileList);
+                            if(list.size() != 0){
+                                fileTextList.add(list) ;
+                            }
+                        }
                         iOSSFileService.updateFileInfo(oSSFileList);
                         valuesBuilder.append("," + "'" + valueTemp + "'");
                     } else {
@@ -972,6 +982,7 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
         } catch (Exception e) {
             e.printStackTrace();
         }
+        mapFullText.put("file_text", fileTextList);
         mapFullText.put("content", sb.toString());
         String insertField = mustFields + fieldsBuilder.toString();
         String insertValue = mustValues + valuesBuilder.toString();
@@ -999,6 +1010,33 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
         }
     }
 
+    /**
+     * //读取word或PDF文件内容
+     * @param oSSFileList
+     */
+    public List<String> readFileContent(List<OSSFile> oSSFileList){
+        //查询文件详细信息
+        List<String> fileIdList = oSSFileList.stream().map(OSSFile::getId).collect(Collectors.toList());
+        oSSFileList = iOSSFileService.getFileInfos(StringUtils.join(fileIdList, ","));
+        List<String> result = new ArrayList<>();
+        for (OSSFile ossFile : oSSFileList) {
+            if(ossFile.getFileName().endsWith(".doc")
+                    || ossFile.getFileName().endsWith(".docx")
+                    || ossFile.getFileName().endsWith(".DOC")
+                    || ossFile.getFileName().endsWith(".DOCX")){
+                String wordContent = ReadWordUtil.readWord(ossFile.getUrl());
+                if(StringUtils.isNotBlank(wordContent)){
+                    result.add(wordContent);
+                }
+            }else if(ossFile.getFileName().endsWith(".pdf")||ossFile.getFileName().endsWith(".PDF")){
+                String pdfContent = ReadPdfUtil.readPdf(ossFile.getUrl());
+                if(StringUtils.isNotBlank(pdfContent)){
+                    result.add(pdfContent);
+                }
+            }
+        }
+        return result;
+    }
     private void esFullText(String cut, List<OnlCgformField> fieldList, List<SysCategory> categoryList, List<SysDictItem> dictItemList, List<Map<String, Object>> mapList, StringBuilder sb, Map.Entry<String, Object> entry) {
         for (OnlCgformField onlCgformField : fieldList) {
 
@@ -1215,10 +1253,11 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
         StringBuilder valuesBuilder = new StringBuilder();
         StringBuilder sb = new StringBuilder();
         Map<String,Object> mapFullText = new HashMap<>();
+        mapFullText.put("id", id);
         mapFullText.put("module_type", "文档库");
         mapFullText.put("serial_number", map.get("serial_number"));
         mapFullText.put("title", map.get("title"));
-        mapFullText.put("file_text", "文件内容-->废止");
+        List<List<String>> fileTextList = new ArrayList<>();
         for (Map.Entry<String, Object> entry : map.entrySet()) {
             String key = entry.getKey();
             Object value = entry.getValue();
@@ -1255,9 +1294,25 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
                                 ossFile.setConnectId(valueTemp);
                                 oSSFileList.add(ossFile);
                             }
+                            if(oSSFileList.size() != 0){
+                                //读取word或PDF文件内容
+                                List<String> list = readFileContent(oSSFileList);
+                                if(list.size() != 0){
+                                    fileTextList.add(list) ;
+                                }
+                            }
                             iOSSFileService.updateFileInfo(oSSFileList);
                             valuesBuilder.append("'" + valueTemp + "'" + ",");
                         } else {
+                            List<OSSFile> oSSFileList = new ArrayList<>();
+                            OSSFile ossFile = new OSSFile();
+                            ossFile.setId((String) value);
+                            oSSFileList.add(ossFile);
+                            //读取word或PDF文件内容
+                            List<String> list = readFileContent(oSSFileList);
+                            if (list.size() != 0) {
+                                fileTextList.add(list);
+                            }
                             valuesBuilder.append("'" + value + "'" + ",");
                         }
 //                        //判断文件是否修改
@@ -1289,7 +1344,9 @@ public class BussDocumentLibraryEOServiceImpl extends ServiceImpl<BussDocumentLi
                 fieldsBuilder.append(key + ",");
             }
         }
+        mapFullText.put("file_text", fileTextList);
         mapFullText.put("content", sb.toString());
+
         String insertField = fieldsBuilder.substring(0, fieldsBuilder.length() - 1);
         String insertValue = valuesBuilder.substring(0, valuesBuilder.length() - 1);
 

jero-boot/jero-boot-modules/src/main/java/com/jero/modules/document/utils/ReadPdfUtil.java

@@ -0,0 +1,40 @@
+package com.jero.modules.document.utils;
+
+import com.jero.common.exception.JeroBootException;
+import org.apache.pdfbox.pdmodel.PDDocument;
+import org.apache.pdfbox.pdmodel.encryption.InvalidPasswordException;
+import org.apache.pdfbox.text.PDFTextStripper;
+import org.apache.pdfbox.text.PDFTextStripperByArea;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * @description
+ * @date 2022/3/21 17:04
+ * @auth zhn
+ */
+public class ReadPdfUtil {
+
+    public static String readPdf(String path) {
+//        path="E:\\DeskTop\\4444.pdf";
+        try (PDDocument document = PDDocument.load(new File(path))) {
+            document.getClass();
+//            if(!document.isEncrypted()) {
+            PDFTextStripperByArea stripper = new PDFTextStripperByArea();
+            stripper.setSortByPosition(true);
+            PDFTextStripper tStripper = new PDFTextStripper();
+
+            String pdfFileInText = tStripper.getText(document);
+            return pdfFileInText;
+
+//            String[] lines = pdfFileInText.split("\\r?\\n");
+//            for(String line : lines) {
+//                System.out.println(line);
+//            }
+//            }
+        }catch (IOException e) {
+            throw new JeroBootException("文件内容读取失败");
+        }
+    }
+}

jero-boot/jero-boot-modules/src/main/java/com/jero/modules/document/utils/ReadWordUtil.java

@@ -0,0 +1,45 @@
+package com.jero.modules.document.utils;
+
+import com.jero.common.exception.JeroBootException;
+import org.apache.poi.POIXMLDocument;
+import org.apache.poi.POIXMLTextExtractor;
+import org.apache.poi.hwpf.extractor.WordExtractor;
+import org.apache.poi.openxml4j.opc.OPCPackage;
+import org.apache.poi.xwpf.extractor.XWPFWordExtractor;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+
+/**
+ * @description
+ * @date 2022/3/21 17:04
+ * @auth zhn
+ */
+public class ReadWordUtil {
+
+    public static String readWord(String path) {
+//        String path = "E:\\DeskTop\\1111.doc";
+//        path = "E:\\DeskTop\\22222.docx";
+//        String path = "E:\\DeskTop\\标准所ISO-用户手册.doc";
+        String str = "";
+        try {
+            if (path.endsWith(".doc")) {
+                InputStream is = new FileInputStream(new File(path));
+                WordExtractor ex = new WordExtractor(is);
+                str = ex.getText();
+                ex.close();
+            } else if (path.endsWith("docx")) {
+                OPCPackage opcPackage = POIXMLDocument.openPackage(path);
+                POIXMLTextExtractor extractor = new XWPFWordExtractor(opcPackage);
+                str = extractor.getText();
+                extractor.close();
+            } else {
+                throw new JeroBootException("此文件不是word文件");
+            }
+            return str;
+        } catch (Exception e) {
+            throw new JeroBootException("文件内容读取失败");
+        }
+    }
+}

jero-boot/jero-boot-modules/src/main/java/com/jero/modules/searchcenter/service/impl/DocumentSearchServiceImpl.java

@@ -71,6 +71,10 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
 
     @Override
     public IPage getInfoList(Map<String, Object> map) {
+        boolean flagTemp = jeroElasticsearchTemplate.indexExists(SearchEnum.INDEX_NAME_DOCUMENT.getValue());
+        if(!flagTemp){
+            return new Page(Integer.parseInt(map.get("pageNo").toString()), Integer.parseInt(map.get("pageSize").toString()));
+        }
         //文档库所有的字段
         List<OnlCgformField> fieldList = onlCgformFieldService.getFieldList(ModuleEnum.DOCUMENT_LIBRARY.getValue());
         //需要查询的列表字段
@@ -275,6 +279,12 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
 //            map2.put("content",selectValue);
 //            queryMap.put("match", map2);
 //        }
+        //判断索引是否存在
+        boolean flagTemp = jeroElasticsearchTemplate.indexExists(SearchEnum.FULL_TEXT_SEARCH.getValue());
+        if(!flagTemp){
+            return new Page(searchVO.getPage(), searchVO.getPageSize());
+        }
+
         String selectValue = searchVO.getSelectValue();
         String selectValueTwo = searchVO.getSelectValueTwo();
         //测试数据
@@ -313,10 +323,14 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
                 searchVO.getPageSize());
 //        JSONObject queryObject11 = jeroElasticsearchTemplate.buildQuery(null, jsonObject11,highlightMap,null, Integer.parseInt(map.get("pageNo").toString()), Integer.parseInt(map.get("pageSize").toString()));
         //2. 数据查询
-        JSONObject search = jeroElasticsearchTemplate.search("fulltextsearch", "fulltextsearch", queryObject);
+        JSONObject search = jeroElasticsearchTemplate.search(SearchEnum.FULL_TEXT_SEARCH.getValue(),
+                                                             SearchEnum.FULL_TEXT_SEARCH.getValue(),
+                                                             queryObject);
 
         //全部的数据(调试接口用--最后删除即可)
-        JSONObject searchAll = jeroElasticsearchTemplate.search("fulltextsearch", "fulltextsearch", new JSONObject());
+        JSONObject searchAll = jeroElasticsearchTemplate.search(SearchEnum.FULL_TEXT_SEARCH.getValue(),
+                                                                SearchEnum.FULL_TEXT_SEARCH.getValue(),
+                                                                new JSONObject());
 
 
         List<Map<String, Object>> list = (List<Map<String, Object>>) (((Map) search.get("hits")).get("hits"));
@@ -331,6 +345,7 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
             list = list.stream().filter(e -> searchVO.getIds().contains((String) e.get("id"))).collect(Collectors.toList());
         }
 
+        List<Map<String, Object>> listNew = new ArrayList<>();
         for (Map<String, Object> stringObjectMap : list) {
             Map<String, Object> mapSource = (Map<String, Object>) stringObjectMap.get("_source");
             if (StringUtils.isNotBlank(selectValueTwo)) {
@@ -338,7 +353,12 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
                 int count = 0;
                 for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
                     String key = entry.getKey();
-                    String value = (String) entry.getValue();
+                    String value = null;
+                    try {
+                        value = (String) entry.getValue();
+                    } catch (Exception e) {
+                        listNew.add(stringObjectMap);
+                    }
                     //判断value中是否包含二次搜索中的字符
                     boolean flag = judgeContains(value, selectValueTwo);
                     if (flag) {
@@ -376,9 +396,18 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
 //            }
                 //-----------------------------------------------------------------------------
                 //处理高亮module_type,title,serial_number,file_text
+
                 for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
                     String key = entry.getKey();
-                    String value = (String) entry.getValue();
+                    String value = null;
+                    try {
+                        //文件内容为字符串格式
+                        value = (String) entry.getValue();
+                    } catch (Exception e) {
+                        //文件内容为集合格式
+                        listNew.add(stringObjectMap);
+                        e.printStackTrace();
+                    }
                     StringBuilder sb = new StringBuilder();
                     if (StringUtils.isNotBlank(value)) {
                         for (int i = 0; i < value.length(); i++) {
@@ -390,10 +419,105 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
                             sb.append(String.valueOf(chStr));
                         }
                     }
-                    entry.setValue(sb.toString());
+                    if(ObjectUtils.isNotEmpty(sb)){
+                        entry.setValue(sb.toString());
+                    }
                 }
             }
         }
+
+        //若一条标准含有多条发布稿或报批稿等文件,需要对每个附件进行搜索并多行展示,
+        //例如:GB 7258中含有两个发布稿,并且两个发布稿都有搜索内容,则GB 7258在列表中应该展示两行,
+        //展示不同发布稿的正文内容
+        List<Map<String, Object>> result = new ArrayList<>();
+        List<String> idList = new ArrayList<>();
+        for (Map<String, Object> map : listNew) {
+            idList.add((String) map.get("_id"));
+            Map<String, Object> mapSource = (Map<String, Object>) map.get("_source");
+            String moduleType = (String) mapSource.get("module_type");
+            String serialNumber = (String) mapSource.get("serial_number");
+            String title = (String) mapSource.get("title");
+            String content = (String) mapSource.get("content");
+            String id = (String) mapSource.get("id");
+            int count = 0;
+            if(StringUtils.isNotBlank(selectValueTwo)){
+                //二次查询
+                for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
+                    String key = entry.getKey();
+                    if("file_text".equals(key)){
+                        for (List<String> valueList : (List<List<String>>) entry.getValue()) {
+                            for (String s : valueList) {
+                                StringBuilder sb = new StringBuilder();
+                                if (StringUtils.isNotBlank(s)) {
+                                    //判断value中是否包含二次搜索中的字符
+                                    boolean flag = judgeContains(s, selectValueTwo);
+                                    if(flag){
+                                        for (int i = 0; i < s.length(); i++) {
+                                            String chStr = String.valueOf(s.charAt(i));
+                                            if (selectValue.contains(String.valueOf(chStr))) {
+                                                chStr = "<span class='highlight-class'>" + chStr + "</span>";
+                                                count ++;
+                                            }
+                                            sb.append(String.valueOf(chStr));
+                                        }
+                                    }
+                                }
+                                if(count > 0){
+                                    Map<String,Object> mapTemp = new HashMap<>();
+                                    mapTemp.put("id",id);
+                                    mapTemp.put("module_type",moduleType);
+                                    mapTemp.put("serial_number",serialNumber);
+                                    mapTemp.put("title",title);
+                                    mapTemp.put("content",content);
+                                    mapTemp.put("file_text",sb.toString());
+                                    result.add(mapTemp);
+                                }
+                            }
+                        }
+                    }
+                }
+            }else{
+                //首次查询
+                for (Map.Entry<String, Object> entry : mapSource.entrySet()) {
+                    String key = entry.getKey();
+                    if("file_text".equals(key)){
+                        for (List<String> valueList : (List<List<String>>) entry.getValue()) {
+                            for (String s : valueList) {
+                                StringBuilder sb = new StringBuilder();
+                                if (StringUtils.isNotBlank(s)) {
+                                    for (int i = 0; i < s.length(); i++) {
+                                        String chStr = String.valueOf(s.charAt(i));
+                                        if (selectValue.contains(String.valueOf(chStr))) {
+                                            chStr = "<span class='highlight-class'>" + chStr + "</span>";
+                                            count ++;
+                                        }
+                                        sb.append(String.valueOf(chStr));
+                                    }
+                                }
+                                if(count > 0){
+                                    Map<String,Object> mapTemp = new HashMap<>();
+                                    mapTemp.put("id",id);
+                                    mapTemp.put("module_type",moduleType);
+                                    mapTemp.put("serial_number",serialNumber);
+                                    mapTemp.put("title",title);
+                                    mapTemp.put("content",content);
+                                    mapTemp.put("file_text",sb.toString());
+                                    result.add(mapTemp);
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+        int a=0;
+        //在原始数据中删除idList,再将result填入到原始数据中
+        if(result.size() != 0){
+            mapList.removeIf(e->idList.contains(e.get("id")));
+            mapList.addAll(result);
+        }
+
+
         //处理分页
         IPage page = new Page(searchVO.getPage(), searchVO.getPageSize());
         if (StringUtils.isNotBlank(selectValueTwo)) {
@@ -449,6 +573,7 @@ public class DocumentSearchServiceImpl implements IDocumentSearchService {
     @Override
     public void deleteAll() {
         jeroElasticsearchTemplate.removeIndex(SearchEnum.INDEX_NAME_DOCUMENT.getValue());
+        jeroElasticsearchTemplate.removeIndex(SearchEnum.FULL_TEXT_SEARCH.getValue());
     }