diff --git a/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/controller/OldSystemController.java b/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/controller/OldSystemController.java index 33594ca6..dd345bdd 100644 --- a/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/controller/OldSystemController.java +++ b/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/controller/OldSystemController.java @@ -72,6 +72,28 @@ public class OldSystemController { oldSystemService.importOldSystemDataFix(file, type, response); } + @PostMapping("/importOldSystemFileFix") + @AutoLog(value = "老法规系统-老法规企标国标外标文件数据修正") + @ApiOperation(value="老法规系统-老法规企标国标外标文件数据修正", notes="老法规系统-老法规企标国标外标文件数据修正", produces = "application/octet-stream") + @ApiOperationSupport(order = 12) + public void importOldSystemDataFileFix(@RequestParam("file") MultipartFile file, + @ApiParam(value = "0不导入 1导入企标 2导入国标 3外标", required = true) + @RequestParam("type") String type, + HttpServletResponse response) { + oldSystemService.importOldSystemDataFileFix(file, type, response); + } + + @PostMapping("/importOldSystemFileFixFinal") + @AutoLog(value = "老法规系统-老法规企标国标外标文件数据最终修正") + @ApiOperation(value="老法规系统-老法规企标国标外标文件数据最终修正", notes="老法规系统-老法规企标国标外标文件数据最终修正", produces = "application/octet-stream") + @ApiOperationSupport(order = 12) + public void importOldSystemFileFixFinal(@RequestParam("file") MultipartFile file, + @ApiParam(value = "0不导入 1导入企标 2导入国标 3外标", required = true) + @RequestParam("type") String type, + HttpServletResponse response) { + oldSystemService.importOldSystemDataFileFixFinal(file, type, response); + } + @PostMapping("/importOldESAuthAllExcel") @AutoLog(value = "老法规系统-老法规企标所有人权限数据导入") @ApiOperation(value="老法规系统-老法规企标所有人权限数据导入", notes="老法规系统-老法规企标所有人权限数据导入", produces = "application/octet-stream") diff --git a/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/OldSystemService.java b/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/OldSystemService.java index 3d8d9cb0..8df5cfdb 100644 --- a/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/OldSystemService.java +++ b/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/OldSystemService.java @@ -89,4 +89,20 @@ public interface OldSystemService { * @param response */ void importOldSystemDataFix(MultipartFile file, String type, HttpServletResponse response); + + /** + * 老法规数据导入和文件修正 + * @param file + * @param type + * @param response + */ + void importOldSystemDataFileFix(MultipartFile file, String type, HttpServletResponse response); + + /** + * 老法规数据导入和文件最终修正 + * @param file + * @param type + * @param response + */ + void importOldSystemDataFileFixFinal(MultipartFile file, String type, HttpServletResponse response); } diff --git a/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/impl/OldSystemServiceImpl.java b/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/impl/OldSystemServiceImpl.java index dabce085..5a3be927 100644 --- a/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/impl/OldSystemServiceImpl.java +++ b/laws-modules/src/main/java/com/jero/modules/laws/oldSystem/service/impl/OldSystemServiceImpl.java @@ -245,6 +245,220 @@ public class OldSystemServiceImpl implements OldSystemService { } } + @Override + public void importOldSystemDataFileFix(MultipartFile file, String type, HttpServletResponse response) { + List errImportList = new ArrayList<>(); + + // 验证文件是否为压缩格式file.getContentType()).contains("xls") + if (!Objects.requireNonNull(file.getOriginalFilename()).contains("xls")) { + throw new JeroBootException("文件格式不正确，请上传Excel。"); + } + ImportParams params = new ImportParams(); + params.setTitleRows(0); + params.setNeedSave(true); + // 获取数据 + List allDatalist = null; + + try { + // 获取当前时间戳（毫秒） + long startTime = System.currentTimeMillis(); + @Cleanup + InputStream inputStream = file.getInputStream(); + allDatalist = ExcelImportUtil.importExcel(inputStream, OldSystemImportDTO.class, params); + // 删除空行 + allDatalist.removeIf(OldSystemImportDTO::isEmptyRow); + log.info("获取数据成功，数据条数：{}", allDatalist.size()); + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("数据转换执行时间：" + executionTime + " 毫秒"); + } catch (Exception e) { + log.error("导入失败；{}", e.getMessage()); + } + if (allDatalist == null) { + throw new JeroBootException("导入失败，数据为空"); + } + // 导入企标 + switch (type) { + case "0": + break; + case "1": + // 导入企标 + this.oldSysESDataFileFix(allDatalist, errImportList); + break; + case "2": + // 导入国标 + this.oldSysGBDataFileFix(allDatalist, errImportList); + break; + case "3": + // 导入外标 + this.oldSysFBDataFileFix(allDatalist, errImportList); + + } + // 统一清理代替被代替重复数据 + lawsReplacedStandardService.clearRepeatData(); + if (!errImportList.isEmpty()) { + this.exportWithExcel(response, errImportList, "OldSystemStandardErrorData.xls"); + } + } + + @Override + public void importOldSystemDataFileFixFinal(MultipartFile file, String type, HttpServletResponse response) { + List errImportList = new ArrayList<>(); + + // 验证文件是否为压缩格式file.getContentType()).contains("xls") + if (!Objects.requireNonNull(file.getOriginalFilename()).contains("xls")) { + throw new JeroBootException("文件格式不正确，请上传Excel。"); + } + ImportParams params = new ImportParams(); + params.setTitleRows(0); + params.setNeedSave(true); + // 获取数据 + List allDatalist = null; + + try { + // 获取当前时间戳（毫秒） + long startTime = System.currentTimeMillis(); + @Cleanup + InputStream inputStream = file.getInputStream(); + allDatalist = ExcelImportUtil.importExcel(inputStream, OldSystemImportDTO.class, params); + // 删除空行 + allDatalist.removeIf(OldSystemImportDTO::isEmptyRow); + log.info("获取数据成功，数据条数：{}", allDatalist.size()); + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("数据转换执行时间：" + executionTime + " 毫秒"); + } catch (Exception e) { + log.error("导入失败；{}", e.getMessage()); + } + if (allDatalist == null) { + throw new JeroBootException("导入失败，数据为空"); + } + // 导入企标 + switch (type) { + case "0": + break; + case "1": + // 导入企标 + this.oldSysESDataFileFixFinal(allDatalist, errImportList); + break; + case "2": + // 导入国标 + this.oldSysGBDataFileFix(allDatalist, errImportList); + break; + case "3": + // 导入外标 + this.oldSysFBDataFileFixFinal(allDatalist, errImportList); + + } + // 统一清理代替被代替重复数据 + lawsReplacedStandardService.clearRepeatData(); + if (!errImportList.isEmpty()) { + this.exportWithExcel(response, errImportList, "OldSystemStandardErrorData.xls"); + } + } + + private void oldSysESDataFileFixFinal(List allDatalist, List errImportList) { + // 过滤字段 根据标准号字段去重 + allDatalist = allDatalist.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() + -> new TreeSet<>(Comparator.comparing(OldSystemImportDTO::getBzh))), ArrayList::new)); + log.info("根据标准号去重后的企标相关数据条数：{}", allDatalist.size()); + long startTime = System.currentTimeMillis(); + // 转换数据 + List esList = esService.list(); + Map esMap = esList.stream() + .collect(Collectors.toMap(LawsEnterpriseStandard::getStandardNumber, o -> o)); + + List ossFileList = new ArrayList<>(); + for (OldSystemImportDTO dto : allDatalist) { + LawsEnterpriseStandard es = esMap.get(dto.getBzh().replace("—", "-")); + if (es == null) { + dto.setErrorInfo("企标数据不存在"); + errImportList.add(dto); + continue; + } + String filePath = dto.getCybzh() + "_" + dto.getBzmc() + ".pdf"; + filePath = filePath.replaceAll("/", "_"); + String path = "Standard/20240307/" + filePath; + if (MinioUtil.doesObjectExist(path)) { + log.info("文件存在"); + // 存入关系表 + OSSFile ossFile = new OSSFile(); + ossFile.setUrl(path); + ossFile.setStandardNo(es.getStandardNumber()); + ossFile.setStandardId(es.getId()); + ossFile.setFileName(FileUtil.getName(path)); + ossFile.setStandardFileType("publish_of_original"); + ossFileList.add(ossFile); + errImportList.add(dto); + } else { + log.info("文件不存在"); + dto.setErrorInfo("文件不存在"); + errImportList.add(dto); + } + } + if (!ossFileList.isEmpty()) { + ossFileService.saveBatch(ossFileList); + } + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("代码执行时间：" + executionTime + " 毫秒"); + } + + private void oldSysFBDataFileFixFinal(List allDatalist, List errImportList) { + // 过滤字段 根据标准号字段去重 + allDatalist = allDatalist.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() + -> new TreeSet<>(Comparator.comparing(OldSystemImportDTO::getBzh))), ArrayList::new)); + log.info("根据标准号去重后的外标相关数据条数：{}", allDatalist.size()); + long startTime = System.currentTimeMillis(); + // 转换数据 + List fbList = overseasStandardService.list(); + Map fbMap = fbList.stream() + .collect(Collectors.toMap(LawsOverseasStandard::getStandardNumber, o -> o)); + + List ossFileList = new ArrayList<>(); + for (OldSystemImportDTO dto : allDatalist) { + LawsOverseasStandard fb = fbMap.get(dto.getBzh().replace("—", "-")); + if (fb == null) { + dto.setErrorInfo("外标数据不存在"); + errImportList.add(dto); + continue; + } + String filePath = dto.getCybzh() + "_" + dto.getBzmc() + ".pdf"; + filePath = filePath.replaceAll("/", "_"); + String path = "Standard/20240307/" + filePath; + if (MinioUtil.doesObjectExist(path)) { + log.info("文件存在"); + // 存入关系表 + OSSFile ossFile = new OSSFile(); + ossFile.setUrl(path); + ossFile.setStandardNo(fb.getStandardNumber()); + ossFile.setStandardId(fb.getId()); + ossFile.setFileName(FileUtil.getName(path)); + ossFile.setStandardFileType("publish_of_original"); + ossFileList.add(ossFile); + errImportList.add(dto); + } else { + log.info("文件不存在"); + dto.setErrorInfo("文件不存在"); + errImportList.add(dto); + } + } + if (!ossFileList.isEmpty()) { + ossFileService.saveBatch(ossFileList); + } + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("代码执行时间：" + executionTime + " 毫秒"); + } + private void oldSysFBDataFix(List allDatalist, List errImportList) { List fbDatalist = allDatalist.stream().filter(o -> o.getBzfl().contains("ISO") || o.getBzfl().contains("引进") || @@ -502,6 +716,160 @@ public class OldSystemServiceImpl implements OldSystemService { log.info("代码执行时间：" + executionTime + " 毫秒"); } + private void oldSysESDataFileFix(List esDatalist, List errImportList) { + esDatalist = esDatalist.stream().filter(o -> o.getBzfl().contains("企业标准")) + .collect(Collectors.toList()); + log.info("企标相关数据条数：{}", esDatalist.size()); + // 首先，使用Map来记录每个标准号出现的次数 + Map frequencyMap = esDatalist.stream() + .collect(Collectors.groupingBy(OldSystemImportDTO::getBzh, Collectors.counting())); + // 然后，过滤掉那些出现次数超过1次的数据 + esDatalist = esDatalist.stream() + .filter(dto -> frequencyMap.get(dto.getBzh()) == 1) + .collect(Collectors.toList()); + log.info("删除掉所有重复的数据后相关数据条数：{}", esDatalist.size()); + List esList = esService.list(); + Map esMap = esList.stream().collect(Collectors.toMap(LawsEnterpriseStandard::getOldSystemId, o -> o)); + List ossFileList = ossFileService.list(); + Map> ossFileMap = ossFileList.stream() + .filter(o -> o.getStandardId() != null) + .collect(Collectors.groupingBy(OSSFile::getStandardId)); + long startTime = System.currentTimeMillis(); + // 根据老法规中的标准号找到数据库中数据 再去ossFile表中找到相应数据，如果没有则将数据存入errImportList + for (OldSystemImportDTO dto : esDatalist) { + if (StrUtil.isBlank(dto.getSclj())) { + dto.setErrorInfo("没有需要导入的文件"); + errImportList.add(dto); + continue; + } + LawsEnterpriseStandard es = esMap.get(dto.getId()); + if (es == null) { + dto.setErrorInfo("未找到对应的企标数据"); + errImportList.add(dto); + continue; + } + List ossFile = ossFileMap.get(es.getId()); + if (ossFile == null || ossFile.isEmpty()) { + dto.setErrorInfo("未找到对应的文件数据"); + errImportList.add(dto); + } + } + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("代码执行时间：" + executionTime + " 毫秒"); + } + + private void oldSysGBDataFileFix(List allDatalist, List errImportList) { + List gbDatalist = allDatalist.stream().filter(o -> + o.getBzfl().contains("国家标准/") || // “/”是为了避免匹配到“美国国家标准”这个外标 + o.getBzfl().contains("团体标准") || + o.getBzfl().contains("地方标准") || + o.getBzfl().contains("行业标准")).collect(Collectors.toList()); + log.info("国标相关数据条数：{}", gbDatalist.size()); + + // 首先，使用Map来记录每个标准号出现的次数 + Map frequencyMap = gbDatalist.stream() + .collect(Collectors.groupingBy(OldSystemImportDTO::getBzh, Collectors.counting())); + + // 然后，过滤掉那些出现次数超过1次的数据 + gbDatalist = gbDatalist.stream() + .filter(dto -> frequencyMap.get(dto.getBzh()) == 1) + .collect(Collectors.toList()); + + log.info("删除掉所有重复的数据后相关数据条数：{}", gbDatalist.size()); + + long startTime = System.currentTimeMillis(); + // 转换数据 + List gbList = gbService.list(); + Map gbMap = gbList.stream().collect(Collectors.toMap(LawsDomesticStandard::getStandardNumber, o -> o)); + List ossFileList = ossFileService.list(); + Map> ossFileMap = ossFileList.stream() + .filter(o -> o.getStandardId() != null) + .collect(Collectors.groupingBy(OSSFile::getStandardId)); + for (OldSystemImportDTO dto : gbDatalist) { + // 先将标准号中的中文横杠替换为英文横杠 + String newBzh = dto.getBzh().replace("—", "-"); + dto.setBzh(newBzh); + } + // 导入企标 + for (OldSystemImportDTO dto : gbDatalist) { + LawsDomesticStandard gb = gbMap.get(dto.getBzh()); + if (StrUtil.isBlank(dto.getSclj())) { + dto.setErrorInfo("没有需要导入的文件"); + errImportList.add(dto); + continue; + } + if (gb == null) { + dto.setErrorInfo("未找到对应的国标数据"); + errImportList.add(dto); + continue; + } + List ossFile = ossFileMap.get(gb.getId()); + if (ossFile == null || ossFile.isEmpty()) { + dto.setErrorInfo("未找到对应的文件数据"); + errImportList.add(dto); + } + } + + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("代码执行时间：" + executionTime + " 毫秒"); + } + + private void oldSysFBDataFileFix(List allDatalist, List errImportList) { + List fbDatalist = allDatalist.stream().filter(o -> o.getBzfl().contains("ISO") || + o.getBzfl().contains("引进") || + o.getBzfl().contains("国外")).collect(Collectors.toList()); + log.info("外标相关数据条数：{}", fbDatalist.size()); + + // 过滤字段 根据标准号字段去重 + fbDatalist = fbDatalist.stream().collect(Collectors.collectingAndThen(Collectors.toCollection(() + -> new TreeSet<>(Comparator.comparing(OldSystemImportDTO::getBzh))), ArrayList::new)); + log.info("根据标准号去重后的外标相关数据条数：{}", fbDatalist.size()); + long startTime = System.currentTimeMillis(); + // 转换数据 + List fbList = overseasStandardService.list(); + Map fbMap = fbList.stream() + .collect(Collectors.toMap(LawsOverseasStandard::getStandardNumber, o -> o)); + List ossFileList = ossFileService.list(); + Map> ossFileMap = ossFileList.stream() + .filter(o -> o.getStandardId() != null) + .collect(Collectors.groupingBy(OSSFile::getStandardId)); + for (OldSystemImportDTO dto : fbDatalist) { + // 先将标准号中的中文横杠替换为英文横杠 + String newBzh = dto.getBzh().replace("—", "-"); + dto.setBzh(newBzh); + } + // 导入企标 + for (OldSystemImportDTO dto : fbDatalist) { + LawsOverseasStandard fb = fbMap.get(dto.getBzh()); + if (StrUtil.isBlank(dto.getDoclj())) { + dto.setErrorInfo("没有需要导入的文件"); + errImportList.add(dto); + continue; + } + if (fb == null) { + dto.setErrorInfo("未找到对应的外标数据"); + errImportList.add(dto); + continue; + } + List ossFile = ossFileMap.get(fb.getId()); + if (ossFile == null || ossFile.isEmpty()) { + dto.setErrorInfo("未找到对应的文件数据"); + errImportList.add(dto); + } + } + // 获取当前时间戳（毫秒） + long endTime = System.currentTimeMillis(); + // 计算执行时间（毫秒） + long executionTime = endTime - startTime; + log.info("代码执行时间：" + executionTime + " 毫秒"); + } + private void oldSysFBImport(List allDatalist, List errImportList) { List fbDatalist = allDatalist.stream().filter(o -> o.getBzfl().contains("ISO") || o.getBzfl().contains("引进") ||