导入规则校验完善

This commit is contained in:
wangchengjun
2022-01-11 00:47:00 +08:00
parent 01269d3260
commit f269ee11cb
3 changed files with 537 additions and 17 deletions
@@ -10,7 +10,7 @@ import java.lang.annotation.*;
* @Author wcj
* @Date 2022年1月6日
*/
@Target(ElementType.METHOD)
@Target(ElementType.FIELD)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface ExcelImport {
@@ -41,4 +41,25 @@ public @interface ExcelImport {
* @return
*/
String dateFormat() default "yyyy-MM-dd HH:mm:ss";
/**
* 方法描述: 数据code
*
* @return 返回类型: String
*/
String dicCode();
/**
* 方法描述: 数据Text
*
* @return 返回类型: String
*/
String dicText() default "";
/**
* 方法描述: 数据字典表
*
* @return 返回类型: String
*/
String dictTable() default "";
}
@@ -2,13 +2,47 @@ package com.jero.common.util;
import com.alibaba.fastjson.JSONObject;
import com.baomidou.mybatisplus.extension.service.IService;
import lombok.extern.slf4j.Slf4j;
import com.jero.common.api.CommonAPI;
import com.jero.common.api.vo.Result;
import com.jero.common.aspect.annotation.ExcelImport;
import com.jero.common.constant.CommonConstant;
import com.jero.common.system.vo.DictModel;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.apache.poi.POIXMLDocument;
import org.apache.poi.hssf.usermodel.HSSFWorkbook;
import org.apache.poi.openxml4j.exceptions.InvalidFormatException;
import org.apache.poi.openxml4j.opc.OPCPackage;
import org.apache.poi.poifs.filesystem.POIFSFileSystem;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.jeecgframework.core.util.ApplicationContextUtil;
import org.jeecgframework.dict.service.AutoPoiDictServiceI;
import org.jeecgframework.poi.excel.ExcelImportCheckUtil;
import org.jeecgframework.poi.excel.annotation.Excel;
import org.jeecgframework.poi.excel.annotation.ExcelCollection;
import org.jeecgframework.poi.excel.annotation.ExcelTarget;
import org.jeecgframework.poi.excel.entity.ImportParams;
import org.jeecgframework.poi.excel.entity.params.ExcelCollectionParams;
import org.jeecgframework.poi.excel.entity.params.ExcelImportEntity;
import org.jeecgframework.poi.exception.excel.ExcelImportException;
import org.jeecgframework.poi.exception.excel.enums.ExcelImportEnum;
import org.jeecgframework.poi.util.ExcelUtil;
import org.jeecgframework.poi.util.PoiPublicUtil;
import org.springframework.util.CollectionUtils;
import java.io.File;
import java.io.IOException;
import java.util.List;
import java.io.InputStream;
import java.io.PushbackInputStream;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.ParameterizedType;
import java.util.*;
import java.util.regex.Pattern;
/**
* 导出返回信息
@@ -16,7 +50,7 @@ import java.util.List;
@Slf4j
public class ImportExcelUtil {
public static Result<?> imporReturnRes(int errorLines,int successLines,List<String> errorMessage) throws IOException {
public static Result<?> imporReturnRes(int errorLines, int successLines, List<String> errorMessage) throws IOException {
if (errorLines == 0) {
return Result.OK("" + successLines + "行数据全部导入成功!");
} else {
@@ -38,12 +72,387 @@ public class ImportExcelUtil {
}
}
public static List<String> importDateSave(List<Object> list, Class serviceClass,List<String> errorMessage,String errorFlag) {
IService bean =(IService) SpringContextUtils.getBean(serviceClass);
/**
* 导入校验
*
* @param inputstream
* @param pojoClass
* @param params
* @return
*/
public static Result<?> importReturnMessage(InputStream inputstream, Class<?> pojoClass, ImportParams params) {
JSONObject result = new JSONObject(5);
Workbook book = null;
//错误航速
int errorNum = 0;
//正确行数
int successNum = 0;
//错误信息集合
List<String> errorMessage = new ArrayList<>();
if (!(inputstream.markSupported())) {
inputstream = new PushbackInputStream(inputstream, 8);
}
try {
if (POIFSFileSystem.hasPOIFSHeader(inputstream)) {
book = new HSSFWorkbook(inputstream);
} else if (POIXMLDocument.hasOOXMLHeader(inputstream)) {
book = new XSSFWorkbook(OPCPackage.open(inputstream));
}
} catch (IOException e) {
e.printStackTrace();
} catch (InvalidFormatException e) {
e.printStackTrace();
}
for (int i = 0; i < params.getSheetNum(); i++) {
Row row = null;
//跳过表头和标题行
Iterator<Row> rows;
try {
rows = book.getSheetAt(i).rowIterator();
} catch (Exception e) {
//为空说明读取不到,故不是excel
throw new RuntimeException("请导入正确格式的excel文件!");
}
for (int j = 0; j < params.getTitleRows() + params.getHeadRows(); j++) {
try {
row = rows.next();
} catch (NoSuchElementException e) {
//为空说明标题不出在,excel格式错误
throw new RuntimeException("请填写内容标题!");
}
}
Sheet sheet = book.getSheetAt(i);
Map<Integer, String> titlemap = null;
try {
titlemap = getTitleMap(sheet, params);
} catch (Exception e) {
e.printStackTrace();
}
Set<Integer> columnIndexSet = titlemap.keySet();
Integer maxColumnIndex = Collections.max(columnIndexSet);
Integer minColumnIndex = Collections.min(columnIndexSet);
while (rows.hasNext() && (row == null || sheet.getLastRowNum() - row.getRowNum() > params.getLastOfInvalidRow())) {
row = rows.next();
Map<String, ExcelImportEntity> excelParams = new HashMap<String, ExcelImportEntity>();
List<ExcelCollectionParams> excelCollection = new ArrayList<ExcelCollectionParams>();
String targetId = null;
if (!Map.class.equals(pojoClass)) {
Field fileds[] = PoiPublicUtil.getClassFields(pojoClass);
ExcelTarget etarget = pojoClass.getAnnotation(ExcelTarget.class);
if (etarget != null) {
targetId = etarget.value();
}
try {
getAllExcelField(targetId, fileds, excelParams, excelCollection, pojoClass, null);
// errorMessage.add(message);
} catch (Exception e) {
errorMessage.add("" + row.getRowNum() + "行数据解析错误。");
// e.printStackTrace();
}
}
try {
int firstCellNum = row.getFirstCellNum();
if (firstCellNum > minColumnIndex) {
firstCellNum = minColumnIndex;
}
int lastCellNum = row.getLastCellNum();
if (lastCellNum < maxColumnIndex + 1) {
lastCellNum = maxColumnIndex + 1;
}
for (int j = firstCellNum, le = lastCellNum; j < le; j++) {
String titleString = (String) titlemap.get(j);
if (excelParams.containsKey(titleString) || Map.class.equals(pojoClass)) {
successNum += 1;
//获取数据值
Cell cell = row.getCell(j);
String keyValue = getKeyValue(cell);
String message = checkValue(keyValue, titleString, pojoClass);
if (StringUtils.isNotEmpty(message)) {
errorMessage.add(message);
}
} else {
if (excelCollection.size() > 0) {
Iterator var33 = excelCollection.iterator();
ExcelCollectionParams param = (ExcelCollectionParams) var33.next();
if (param.getExcelParams().containsKey(titleString)) {
successNum += 1;
} else {
errorNum += 1;
}
} else {
errorNum += 1;
}
}
}
// if(successNum<errorNum){
// return false;
// }else if(successNum>errorNum){
// if(errorNum>0){
//// double newNumber = (double) successNum / (successNum + errorNum);
//// BigDecimal bg = new BigDecimal(newNumber);
//// double f1 = bg.setScale(1, BigDecimal.ROUND_HALF_UP).doubleValue();
//// if(f1<screenRate){
//// return false;
//// }else{
//// return true;
//// }
// }else{
// return true;
// }
// }else if(successNum==errorNum){
// return false;
// }else{
// return false;
// }
} catch (ExcelImportException e) {
if (!e.getType().equals(ExcelImportEnum.VERIFY_ERROR)) {
errorMessage.add("" + row.getRowNum() + "行值校验失败。");
// throw new ExcelImportException(e.getType(), e);
}
}
}
}
//总行数
int totalCount = successNum + errorNum;
result.put("totalCount", totalCount);
result.put("errorCount", errorNum);
result.put("successCount", successNum);
result.put("msg", "总数据行数:" + totalCount + ",正确数据行数:" + successNum + ",错误数据行数:" + errorNum);
String fileUrl = PmsUtil.saveErrorTxtByList(errorMessage, "userImportExcelErrorLog");
int lastIndex = fileUrl.lastIndexOf(File.separator);
String fileName = fileUrl.substring(lastIndex + 1);
result.put("fileUrl", "/sys/common/static/" + fileUrl);
result.put("fileName", fileName);
Result res = Result.OK(result);
res.setCode(201);
res.setMessage("文件导入成功,但有错误。");
return res;
}
/**
* 校验字段内容
*
* @param keyValue
* @param pojoClass
* @return
*/
private static String checkValue(String keyValue, String titleString, Class<?> pojoClass) {
for (Field field : oConvertUtils.getAllFields(pojoClass)) {
if (!field.getName().equals(titleString)) {
continue;
}
if (field.getAnnotation(ExcelImport.class) != null) {
String code = field.getAnnotation(ExcelImport.class).dicCode();
String text = field.getAnnotation(ExcelImport.class).dicText();
String table = field.getAnnotation(ExcelImport.class).dictTable();
boolean required = field.getAnnotation(ExcelImport.class).required();
int length = field.getAnnotation(ExcelImport.class).length();
String regular = field.getAnnotation(ExcelImport.class).regular();
//校验是否必填
if (required && StringUtils.isEmpty(keyValue)) {
return titleString + "为必填字段。 ";
}
//校验长度
if (StringUtils.isNotEmpty(keyValue)) {
if (keyValue.length() > length) {
return titleString + "长度超出范围。 ";
}
//校验字典值
if (StringUtils.isNotEmpty(code)) {
//翻译字典值对应的txt
List<DictModel> dictModelList = translateDictValue(code, text, table);
if (!CollectionUtils.isEmpty(dictModelList)) {
long count = dictModelList.stream().filter(e -> e.getText().equals(keyValue)).count();
if (count <= 0) {
return titleString + "未查到相关字典。 ";
}
}
}
//校验正则
if (StringUtils.isNotEmpty(regular)) {
boolean matches = Pattern.matches(regular, keyValue);
if (!matches) {
return titleString + "不符合格式要求。 ";
}
}
}
}
}
return null;
}
/**
* 获取字典数据
*
* @param code
* @param text
* @param table
* @return
*/
private static List<DictModel> translateDictValue(String code, String text, String table) {
CommonAPI commonAPI = SpringContextUtils.getBean(CommonAPI.class);
List<DictModel> tmpValue = null;
if (!org.springframework.util.StringUtils.isEmpty(table)) {
tmpValue = commonAPI.queryTableDictItemsByCode(table, text, code);
} else {
tmpValue = commonAPI.queryDictItemsByCode(code);
}
return tmpValue;
}
/**
* 获取需要导出的全部字段
*
* @param targetId 目标ID
* @param fields
* @param excelCollection
* @throws Exception
*/
public static void getAllExcelField(String targetId, Field[] fields, Map<String, ExcelImportEntity> excelParams, List<ExcelCollectionParams> excelCollection, Class<?> pojoClass, List<Method> getMethods) throws Exception {
ExcelImportEntity excelEntity = null;
for (int i = 0; i < fields.length; i++) {
Field field = fields[i];
if (PoiPublicUtil.isNotUserExcelUserThis(null, field, targetId)) {
continue;
}
if (PoiPublicUtil.isCollection(field.getType())) {
// 集合对象设置属性
ExcelCollectionParams collection = new ExcelCollectionParams();
collection.setName(field.getName());
Map<String, ExcelImportEntity> temp = new HashMap();
ParameterizedType pt = (ParameterizedType) field.getGenericType();
Class<?> clz = (Class) pt.getActualTypeArguments()[0];
collection.setType(clz);
getExcelFieldList(targetId, PoiPublicUtil.getClassFields(clz), clz, temp, (List) null);
collection.setExcelParams(temp);
collection.setExcelName(((ExcelCollection) field.getAnnotation(ExcelCollection.class)).name());
additionalCollectionName(collection);
excelCollection.add(collection);
} else if (PoiPublicUtil.isJavaClass(field)) {
addEntityToMap(targetId, field, (ExcelImportEntity) excelEntity, pojoClass, getMethods, excelParams);
} else {
List<Method> newMethods = new ArrayList<Method>();
if (getMethods != null) {
newMethods.addAll(getMethods);
}
newMethods.add(PoiPublicUtil.getMethod(field.getName(), pojoClass));
getAllExcelField(targetId, PoiPublicUtil.getClassFields(field.getType()), excelParams, excelCollection, field.getType(), newMethods);
}
}
}
public static void getExcelFieldList(String targetId, Field[] fields, Class<?> pojoClass, Map<String, ExcelImportEntity> temp, List<Method> getMethods) throws Exception {
ExcelImportEntity excelEntity = null;
for (int i = 0; i < fields.length; i++) {
Field field = fields[i];
if (!PoiPublicUtil.isNotUserExcelUserThis((List) null, field, targetId)) {
if (PoiPublicUtil.isJavaClass(field)) {
addEntityToMap(targetId, field, (ExcelImportEntity) excelEntity, pojoClass, getMethods, temp);
} else {
List<Method> newMethods = new ArrayList();
if (getMethods != null) {
newMethods.addAll(getMethods);
}
newMethods.add(PoiPublicUtil.getMethod(field.getName(), pojoClass, field.getType()));
getExcelFieldList(targetId, PoiPublicUtil.getClassFields(field.getType()), field.getType(), temp, newMethods);
}
}
}
}
/**
* 把这个注解解析放到类型对象中
*
* @param targetId
* @param field
* @param excelEntity
* @param pojoClass
* @param getMethods
* @param temp
* @throws Exception
*/
public static void addEntityToMap(String targetId, Field field, ExcelImportEntity excelEntity, Class<?> pojoClass, List<Method> getMethods, Map<String, ExcelImportEntity> temp) throws Exception {
Excel excel = field.getAnnotation(Excel.class);
excelEntity = new ExcelImportEntity();
excelEntity.setType(excel.type());
excelEntity.setSaveUrl(excel.savePath());
excelEntity.setSaveType(excel.imageType());
excelEntity.setReplace(excel.replace());
excelEntity.setDatabaseFormat(excel.databaseFormat());
excelEntity.setVerify(ExcelImportCheckUtil.getImportVerify(field));
excelEntity.setSuffix(excel.suffix());
excelEntity.setNumFormat(excel.numFormat());
excelEntity.setGroupName(excel.groupName());
//update-begin-author:taoYan date:20180202 for:TASK #2067 【bug excel 问题】excel导入字典文本翻译问题
excelEntity.setMultiReplace(excel.multiReplace());
if (StringUtils.isNotEmpty(excel.dicCode())) {
AutoPoiDictServiceI jeecgDictService = null;
try {
jeecgDictService = ApplicationContextUtil.getContext().getBean(AutoPoiDictServiceI.class);
} catch (Exception e) {
}
if (jeecgDictService != null) {
String[] dictReplace = jeecgDictService.queryDict(excel.dictTable(), excel.dicCode(), excel.dicText());
if (excelEntity.getReplace() != null && dictReplace != null && dictReplace.length != 0) {
excelEntity.setReplace(dictReplace);
}
}
}
//update-end-author:taoYan date:20180202 for:TASK #2067 【bug excel 问题】excel导入字典文本翻译问题
ExcelImportCheckUtil.getExcelField(targetId, field, excelEntity, excel, pojoClass);
if (getMethods != null) {
List<Method> newMethods = new ArrayList<Method>();
newMethods.addAll(getMethods);
newMethods.add(excelEntity.getMethod());
excelEntity.setMethods(newMethods);
}
temp.put(excelEntity.getName(), excelEntity);
}
/**
* 追加集合名称到前面
*
* @param collection
*/
private static void additionalCollectionName(ExcelCollectionParams collection) {
Set<String> keys = new HashSet();
keys.addAll(collection.getExcelParams().keySet());
Iterator var3 = keys.iterator();
while (var3.hasNext()) {
String key = (String) var3.next();
collection.getExcelParams().put(collection.getExcelName() + "_" + key, collection.getExcelParams().get(key));
collection.getExcelParams().remove(key);
}
}
public static List<String> importDateSave(List<Object> list, Class serviceClass, List<String> errorMessage, String errorFlag) {
IService bean = (IService) SpringContextUtils.getBean(serviceClass);
for (int i = 0; i < list.size(); i++) {
try {
boolean save = bean.save(list.get(i));
if(!save){
if (!save) {
throw new Exception(errorFlag);
}
} catch (Exception e) {
@@ -54,11 +463,11 @@ public class ImportExcelUtil {
errorMessage.add("" + lineNumber + " 行:角色编码已经存在,忽略导入。");
} else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_JOB_CLASS_NAME)) {
errorMessage.add("" + lineNumber + " 行:任务类名已经存在,忽略导入。");
}else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_CODE)) {
} else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_CODE)) {
errorMessage.add("" + lineNumber + " 行:职务编码已经存在,忽略导入。");
}else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_DEPART_ORG_CODE)) {
} else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_DEPART_ORG_CODE)) {
errorMessage.add("" + lineNumber + " 行:部门编码已经存在,忽略导入。");
}else {
} else {
errorMessage.add("" + lineNumber + " 行:未知错误,忽略导入");
log.error(e.getMessage(), e);
}
@@ -67,11 +476,11 @@ public class ImportExcelUtil {
return errorMessage;
}
public static List<String> importDateSaveOne(Object obj, Class serviceClass,List<String> errorMessage,int i,String errorFlag) {
IService bean =(IService) SpringContextUtils.getBean(serviceClass);
public static List<String> importDateSaveOne(Object obj, Class serviceClass, List<String> errorMessage, int i, String errorFlag) {
IService bean = (IService) SpringContextUtils.getBean(serviceClass);
try {
boolean save = bean.save(obj);
if(!save){
if (!save) {
throw new Exception(errorFlag);
}
} catch (Exception e) {
@@ -82,15 +491,105 @@ public class ImportExcelUtil {
errorMessage.add("" + lineNumber + " 行:角色编码已经存在,忽略导入。");
} else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_JOB_CLASS_NAME)) {
errorMessage.add("" + lineNumber + " 行:任务类名已经存在,忽略导入。");
}else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_CODE)) {
} else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_CODE)) {
errorMessage.add("" + lineNumber + " 行:职务编码已经存在,忽略导入。");
}else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_DEPART_ORG_CODE)) {
} else if (message.contains(CommonConstant.SQL_INDEX_UNIQ_DEPART_ORG_CODE)) {
errorMessage.add("" + lineNumber + " 行:部门编码已经存在,忽略导入。");
}else {
} else {
errorMessage.add("" + lineNumber + " 行:未知错误,忽略导入");
log.error(e.getMessage(), e);
}
}
return errorMessage;
}
/**
* 获取文件名称标题
*
* @throws Exception
* @Author JEECG
* @date 20201023
*/
private static Map<Integer, String> getTitleMap(Sheet sheet, ImportParams params) throws Exception {
Map<Integer, String> titlemap = new HashMap<Integer, String>();
Iterator<Cell> cellTitle = null;
String collectionName = null;
Row headRow = null;
int headBegin = params.getTitleRows();
int allRowNum = sheet.getPhysicalNumberOfRows();
while (headRow == null && headBegin < allRowNum) {
headRow = sheet.getRow(headBegin++);
}
if (headRow == null) {
throw new Exception("不识别该文件");
}
if (ExcelUtil.isMergedRegion(sheet, headRow.getRowNum(), 0)) {
params.setHeadRows(2);
} else {
params.setHeadRows(1);
}
cellTitle = headRow.cellIterator();
while (cellTitle.hasNext()) {
Cell cell = cellTitle.next();
String value = getKeyValue(cell);
if (StringUtils.isNotEmpty(value)) {
titlemap.put(cell.getColumnIndex(), value);//加入表头列表
}
}
//多行表头
for (int j = headBegin; j < headBegin + params.getHeadRows() - 1; j++) {
headRow = sheet.getRow(j);
cellTitle = headRow.cellIterator();
while (cellTitle.hasNext()) {
Cell cell = cellTitle.next();
String value = getKeyValue(cell);
if (StringUtils.isNotEmpty(value)) {
int columnIndex = cell.getColumnIndex();
//当前cell的上一行是否为合并单元格
if (ExcelUtil.isMergedRegion(sheet, cell.getRowIndex() - 1, columnIndex)) {
collectionName = ExcelUtil.getMergedRegionValue(sheet, cell.getRowIndex() - 1, columnIndex);
if (params.isIgnoreHeader(collectionName)) {
titlemap.put(cell.getColumnIndex(), value);
} else {
titlemap.put(cell.getColumnIndex(), collectionName + "_" + value);
}
} else {
titlemap.put(cell.getColumnIndex(), value);
}
}
}
}
return titlemap;
}
/**
* 获取key的值,针对不同类型获取不同的值
*
* @param cell
* @return
* @Author JEECG
* @date 20201023
*/
private static String getKeyValue(Cell cell) {
if (cell == null) {
return null;
}
Object obj = null;
switch (cell.getCellTypeEnum()) {
case STRING:
obj = cell.getStringCellValue();
break;
case BOOLEAN:
obj = cell.getBooleanCellValue();
break;
case NUMERIC:
obj = cell.getNumericCellValue();
break;
case FORMULA:
obj = cell.getCellFormula();
break;
}
return obj == null ? null : obj.toString().trim();
}
}
@@ -32,7 +32,7 @@
<!--通过查询指定table的 text code key 获取字典值-->
<select id="queryTableDictTextByKey" parameterType="String" resultType="String">
select ${text} as "text" from ${table} where ${code}= #{key}
select distinct ${text} as "text" from ${table} where ${code}= #{key}
</select>
<!--通过查询指定table的 text code key 获取字典值,包含value-->