背景
最近做了两个项目,都有excel导入,而且代码量比较大,重性比较高,也没有什么技术含量 为了解决以后更快的开发导入,梳理出了一套可以借鉴的模版供大家参考。
项目依赖jar包
java 8 、Springboot、lombok、commons-lang3、hibernate-validator、Collection4
org.projectlombok lombok
cn.afterturn easypoi-spring-boot-starter 4.3.0
org.apache.commons commons-lang3 3.12.0
org.hibernate hibernate-validator 6.0.1.Final
整体结构
1、读取excel内容 2、把excel行转成行对象 3、验证必填项「不符合规则删除」 4、验证格式「不符合规则删除」 5、验证重复项「不符合规则删除」 6、自定义规则验证 7、分组:如果满足5,需要进行处理是更新还是插入
定义需要的类
验证后消息汇总
@Data
@Accessors(chain = true)
@ApiModel
public class ExcelVerifyVo {@ApiModelProperty(example = "总条数")private int total;@ApiModelProperty(example = "正确数据条数")private int rightTotal;@ApiModelProperty(example = "异常数据条数")private int errorTotal;@ApiModelProperty(example = "不能为空提示")private List emptyErrorHint = new ArrayList<>();@ApiModelProperty(example = "信息格式错误提示")private List formatErrorHint = new ArrayList<>();@ApiModelProperty(example = "编号重复提示")private List repetitiveErrorHint = new ArrayList<>();@ApiModelProperty(example = "丢弃的编号提示")private List rejectErrorHint = new ArrayList<>();@ApiModelProperty(example = "编号存在提示")private List existHint = new ArrayList<>();@JsonIgnoreprivate List addList = new ArrayList<>();@JsonIgnoreprivate List updateList = new ArrayList<>();
}
所有excel行的父类
@Data
@Accessors(chain = true)
public class ExcelRowDetail {private String sheetName;private int rowNo;public static String getSheetNameFieldName(){return "sheetName";}public static String getRowNoFieldName(){return "rowNo";}/*** @description: 获取类属性名称对应excel列,默认是属性名称【有机会会补全order注解】* @params: [clazz]* @return: java.util.Map* @throws*/public static Map getFieldMappingExcelColumnMap(Class clazz){Map fieldMap = Maps.newHashMap();Field[] Fields = clazz.getDeclaredFields();if(!Objects.isNull(Fields) && Fields.length > 0){int colNum = 0;for(Field field : Fields){ExcelFeildIgnore annotation = field.getAnnotation(ExcelFeildIgnore.class);if(Objects.isNull(annotation)){fieldMap.put(colNum,field.getName());colNum ++;}}}return fieldMap;}
}
excel行内容
@Data
@Accessors(chain = true)
public class ExcelContent extends ExcelRowDetail{@ApiModelProperty(name = "")@NotBlankprivate String code;@ApiModelProperty(name = "")@NotBlank@Pattern(regexp = RegexpConsts.PIMERF_REGEXP)private String name;
}
excel工具类
@UtilityClass
public class ExcelUtil {/*** @description: 读取文件内容* @params: [file, rowCount, columnCount]* @return: java.util.List* @throws*/@SneakyThrowspublic static List readExcel(MultipartFile file){Workbook wb = getWorkbook(file.getInputStream(),file.getOriginalFilename());return readExcelContent(wb);}public static String getSuffixByName(String filename) {String[] arr = filename.split("\\.");int suffixIndex = arr.length - 1;return strArray[suffixIndex];}@SneakyThrowspublic static Workbook getWorkbook(InputStream inputStream, String fileName){String extName = getSuffixByName(fileName);Workbook wb;if (ExcelVersion.V2003.getSuffix().equals(extName)) {wb = new HSSFWorkbook(inputStream);} else if (ExcelVersion.V2007.getSuffix().equals(extName)) {wb = new XSSFWorkbook(inputStream);} else {throw new ServiceException(HttpStatus.BAD_REQUEST.value(),"Invalid excel version");}return wb;}/*** @description: 只获取第一个sheet页* @params: [wb, rowCount, columnCount]* @return: java.util.List* @throws*/public static List readExcelContent(Workbook wb){if(Objects.isNull(wb)){throw new ServiceException(HttpStatus.BAD_REQUEST.value(),"文件不存在Sheet页!");}// 开始读取数据List sheetPOs = new ArrayList<>();// 解析sheetfor (int i = 0; i < wb.getNumberOfSheets(); i++) {Sheet sheet = wb.getSheetAt(i);List> dataList = new ArrayList<>();ExcelSheetPo sheetPO = new ExcelSheetPo();sheetPO.setSheetName(sheet.getSheetName());sheetPO.setDataList(dataList);int firstRowNum = sheet.getFirstRowNum();int lastRowNum = sheet.getLastRowNum();// 解析sheet 的行for (int j = firstRowNum; j <= lastRowNum; j++) {List rowValue = new ArrayList<>();Row row = sheet.getRow(j);if (row == null) {dataList.add(null);continue;}int readColumnCount = row.getLastCellNum();// 解析row 的列for (int k = 0; k < readColumnCount; k++) {Cell cell = row.getCell(k);rowValue.add(getCellValue(cell));}dataList.add(rowValue);}sheetPOs.add(sheetPO);return sheetPOs;}return sheetPOs;}private static Object getCellValue(Cell cell) {Object columnValue = null;if (cell != null) {CellType cellType = cell.getCellType();if(cellType.getCode() == CellType.NUMERIC.getCode()){boolean isDate = HSSFDateUtil.isCellDateFormatted(cell);if(isDate){return DateUtil.formatDate(cell.getDateCellValue(),DateUtil.DATE_PATTERN_YYYY_MM_DD);}}try {columnValue = cell.getStringCellValue();}catch (Exception e){columnValue = cell.toString();}}return columnValue;}/*** @description: 填充excel内容* @params: [file 上传的文件, clazz 实体类,用于和excel中列相对应,必须继承 ExcelRowDetail, startRowIndex 开始行索引, endColumnIndex 结束列索引]* @return: java.util.List* @throws*/@SneakyThrowspublic static List populateExcelContent(MultipartFile file, Class clazz, int startRowIndex) {if(startRowIndex < 0){throw new ServiceException(HttpStatus.BAD_REQUEST.value(),"startRowIndex or endColumnIndex must gt; 0");}if(Objects.isNull(clazz)){throw new ServiceException(HttpStatus.BAD_REQUEST.value(),"clazz must be not null ");}T t = clazz.newInstance();if(!(t instanceof ExcelRowDetail)){throw new ServiceException(HttpStatus.BAD_REQUEST.value(),"clazz must instanceof ExcelRowDetail");}List excelSheetPos = readExcel(file);return populateExcelContent(excelSheetPos,clazz,startRowIndex);}/*** @description: excel 内容读取出来后,统一使用String接* @params: [inputStream, clazz, startRowIndex, endColumnIndex]* @return: java.util.List* @throws*/@SneakyThrowspublic static List populateExcelContent(List excelSheetPos, Class clazz, int startRowIndex){List result = new ArrayList<>();if(CollectionUtils.isEmpty(excelSheetPos)){return result;}Map fieldMappingExcelColumnMap = ExcelRowDetail.getFieldMappingExcelColumnMap(clazz);int endColumnIndex = fieldMappingExcelColumnMap.size() - 1;Map fieldCache = Maps.newHashMap();Field[] fields = clazz.getDeclaredFields();// 当前类for(Field field : fields){field.setAccessible(true);fieldCache.put(field.getName(),field);}// 父类Class super T> superclass = clazz.getSuperclass();fields = superclass.getDeclaredFields();for(Field field : fields){field.setAccessible(true);fieldCache.put(field.getName(),field);}for(ExcelSheetPo excelSheetPo : excelSheetPos){String sheetName = excelSheetPo.getSheetName();List> dataList = excelSheetPo.getDataList();if(CollectionUtils.isEmpty(dataList) || dataList.size()-1 < startRowIndex){continue;}int rowCount = dataList.size();for(int i = startRowIndex;i < rowCount ; i++ ){List rowContent = dataList.get(i);if(CollectionUtils.isEmpty(rowContent)){continue;}T t = clazz.newInstance();for(int k = 0 ; k <= endColumnIndex;k++){String value = null;try {Object o = rowContent.get(k);if(!Objects.isNull(o)){value = o.toString();}}catch (Exception e){// ignore}String fieldName = fieldMappingExcelColumnMap.get(k);Field field = fieldCache.get(fieldName);field.set(t,value);}// 设置sheet名称和行号Field field = fieldCache.get(ExcelRowDetail.getSheetNameFieldName());field.set(t,sheetName);field = fieldCache.get(ExcelRowDetail.getRowNoFieldName());field.set(t,i+1);result.add(t);}}return result;}/*** @description: 验证必填项,根据注解* @params: [contentList, vo]* @return: void* @throws*/@SneakyThrowspublic static void verifyExcelRequiredField(List contentList, ExcelVerifyVo vo) {List emptyErrorHint = vo.getEmptyErrorHint();Map> emptyVerifySheetRow = Maps.newHashMap();Iterator iterator = contentList.iterator();while(iterator.hasNext()){T t = iterator.next();Class clazz = t.getClass();Class superclass = clazz.getSuperclass();Field sheetNameField = superclass.getDeclaredField(ExcelRowDetail.getSheetNameFieldName());sheetNameField.setAccessible(true);String sheetName = (String)sheetNameField.get(t);Field rowNoField = superclass.getDeclaredField(ExcelRowDetail.getRowNoFieldName());rowNoField.setAccessible(true);Integer rowNo = (Integer) rowNoField.get(t);// 验证不通过标识boolean checked = false;Field[] declaredFields = clazz.getDeclaredFields();for(Field field : declaredFields){field.setAccessible(true);NotBlank notBlank = field.getAnnotation(NotBlank.class);boolean flag = !Objects.isNull(notBlank) && (Objects.isNull(field.get(t)) || StringUtils.isBlank(field.get(t).toString()));if(flag){checked = true;break;}}if(checked){if(emptyVerifySheetRow.containsKey(sheetName)){emptyVerifySheetRow.get(sheetName).add(rowNo);}else{List inner = new ArrayList<>();inner.add(rowNo);emptyVerifySheetRow.put(sheetName,inner);}iterator.remove();}}if(emptyVerifySheetRow.size() > 0){// 有必填项验证不通过emptyVerifySheetRow.forEach((sheetName,rowList)->{StringBuilder sb = new StringBuilder();sb.append(sheetName).append("中第").append(StringUtils.join(rowList,"、")).append("行:信息填写不全;");emptyErrorHint.add(sb.toString());});emptyErrorHint.add("继续导入将跳过此数据;");}}/*** @description: 验证excel格式错误* @params: [contentList, vo]* @return: void* @throws*/@SneakyThrowspublic static void verifyExcelFormatErrorField(List contentList, ExcelVerifyVo vo) {if(!CollectionUtils.isNotEmpty(contentList)){return ;}Map> sheetErrorHintMap = Maps.newHashMap();Iterator iterator = contentList.iterator();while(iterator.hasNext()){boolean flag = false;T rowContent = iterator.next();Class clazz = rowContent.getClass();// 验证字段Field[] fields = clazz.getDeclaredFields();for(Field field : fields){field.setAccessible(true);// TODO 根据场景不同可以增加不同的校验规则Pattern patternAnno = field.getAnnotation(Pattern.class);Object callObj = field.get(rowContent);if(!Objects.isNull(patternAnno)){String value;if(Objects.isNull(callObj) || StringUtils.isBlank(value = callObj.toString())){continue;}String regexp = patternAnno.regexp().replace("\n", "\\\\n");java.util.regex.Pattern compile = java.util.regex.Pattern.compile(regexp);Matcher matcher = compile.matcher(value);if(!matcher.matches()){flag = true;break;}}}if(flag){iterator.remove();// 验证不通过Class> superclass = clazz.getSuperclass();Field field = superclass.getDeclaredField(ExcelRowDetail.getRowNoFieldName());field.setAccessible(true);Integer row = (Integer) field.get(rowContent);Field sheetNameField = superclass.getDeclaredField(ExcelRowDetail.getSheetNameFieldName());sheetNameField.setAccessible(true);String sheetName = sheetNameField.get(rowContent).toString();if(sheetErrorHintMap.containsKey(sheetName)){sheetErrorHintMap.get(sheetName).add(row);}else{List rows = new ArrayList<>();rows.add(row);sheetErrorHintMap.put(sheetName,rows);}}}if(sheetErrorHintMap.isEmpty()){return ;}List list = vo.getFormatErrorHint();for(Map.Entry> entry : sheetErrorHintMap.entrySet()){String sheetName = entry.getKey();StringBuilder sb = new StringBuilder();List rows = entry.getValue();Collections.sort(rows);String join = StringUtils.join(rows, "、");sb.append(sheetName).append("中第").append(join).append("行:信息格式错误,请检查;");list.add(sb.toString());}}/*** @description: 验证excel中重复的标识* @params: [groupRep>, vo]* @return: void* @throws*/@SneakyThrowspublic static void verifyExcelRepetitive(Map> groupRep, ExcelVerifyVo vo) {if(Objects.isNull(groupRep) || groupRep.size() == 0){return ;}List repetitiveErrorHint = vo.getRepetitiveErrorHint();for(Map.Entry> entry : groupRep.entrySet()){String uniqueKey = entry.getKey();List contentList = entry.getValue();StringBuilder sb = new StringBuilder();sb.append("【").append(uniqueKey).append("】").append("编号重复:");// 把rep中的数据,按照sheet页进行分组Map> innerGroup = Maps.newHashMap();for(T t : contentList){Class> clazz = t.getClass();Class> superclass = clazz.getSuperclass();Field field = superclass.getDeclaredField(ExcelRowDetail.getSheetNameFieldName());field.setAccessible(true);String sheetName = (String)field.get(t);if(innerGroup.containsKey(sheetName)){innerGroup.get(sheetName).add(t);}else{List inner = new ArrayList<>();inner.add(t);innerGroup.put(sheetName,inner);}}for(Map.Entry> sEntry : innerGroup.entrySet()){String sheetName = sEntry.getKey();List rowContents = sEntry.getValue();sb.append(sheetName).append("中第");List rows = new ArrayList<>();for(T content : rowContents){Class> clazz = content.getClass();Class> superclass = clazz.getSuperclass();Field field = superclass.getDeclaredField(ExcelRowDetail.getRowNoFieldName());field.setAccessible(true);Integer row = (Integer) field.get(content);rows.add(row);}Collections.sort(rows);String join = StringUtils.join(rows, "、");sb.append(join).append("行").append("、");sb.deleteCharAt(sb.length()-1).append(";");repetitiveErrorHint.add(sb.toString());}}repetitiveErrorHint.add("继续操作将导入最后一条数据;");}public static void excelAlreadyExistRejectedNoHint(List nos, ExcelVerifyVo vo) {if(!CollectionUtils.isNotEmpty(nos)){return ;}List rejectErrorHint = vo.getRejectErrorHint();StringBuilder sb = new StringBuilder();sb.append("编号【").append(org.apache.commons.lang3.StringUtils.join(nos, SplitSymbolEnum.SPLIT_PAUSE.getSymbol()));sb.append("】").append("系统中已存在;");rejectErrorHint.add(sb.toString());rejectErrorHint.add("继续将跳过此数据;");}@SneakyThrowspublic static void verifyExcelAlreadyExist(Map> existsSheetGroup, ExcelVerifyVo vo,String uniqueFeildName) {List existErrorHint = vo.getExistHint();if(existsSheetGroup.isEmpty()){return ;}for(Map.Entry> entry : existsSheetGroup.entrySet()){String sheetName = entry.getKey();List existsList = entry.getValue();StringBuilder sb = new StringBuilder();sb.append(sheetName).append("中第");List rowNos = new ArrayList<>();List nos = new ArrayList<>();for (T excelContent : existsList){Class> clazz = excelContent.getClass();Field uniqueKeyField = clazz.getDeclaredField(uniqueFeildName);uniqueKeyField.setAccessible(true);String uniqueKeyValue = (String)uniqueKeyField.get(excelContent);Class> superclass = clazz.getSuperclass();Field rowNoField = superclass.getDeclaredField(ExcelRowDetail.getRowNoFieldName());rowNoField.setAccessible(true);Integer rowNo = (Integer) rowNoField.get(excelContent);rowNos.add(rowNo);nos.add(uniqueKeyValue);}Collections.sort(rowNos);sb.append(StringUtils.join(rowNos,"、")).append("行;编号【");sb.append(StringUtils.join(nos,"、")).append("】的唯一编号已存在;");existErrorHint.add(sb.toString());}existErrorHint.add("继续导入将更新数据;");}
}
读取excel内容
@Overridepublic ExcelVerifyVo importVerify(MultipartFile file) {ExcelVerifyVo vo = new ExcelVerifyVo();List contentList = ExcelUtil.populateExcelContent(file,ExcelContent.class,1);if(CollectionUtils.isEmpty(contentList)){return vo;}vo.setTotal(contentList.size());// ------------过滤,验证必填项------------>ExcelUtil.verifyExcelRequiredField(contentList,vo);// ------------过滤,验证格式------------>ExcelUtil.verifyExcelFormatErrorField(contentList,vo);// ------------过滤,验证重复------------>Map groupContent = Maps.newHashMap();Map> groupRep = Maps.newHashMap();contentList.forEach(excelContent -> {if(groupContent.containsKey(excelContent.getPimerNo())){if(groupRep.containsKey(excelContent.getPimerNo())){groupRep.get(excelContent.getPimerNo()).add(excelContent);}else{List inner = new ArrayList<>();inner.add(groupContent.get(excelContent.getPimerNo()));inner.add(excelContent);groupRep.put(excelContent.getPimerNo(),inner);}}// 存在就更新,保留最后一条groupContent.put(excelContent.getPimerNo(),excelContent);});contentList.clear();groupContent.forEach((key, value)-> contentList.add(value));ExcelUtil.verifyExcelRepetitive(groupRep,vo);groupContent.clear();// ------------验证权限 非实验员自己的数据,需要丢弃掉------------>List addList = vo.getAddList();List updateList = vo.getUpdateList();List nos = new ArrayList<>();Iterator iterator = contentList.iterator();SysUser currentUser = UserUtil.getCurrentUser();while(iterator.hasNext()){ExcelContent excelContent = iterator.next();Data data = repository.findByNo(excelContent.getNo());if(!Objects.isNull(data)){updateList.add(excelContent);}else{addList.add(excelContent);}}ExcelUtil.excelAlreadyExistRejectedNoHint(nos,vo);// ------------过滤,验证重复------------>Map> existsSheetGroup = Maps.newHashMap();updateList.forEach(excelContent -> {if(existsSheetGroup.containsKey(excelContent.getSheetName())){existsSheetGroup.get(excelContent.getSheetName()).add(excelContent);}else{List inner = new ArrayList();inner.add(excelContent);existsSheetGroup.put(excelContent.getSheetName(),inner);}});ExcelUtil.verifyExcelAlreadyExist(existsSheetGroup,vo,"pimerNo");vo.setRightTotal(contentList.size());vo.setErrorTotal(vo.getTotal()-vo.getRightTotal());return vo;}
持久化
持久化动作就比较简单了
@Overridepublic int importCommit(MultipartFile file) {ExcelVerifyVo vo = this.importVerify(file);if(vo.getRightTotal() == 0){return 0;}List addList = vo.getAddList();List updateList = vo.getUpdateList();if(CollectionUtils.isEmpty(addList) && CollectionUtils.isEmpty(updateList)){throw new ServiceException(HttpStatus.NO_CONTENT.value(),"excel中无内容!");}if(CollectionUtils.isNotEmpty(addList)){List insertList = new ArrayList();Data data;for(ExcelContent excelContent : addList){data = new Data();BeanUtils.copyProperties(excelContent,data);pimer.setId(IdUtils.getUuid());insertList.add(data);}pimerRepository.saveAll(insertList);}if(CollectionUtils.isNotEmpty(updateList)){List updateDataList = new ArrayList<>();// 这里会有并发问题【根据项目的实际情况处理】updateList.forEach(excelContent->{Data data = repository.findByNo(excelContent.getNo());BeanUtils.copyProperties(excelContent,data);}repository.saveAll(updateList);}return vo.getRightTotal();}