From 85953347dbd55b1c70ce54e3335945bd5d75e778 Mon Sep 17 00:00:00 2001 From: aipper Date: Mon, 24 Nov 2025 16:14:12 +0800 Subject: [PATCH] test --- .../AnjuanAndJuanneiController.java | 153 +++++++++--------- 1 file changed, 79 insertions(+), 74 deletions(-) diff --git a/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java b/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java index 548eb30..755e267 100644 --- a/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java +++ b/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java @@ -239,7 +239,7 @@ public class AnjuanAndJuanneiController { public AjaxJson uploadSimpleFilesAnjuan(HttpServletRequest request, String fondscode, Integer recId, String tableName) { Integer successNum = 0; Integer falseNum = 0; - + // 验证参数 if (recId == null || recId <= 0) { return AjaxJson.returnExceptionInfo("记录ID无效"); @@ -250,7 +250,7 @@ public class AnjuanAndJuanneiController { if (StringUtil.isEmpty(tableName)) { return AjaxJson.returnExceptionInfo("表名不能为空"); } - + // 创建文件在服务器端存放路径 String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId; File fileDir = new File(dir); @@ -261,44 +261,49 @@ public class AnjuanAndJuanneiController { return AjaxJson.returnExceptionInfo("创建目录失败"); } } - + // 验证目录是否可写 if (!fileDir.canWrite()) { logger.error("目录无写权限: {}", dir); return AjaxJson.returnExceptionInfo("目录无写权限"); } - + // 使用NIO方式迭代处理文件，避免一次性加载所有文件到内存 if (request instanceof MultipartHttpServletRequest) { MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request; - Iterator iterator = multipartRequest.getFileNames(); - int fileIndex = 0; - while (iterator.hasNext()) { + // 获取所有文件，包括同名字段的多个文件 + Map fileMap = multipartRequest.getFileMap(); + logger.info("接收到文件数量: {}", fileMap.size()); + + int fileIndex = 0; + for (Map.Entry entry : fileMap.entrySet()) { boolean acquired = false; try { uploadSemaphore.acquire(); // 最多并行处理3个文件 acquired = true; fileIndex++; - String name = iterator.next(); - MultipartFile file0 = multipartRequest.getFile(name); + String fieldName = entry.getKey(); + MultipartFile file0 = entry.getValue(); + logger.info("处理第{}个文件: 字段名={}, 文件名={}", fileIndex, fieldName, file0.getOriginalFilename()); + if (file0 == null || file0.isEmpty()) { logger.warn("第{}个文件为空，跳过", fileIndex); falseNum++; continue; } - + // 使用零拷贝方式处理单个文件，最大程度优化内存使用 AjaxJson json2 = uploadFilesByPathAnjuanZeroCopy(file0, fondscode, dir, recId, tableName, request); - + if ("101".equals(json2.getCode())) { falseNum++; } if ("100".equals(json2.getCode())) { successNum++; } - + // 异步处理OCR，避免阻塞 String originalFilename = file0.getOriginalFilename(); if (originalFilename != null) { @@ -315,7 +320,7 @@ public class AnjuanAndJuanneiController { } } } - + // 显式释放资源 try { if (file0.getInputStream() != null) { @@ -337,14 +342,14 @@ public class AnjuanAndJuanneiController { } else { return AjaxJson.returnExceptionInfo("请求类型不支持"); } - + Map map7 = new HashMap<>(); map7.put("tableName", tableName + "_temp"); map7.put("tableName2", tableName + "_temp_file"); map7.put("id", recId); danganguanliService.wsajmlTempCount(map7); - - AjaxJson json = AjaxJson.returnInfo("成功上传数successNum，失败上传数falseNum"); + + AjaxJson json = AjaxJson.returnInfo("成功上传数:"+successNum+"，失败上传数:"+falseNum); json.put("successNum", successNum); json.put("falseNum", falseNum); return json; @@ -354,18 +359,18 @@ public class AnjuanAndJuanneiController { private AjaxJson uploadFilesByPathAnjuan(MultipartFile file,String fondscode, String dir, Integer recId,String tableName, HttpServletRequest request) { AjaxJson json = null; File files = null; - + try { String originalFilename = file.getOriginalFilename(); if (StringUtil.isEmpty(originalFilename)) { return AjaxJson.returnExceptionInfo("文件名为空"); } - + int index = originalFilename.lastIndexOf(".") + 1; if (index <= 0 || index >= originalFilename.length()) { return AjaxJson.returnExceptionInfo("文件格式不正确"); } - + String fileType = originalFilename.substring(index); String file_name_server=StringUtil.generaterUUID()+"."+fileType; @@ -375,10 +380,10 @@ public class AnjuanAndJuanneiController { map5.put("tableName",tableName + "_temp_file"); map5.put("conditionSql","rec_id= '"+recId+"' and file_status=1 "); int pageNo =danganguanliService.selectObjectCount(map5)+1; - + // 使用File.separator确保跨平台兼容性 files = new File(dir + File.separator + file_name_server); - + // 检查目标文件是否已存在 if (files.exists()) { logger.warn("目标文件已存在，将覆盖: {}", files.getAbsolutePath()); @@ -388,7 +393,7 @@ public class AnjuanAndJuanneiController { return AjaxJson.returnExceptionInfo("无法删除已存在的文件"); } } - + // 验证目录是否存在且可写 File parentDir = files.getParentFile(); if (parentDir == null || !parentDir.exists()) { @@ -398,10 +403,10 @@ public class AnjuanAndJuanneiController { logger.error("目录无写权限: {}", parentDir.getAbsolutePath()); return AjaxJson.returnExceptionInfo("目录无写权限"); } - + // 文件传输 file.transferTo(files); - + // 验证文件是否成功写入 if (!files.exists() || files.length() == 0) { logger.error("文件传输失败或文件为空: {}", files.getAbsolutePath()); @@ -441,7 +446,7 @@ public class AnjuanAndJuanneiController { String newName_pdf=file_name_server.replace("."+fileType,".pdf"); String sourcePath = dir + File.separator + file_name_server; String targetPath = dir + File.separator + newName_pdf; - + boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath, targetPath); if (!pdfCreated) { logger.warn("PDF文件生成失败: {} -> {}", sourcePath, targetPath); @@ -452,7 +457,7 @@ public class AnjuanAndJuanneiController { FileTool.copyFile(targetPath, originalPath); } } - + //mxf格式的文件需要转换一份mp4给前端展示 if (fileType.equalsIgnoreCase("mxf")) { String replaceMp4; @@ -463,15 +468,15 @@ public class AnjuanAndJuanneiController { } VideoConvertUtil.convert(files.getPath(), replaceMp4); } - + json = AjaxJson.returnInfo("上传文件成功"); json.put("file",files); json.put("file_name_server",file_name_server); - + } catch (Exception e) { logger.error("上传文件失败: {}", file.getOriginalFilename(), e); json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage()); - + // 清理失败的文件 if (files != null && files.exists()) { try { @@ -486,7 +491,7 @@ public class AnjuanAndJuanneiController { } return json; } - + /** * NIO方式上传文件，避免OOM问题 */ @@ -495,18 +500,18 @@ public class AnjuanAndJuanneiController { Path targetPath = null; InputStream inputStream = null; FileChannel fileChannel = null; - + try { String originalFilename = file.getOriginalFilename(); if (StringUtil.isEmpty(originalFilename)) { return AjaxJson.returnExceptionInfo("文件名为空"); } - + int index = originalFilename.lastIndexOf(".") + 1; if (index <= 0 || index >= originalFilename.length()) { return AjaxJson.returnExceptionInfo("文件格式不正确"); } - + String fileType = originalFilename.substring(index); String file_name_server = StringUtil.generaterUUID() + "." + fileType; @@ -516,16 +521,16 @@ public class AnjuanAndJuanneiController { map5.put("tableName", tableName + "_temp_file"); map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 "); int pageNo = danganguanliService.selectObjectCount(map5) + 1; - + // 使用NIO Path和Files API targetPath = Paths.get(dir, file_name_server); - + // 检查目标文件是否已存在 if (Files.exists(targetPath)) { logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath()); Files.deleteIfExists(targetPath); } - + // 验证父目录是否存在且可写 Path parentDir = targetPath.getParent(); if (parentDir == null || !Files.exists(parentDir)) { @@ -535,25 +540,25 @@ public class AnjuanAndJuanneiController { logger.error("目录无写权限: {}", parentDir.toAbsolutePath()); return AjaxJson.returnExceptionInfo("目录无写权限"); } - + // 使用NIO方式写入文件，避免内存OOM inputStream = file.getInputStream(); - fileChannel = FileChannel.open(targetPath, - java.nio.file.StandardOpenOption.CREATE, + fileChannel = FileChannel.open(targetPath, + java.nio.file.StandardOpenOption.CREATE, java.nio.file.StandardOpenOption.WRITE, java.nio.file.StandardOpenOption.TRUNCATE_EXISTING); - + ReadableByteChannel readableByteChannel = Channels.newChannel(inputStream); - + // 使用transferFrom进行高效文件传输 long transferred = fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE); - + // 验证文件是否成功写入 if (!Files.exists(targetPath) || Files.size(targetPath) == 0) { logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath()); return AjaxJson.returnExceptionInfo("文件传输失败"); } - + logger.info("文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath()); String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId; @@ -588,7 +593,7 @@ public class AnjuanAndJuanneiController { String newName_pdf = file_name_server.replace("." + fileType, ".pdf"); Path sourcePath = targetPath; Path pdfPath = Paths.get(dir, newName_pdf); - + boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString()); if (!pdfCreated) { logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath); @@ -603,7 +608,7 @@ public class AnjuanAndJuanneiController { } } } - + //mxf格式的文件需要转换一份mp4给前端展示 if (fileType.equalsIgnoreCase("mxf")) { String replaceMp4; @@ -614,15 +619,15 @@ public class AnjuanAndJuanneiController { } VideoConvertUtil.convert(targetPath.toString(), replaceMp4); } - + json = AjaxJson.returnInfo("上传文件成功"); json.put("file", targetPath.toFile()); json.put("file_name_server", file_name_server); - + } catch (Exception e) { logger.error("上传文件失败: {}", file.getOriginalFilename(), e); json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage()); - + // 清理失败的文件 if (targetPath != null && Files.exists(targetPath)) { try { @@ -651,7 +656,7 @@ public class AnjuanAndJuanneiController { } return json; } - + /** * 零拷贝方式上传文件，最大程度优化内存使用，防止OOM */ @@ -660,18 +665,18 @@ public class AnjuanAndJuanneiController { Path targetPath = null; ReadableByteChannel readableByteChannel = null; FileChannel fileChannel = null; - + try { String originalFilename = file.getOriginalFilename(); if (StringUtil.isEmpty(originalFilename)) { return AjaxJson.returnExceptionInfo("文件名为空"); } - + int index = originalFilename.lastIndexOf(".") + 1; if (index <= 0 || index >= originalFilename.length()) { return AjaxJson.returnExceptionInfo("文件格式不正确"); } - + String fileType = originalFilename.substring(index); String file_name_server = StringUtil.generaterUUID() + "." + fileType; @@ -681,16 +686,16 @@ public class AnjuanAndJuanneiController { map5.put("tableName", tableName + "_temp_file"); map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 "); int pageNo = danganguanliService.selectObjectCount(map5) + 1; - + // 使用NIO Path和Files API targetPath = Paths.get(dir, file_name_server); - + // 检查目标文件是否已存在 if (Files.exists(targetPath)) { logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath()); Files.deleteIfExists(targetPath); } - + // 验证父目录是否存在且可写 Path parentDir = targetPath.getParent(); if (parentDir == null || !Files.exists(parentDir)) { @@ -700,20 +705,20 @@ public class AnjuanAndJuanneiController { logger.error("目录无写权限: {}", parentDir.toAbsolutePath()); return AjaxJson.returnExceptionInfo("目录无写权限"); } - + // 零拷贝实现：直接使用FileChannel进行传输，避免数据在用户空间的拷贝 try (InputStream inputStream = file.getInputStream()) { readableByteChannel = Channels.newChannel(inputStream); - fileChannel = FileChannel.open(targetPath, - StandardOpenOption.CREATE, + fileChannel = FileChannel.open(targetPath, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING); - + // 使用transferFrom实现零拷贝，数据直接从内核缓冲区传输到文件 long transferred = 0; long position = 0; long count = Long.MAX_VALUE; - + // 分块传输大文件，避免单次传输过大导致的问题 final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks while (position < file.getSize()) { @@ -725,16 +730,16 @@ public class AnjuanAndJuanneiController { position += transferredChunk; transferred += transferredChunk; } - + // 强制写入磁盘，确保数据持久化 fileChannel.force(true); - + // 验证文件是否成功写入 if (!Files.exists(targetPath) || Files.size(targetPath) == 0) { logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath()); return AjaxJson.returnExceptionInfo("文件传输失败"); } - + logger.info("零拷贝文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath()); } @@ -770,7 +775,7 @@ public class AnjuanAndJuanneiController { String newName_pdf = file_name_server.replace("." + fileType, ".pdf"); Path sourcePath = targetPath; Path pdfPath = Paths.get(dir, newName_pdf); - + boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString()); if (!pdfCreated) { logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath); @@ -792,7 +797,7 @@ public class AnjuanAndJuanneiController { } } } - + //mxf格式的文件需要转换一份mp4给前端展示 if (fileType.equalsIgnoreCase("mxf")) { String replaceMp4; @@ -803,15 +808,15 @@ public class AnjuanAndJuanneiController { } VideoConvertUtil.convert(targetPath.toString(), replaceMp4); } - + json = AjaxJson.returnInfo("上传文件成功"); json.put("file", targetPath.toFile()); json.put("file_name_server", file_name_server); - + } catch (Exception e) { logger.error("零拷贝上传文件失败: {}", file.getOriginalFilename(), e); json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage()); - + // 清理失败的文件 if (targetPath != null && Files.exists(targetPath)) { try { @@ -840,21 +845,21 @@ public class AnjuanAndJuanneiController { } return json; } - + /** * 零拷贝文件复制方法 */ private void copyFileZeroCopy(Path source, Path target) throws IOException { try (FileChannel sourceChannel = FileChannel.open(source, StandardOpenOption.READ); - FileChannel targetChannel = FileChannel.open(target, - StandardOpenOption.CREATE, + FileChannel targetChannel = FileChannel.open(target, + StandardOpenOption.CREATE, StandardOpenOption.WRITE, StandardOpenOption.TRUNCATE_EXISTING)) { - + long size = sourceChannel.size(); long position = 0; final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks - + while (position < size) { long chunkSize = Math.min(CHUNK_SIZE, size - position); long transferred = targetChannel.transferFrom(sourceChannel, position, chunkSize); @@ -863,7 +868,7 @@ public class AnjuanAndJuanneiController { } position += transferred; } - + // 强制写入磁盘 targetChannel.force(true); }