From b75f05bac9148508a0288c56e415efcb5ba0d4e1 Mon Sep 17 00:00:00 2001 From: aipper Date: Mon, 24 Nov 2025 12:21:46 +0800 Subject: [PATCH] test --- .../AnjuanAndJuanneiController.java | 519 ++++++++++++++++-- 1 file changed, 468 insertions(+), 51 deletions(-) diff --git a/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java b/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java index 0919ece..12c61ea 100644 --- a/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java +++ b/src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java @@ -20,13 +20,26 @@ import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import org.springframework.web.multipart.MultipartHttpServletRequest; import javax.servlet.http.HttpServletRequest; import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; +import java.nio.channels.WritableByteChannel; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; @@ -193,15 +206,12 @@ public class AnjuanAndJuanneiController { @RequestMapping(value = "/uploadSimpleFilesAnjuan", method = RequestMethod.POST) @ApiOperation(value = "传统案卷整理原文-单个或者多个上传") - //file要与表单上传的名字相同 - public AjaxJson uploadSimpleFilesAnjuan(MultipartFile[] file, String fondscode, Integer recId, String tableName, HttpServletRequest request) { + // 使用NIO方式处理多文件上传，避免OOM + public AjaxJson uploadSimpleFilesAnjuan(HttpServletRequest request, String fondscode, Integer recId, String tableName) { Integer successNum = 0; Integer falseNum = 0; // 验证参数 - if (file == null || file.length == 0) { - return AjaxJson.returnExceptionInfo("未选择任何文件"); - } if (recId == null || recId <= 0) { return AjaxJson.returnExceptionInfo("记录ID无效"); } @@ -212,59 +222,84 @@ public class AnjuanAndJuanneiController { return AjaxJson.returnExceptionInfo("表名不能为空"); } - for (int i = 0; i < file.length; i++) { - MultipartFile file0 = file[i]; - - // 验证文件 - if (file0 == null || file0.isEmpty()) { - logger.warn("第{}个文件为空，跳过", i + 1); - falseNum++; - continue; + // 创建文件在服务器端存放路径 + String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId; + File fileDir = new File(dir); + if (!fileDir.exists()) { + boolean created = fileDir.mkdirs(); + if (!created) { + logger.error("创建目录失败: {}", dir); + return AjaxJson.returnExceptionInfo("创建目录失败"); } - - //创建文件在服务器端存放路径 - 使用File.separator确保跨平台兼容性 - String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId; - File fileDir = new File(dir); - if (!fileDir.exists()) { - boolean created = fileDir.mkdirs(); - if (!created) { - logger.error("创建目录失败: {}", dir); + } + + // 验证目录是否可写 + if (!fileDir.canWrite()) { + logger.error("目录无写权限: {}", dir); + return AjaxJson.returnExceptionInfo("目录无写权限"); + } + + // 使用NIO方式迭代处理文件，避免一次性加载所有文件到内存 + if (request instanceof MultipartHttpServletRequest) { + MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request; + Iterator iterator = multipartRequest.getFileNames(); + int fileIndex = 0; + + while (iterator.hasNext()) { + fileIndex++; + String name = iterator.next(); + MultipartFile file0 = multipartRequest.getFile(name); + + if (file0 == null || file0.isEmpty()) { + logger.warn("第{}个文件为空，跳过", fileIndex); falseNum++; continue; } + + // 使用零拷贝方式处理单个文件，最大程度优化内存使用 + AjaxJson json2 = uploadFilesByPathAnjuanZeroCopy(file0, fondscode, dir, recId, tableName, request); + + if ("101".equals(json2.getCode())) { + falseNum++; + } + if ("100".equals(json2.getCode())) { + successNum++; + } + + // 异步处理OCR，避免阻塞 + String originalFilename = file0.getOriginalFilename(); + if (originalFilename != null) { + int index = originalFilename.lastIndexOf(".") + 1; + if (index > 0 && index < originalFilename.length()) { + String fileType = originalFilename.substring(index); + if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) { + OCRProcessingTask ocrTask = new OCRProcessingTask(json2, tableName, youhongIntegrate, youhongBaseUrl, tessPath, ocrLogMapper, danganguanliService); + Thread ocrThread = new Thread(ocrTask, "OCR-Processing-" + fileIndex); + ocrThread.setDaemon(true); + ocrThread.start(); + } + } + } + + // 显式释放资源 + try { + if (file0.getInputStream() != null) { + file0.getInputStream().close(); + } + } catch (IOException e) { + logger.warn("关闭文件流时出错: {}", e.getMessage()); + } } - - // 验证目录是否可写 - if (!fileDir.canWrite()) { - logger.error("目录无写权限: {}", dir); - falseNum++; - continue; - } - - AjaxJson json2 = uploadFilesByPathAnjuan(file0, fondscode, dir, recId, tableName, request); - if ("101".equals(json2.getCode())) { - falseNum++; - } - if ("100".equals(json2.getCode())) { - successNum++; - } - String originalFilename = file0.getOriginalFilename(); - int index = originalFilename.lastIndexOf(".") + 1; - String fileType = originalFilename.substring(index); - //启动一个线程，根据ocr获取图片文字"file_content,"+ - if(!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) { - // 使用线程池替代直接创建线程 - OCRProcessingTask ocrTask = new OCRProcessingTask(json2, tableName, youhongIntegrate, youhongBaseUrl, tessPath, ocrLogMapper, danganguanliService); - Thread ocrThread = new Thread(ocrTask, "OCR-Processing-" + (i + 1)); - ocrThread.setDaemon(true); // 设置为守护线程 - ocrThread.start(); - } + } else { + return AjaxJson.returnExceptionInfo("请求类型不支持"); } - Map map7=new HashMap<>(); - map7.put("tableName",tableName+"_temp"); - map7.put("tableName2",tableName+"_temp_file"); - map7.put("id",recId); + + Map map7 = new HashMap<>(); + map7.put("tableName", tableName + "_temp"); + map7.put("tableName2", tableName + "_temp_file"); + map7.put("id", recId); danganguanliService.wsajmlTempCount(map7); + AjaxJson json = AjaxJson.returnInfo("成功上传数successNum，失败上传数falseNum"); json.put("successNum", successNum); json.put("falseNum", falseNum); @@ -407,6 +442,388 @@ public class AnjuanAndJuanneiController { } return json; } + + /** + * NIO方式上传文件，避免OOM问题 + */ + private AjaxJson uploadFilesByPathAnjuanNIO(MultipartFile file, String fondscode, String dir, Integer recId, String tableName, HttpServletRequest request) { + AjaxJson json = null; + Path targetPath = null; + InputStream inputStream = null; + FileChannel fileChannel = null; + + try { + String originalFilename = file.getOriginalFilename(); + if (StringUtil.isEmpty(originalFilename)) { + return AjaxJson.returnExceptionInfo("文件名为空"); + } + + int index = originalFilename.lastIndexOf(".") + 1; + if (index <= 0 || index >= originalFilename.length()) { + return AjaxJson.returnExceptionInfo("文件格式不正确"); + } + + String fileType = originalFilename.substring(index); + String file_name_server = StringUtil.generaterUUID() + "." + fileType; + + long fileLen = file.getSize() / 1024; + + Map map5 = new HashMap<>(); + map5.put("tableName", tableName + "_temp_file"); + map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 "); + int pageNo = danganguanliService.selectObjectCount(map5) + 1; + + // 使用NIO Path和Files API + targetPath = Paths.get(dir, file_name_server); + + // 检查目标文件是否已存在 + if (Files.exists(targetPath)) { + logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath()); + Files.deleteIfExists(targetPath); + } + + // 验证父目录是否存在且可写 + Path parentDir = targetPath.getParent(); + if (parentDir == null || !Files.exists(parentDir)) { + return AjaxJson.returnExceptionInfo("父目录不存在"); + } + if (!Files.isWritable(parentDir)) { + logger.error("目录无写权限: {}", parentDir.toAbsolutePath()); + return AjaxJson.returnExceptionInfo("目录无写权限"); + } + + // 使用NIO方式写入文件，避免内存OOM + inputStream = file.getInputStream(); + fileChannel = FileChannel.open(targetPath, + java.nio.file.StandardOpenOption.CREATE, + java.nio.file.StandardOpenOption.WRITE, + java.nio.file.StandardOpenOption.TRUNCATE_EXISTING); + + ReadableByteChannel readableByteChannel = Channels.newChannel(inputStream); + + // 使用transferFrom进行高效文件传输 + long transferred = fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE); + + // 验证文件是否成功写入 + if (!Files.exists(targetPath) || Files.size(targetPath) == 0) { + logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath()); + return AjaxJson.returnExceptionInfo("文件传输失败"); + } + + logger.info("文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath()); + + String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId; + String fieldName = + "file_name," + + "rec_id," + + "file_type," + + "file_len," + + "file_path," + + "page_no," + + "file_status," + + "is_divided," + + "file_des," + + "file_name_server"; + String valueName = "'" + originalFilename + "'" + "," + + "'" + recId + "'" + "," + + "'" + fileType + "'" + "," + + "'" + fileLen + "'" + "," + + "'" + file_path + "'" + "," + + "'" + pageNo + "'" + + ",1,-1," + + "'" + dir + "'" + "," + + "'" + file_name_server + "'"; + Map map = new HashMap(); + map.put("tableName", tableName + "_temp_file"); + map.put("fieldName", fieldName); + map.put("valueName", valueName); + danganguanliService.saveObject(map); + + if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) { + //生成一份pdf文件，用于归档章的操作 - 使用NIO + String newName_pdf = file_name_server.replace("." + fileType, ".pdf"); + Path sourcePath = targetPath; + Path pdfPath = Paths.get(dir, newName_pdf); + + boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString()); + if (!pdfCreated) { + logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath); + } else { + // 只有PDF生成成功才复制原始文件 + String newName_pdf_original = newName_pdf.replace(".pdf", "_original.pdf"); + Path originalPath = Paths.get(dir, newName_pdf_original); + try { + Files.copy(pdfPath, originalPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException e) { + logger.warn("复制PDF原始文件失败: {}", e.getMessage()); + } + } + } + + //mxf格式的文件需要转换一份mp4给前端展示 + if (fileType.equalsIgnoreCase("mxf")) { + String replaceMp4; + if ("MXF".equals(fileType)) { + replaceMp4 = targetPath.toString().replace(".MXF", ".mp4"); + } else { + replaceMp4 = targetPath.toString().replace(".mxf", ".mp4"); + } + VideoConvertUtil.convert(targetPath.toString(), replaceMp4); + } + + json = AjaxJson.returnInfo("上传文件成功"); + json.put("file", targetPath.toFile()); + json.put("file_name_server", file_name_server); + + } catch (Exception e) { + logger.error("上传文件失败: {}", file.getOriginalFilename(), e); + json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage()); + + // 清理失败的文件 + if (targetPath != null && Files.exists(targetPath)) { + try { + Files.deleteIfExists(targetPath); + logger.info("清理失败文件成功: {}", targetPath.toAbsolutePath()); + } catch (Exception deleteEx) { + logger.warn("清理失败文件时出错: {}", targetPath.toAbsolutePath(), deleteEx); + } + } + } finally { + // 确保资源被正确释放 + try { + if (fileChannel != null) { + fileChannel.close(); + } + } catch (IOException e) { + logger.warn("关闭FileChannel时出错: {}", e.getMessage()); + } + try { + if (inputStream != null) { + inputStream.close(); + } + } catch (IOException e) { + logger.warn("关闭InputStream时出错: {}", e.getMessage()); + } + } + return json; + } + + /** + * 零拷贝方式上传文件，最大程度优化内存使用，防止OOM + */ + private AjaxJson uploadFilesByPathAnjuanZeroCopy(MultipartFile file, String fondscode, String dir, Integer recId, String tableName, HttpServletRequest request) { + AjaxJson json = null; + Path targetPath = null; + ReadableByteChannel readableByteChannel = null; + FileChannel fileChannel = null; + + try { + String originalFilename = file.getOriginalFilename(); + if (StringUtil.isEmpty(originalFilename)) { + return AjaxJson.returnExceptionInfo("文件名为空"); + } + + int index = originalFilename.lastIndexOf(".") + 1; + if (index <= 0 || index >= originalFilename.length()) { + return AjaxJson.returnExceptionInfo("文件格式不正确"); + } + + String fileType = originalFilename.substring(index); + String file_name_server = StringUtil.generaterUUID() + "." + fileType; + + long fileLen = file.getSize() / 1024; + + Map map5 = new HashMap<>(); + map5.put("tableName", tableName + "_temp_file"); + map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 "); + int pageNo = danganguanliService.selectObjectCount(map5) + 1; + + // 使用NIO Path和Files API + targetPath = Paths.get(dir, file_name_server); + + // 检查目标文件是否已存在 + if (Files.exists(targetPath)) { + logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath()); + Files.deleteIfExists(targetPath); + } + + // 验证父目录是否存在且可写 + Path parentDir = targetPath.getParent(); + if (parentDir == null || !Files.exists(parentDir)) { + return AjaxJson.returnExceptionInfo("父目录不存在"); + } + if (!Files.isWritable(parentDir)) { + logger.error("目录无写权限: {}", parentDir.toAbsolutePath()); + return AjaxJson.returnExceptionInfo("目录无写权限"); + } + + // 零拷贝实现：直接使用FileChannel进行传输，避免数据在用户空间的拷贝 + try (InputStream inputStream = file.getInputStream()) { + readableByteChannel = Channels.newChannel(inputStream); + fileChannel = FileChannel.open(targetPath, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING); + + // 使用transferFrom实现零拷贝，数据直接从内核缓冲区传输到文件 + long transferred = 0; + long position = 0; + long count = Long.MAX_VALUE; + + // 分块传输大文件，避免单次传输过大导致的问题 + final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks + while (position < file.getSize()) { + long chunkSize = Math.min(CHUNK_SIZE, file.getSize() - position); + long transferredChunk = fileChannel.transferFrom(readableByteChannel, position, chunkSize); + if (transferredChunk == 0) { + break; // 传输完成 + } + position += transferredChunk; + transferred += transferredChunk; + } + + // 强制写入磁盘，确保数据持久化 + fileChannel.force(true); + + // 验证文件是否成功写入 + if (!Files.exists(targetPath) || Files.size(targetPath) == 0) { + logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath()); + return AjaxJson.returnExceptionInfo("文件传输失败"); + } + + logger.info("零拷贝文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath()); + } + + String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId; + String fieldName = + "file_name," + + "rec_id," + + "file_type," + + "file_len," + + "file_path," + + "page_no," + + "file_status," + + "is_divided," + + "file_des," + + "file_name_server"; + String valueName = "'" + originalFilename + "'" + "," + + "'" + recId + "'" + "," + + "'" + fileType + "'" + "," + + "'" + fileLen + "'" + "," + + "'" + file_path + "'" + "," + + "'" + pageNo + "'" + + ",1,-1," + + "'" + dir + "'" + "," + + "'" + file_name_server + "'"; + Map map = new HashMap(); + map.put("tableName", tableName + "_temp_file"); + map.put("fieldName", fieldName); + map.put("valueName", valueName); + danganguanliService.saveObject(map); + + if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) { + //生成一份pdf文件，用于归档章的操作 - 使用零拷贝 + String newName_pdf = file_name_server.replace("." + fileType, ".pdf"); + Path sourcePath = targetPath; + Path pdfPath = Paths.get(dir, newName_pdf); + + boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString()); + if (!pdfCreated) { + logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath); + } else { + // 只有PDF生成成功才复制原始文件 - 使用零拷贝 + String newName_pdf_original = newName_pdf.replace(".pdf", "_original.pdf"); + Path originalPath = Paths.get(dir, newName_pdf_original); + try { + // 使用零拷贝复制文件 + copyFileZeroCopy(pdfPath, originalPath); + } catch (IOException e) { + logger.warn("零拷贝复制PDF原始文件失败: {}", e.getMessage()); + // 降级到普通拷贝 + try { + Files.copy(pdfPath, originalPath, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException fallbackEx) { + logger.warn("复制PDF原始文件失败: {}", fallbackEx.getMessage()); + } + } + } + } + + //mxf格式的文件需要转换一份mp4给前端展示 + if (fileType.equalsIgnoreCase("mxf")) { + String replaceMp4; + if ("MXF".equals(fileType)) { + replaceMp4 = targetPath.toString().replace(".MXF", ".mp4"); + } else { + replaceMp4 = targetPath.toString().replace(".mxf", ".mp4"); + } + VideoConvertUtil.convert(targetPath.toString(), replaceMp4); + } + + json = AjaxJson.returnInfo("上传文件成功"); + json.put("file", targetPath.toFile()); + json.put("file_name_server", file_name_server); + + } catch (Exception e) { + logger.error("零拷贝上传文件失败: {}", file.getOriginalFilename(), e); + json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage()); + + // 清理失败的文件 + if (targetPath != null && Files.exists(targetPath)) { + try { + Files.deleteIfExists(targetPath); + logger.info("清理失败文件成功: {}", targetPath.toAbsolutePath()); + } catch (Exception deleteEx) { + logger.warn("清理失败文件时出错: {}", targetPath.toAbsolutePath(), deleteEx); + } + } + } finally { + // 确保资源被正确释放 + try { + if (fileChannel != null) { + fileChannel.close(); + } + } catch (IOException e) { + logger.warn("关闭FileChannel时出错: {}", e.getMessage()); + } + try { + if (readableByteChannel != null) { + readableByteChannel.close(); + } + } catch (IOException e) { + logger.warn("关闭ReadableByteChannel时出错: {}", e.getMessage()); + } + } + return json; + } + + /** + * 零拷贝文件复制方法 + */ + private void copyFileZeroCopy(Path source, Path target) throws IOException { + try (FileChannel sourceChannel = FileChannel.open(source, StandardOpenOption.READ); + FileChannel targetChannel = FileChannel.open(target, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING)) { + + long size = sourceChannel.size(); + long position = 0; + final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks + + while (position < size) { + long chunkSize = Math.min(CHUNK_SIZE, size - position); + long transferred = targetChannel.transferFrom(sourceChannel, position, chunkSize); + if (transferred == 0) { + break; + } + position += transferred; + } + + // 强制写入磁盘 + targetChannel.force(true); + } + } //==================================================================================