test

src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java

@@ -20,13 +20,26 @@ import org.springframework.web.bind.annotation.RequestMapping;
 import org.springframework.web.bind.annotation.RequestMethod;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.multipart.MultipartHttpServletRequest;
 
 import javax.servlet.http.HttpServletRequest;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -193,15 +206,12 @@ public class AnjuanAndJuanneiController {
 
     @RequestMapping(value = "/uploadSimpleFilesAnjuan", method = RequestMethod.POST)
     @ApiOperation(value = "传统案卷整理原文-单个或者多个上传")
-    //file要与表单上传的名字相同
+    // 使用NIO方式处理多文件上传，避免OOM
-    public AjaxJson uploadSimpleFilesAnjuan(MultipartFile[] file, String fondscode, Integer recId, String tableName, HttpServletRequest request) {
+    public AjaxJson uploadSimpleFilesAnjuan(HttpServletRequest request, String fondscode, Integer recId, String tableName) {
         Integer successNum = 0;
         Integer falseNum = 0;
         
         // 验证参数
-        if (file == null || file.length == 0) {
-            return AjaxJson.returnExceptionInfo("未选择任何文件");
-        }
         if (recId == null || recId <= 0) {
             return AjaxJson.returnExceptionInfo("记录ID无效");
         }
@@ -212,59 +222,84 @@ public class AnjuanAndJuanneiController {
             return AjaxJson.returnExceptionInfo("表名不能为空");
         }
         
-        for (int i = 0; i < file.length; i++) {
+        // 创建文件在服务器端存放路径
-            MultipartFile file0 = file[i];
-            
-            // 验证文件
-            if (file0 == null || file0.isEmpty()) {
-                logger.warn("第{}个文件为空，跳过", i + 1);
-                falseNum++;
-                continue;
-            }
-
-            //创建文件在服务器端存放路径 - 使用File.separator确保跨平台兼容性
         String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
         File fileDir = new File(dir);
         if (!fileDir.exists()) {
             boolean created = fileDir.mkdirs();
             if (!created) {
                 logger.error("创建目录失败: {}", dir);
-                    falseNum++;
+                return AjaxJson.returnExceptionInfo("创建目录失败");
-                    continue;
             }
         }
         
         // 验证目录是否可写
         if (!fileDir.canWrite()) {
             logger.error("目录无写权限: {}", dir);
+            return AjaxJson.returnExceptionInfo("目录无写权限");
+        }
+        
+        // 使用NIO方式迭代处理文件，避免一次性加载所有文件到内存
+        if (request instanceof MultipartHttpServletRequest) {
+            MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
+            Iterator<String> iterator = multipartRequest.getFileNames();
+            int fileIndex = 0;
+            
+            while (iterator.hasNext()) {
+                fileIndex++;
+                String name = iterator.next();
+                MultipartFile file0 = multipartRequest.getFile(name);
+                
+                if (file0 == null || file0.isEmpty()) {
+                    logger.warn("第{}个文件为空，跳过", fileIndex);
                     falseNum++;
                     continue;
                 }
                 
-            AjaxJson json2 = uploadFilesByPathAnjuan(file0, fondscode, dir, recId, tableName, request);
+                // 使用零拷贝方式处理单个文件，最大程度优化内存使用
+                AjaxJson json2 = uploadFilesByPathAnjuanZeroCopy(file0, fondscode, dir, recId, tableName, request);
+                
                 if ("101".equals(json2.getCode())) {
                     falseNum++;
                 }
                 if ("100".equals(json2.getCode())) {
                     successNum++;
                 }
+                
+                // 异步处理OCR，避免阻塞
                 String originalFilename = file0.getOriginalFilename();
+                if (originalFilename != null) {
                     int index = originalFilename.lastIndexOf(".") + 1;
+                    if (index > 0 && index < originalFilename.length()) {
                         String fileType = originalFilename.substring(index);
-            //启动一个线程，根据ocr获取图片文字"file_content,"+
                         if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
-                // 使用线程池替代直接创建线程
                             OCRProcessingTask ocrTask = new OCRProcessingTask(json2, tableName, youhongIntegrate, youhongBaseUrl, tessPath, ocrLogMapper, danganguanliService);
-                Thread ocrThread = new Thread(ocrTask, "OCR-Processing-" + (i + 1));
+                            Thread ocrThread = new Thread(ocrTask, "OCR-Processing-" + fileIndex);
-                ocrThread.setDaemon(true); // 设置为守护线程
+                            ocrThread.setDaemon(true);
                             ocrThread.start();
                         }
                     }
+                }
+                
+                // 显式释放资源
+                try {
+                    if (file0.getInputStream() != null) {
+                        file0.getInputStream().close();
+                    }
+                } catch (IOException e) {
+                    logger.warn("关闭文件流时出错: {}", e.getMessage());
+                }
+            }
+        } else {
+            return AjaxJson.returnExceptionInfo("请求类型不支持");
+        }
+        
         Map<String, Object> map7 = new HashMap<>();
         map7.put("tableName", tableName + "_temp");
         map7.put("tableName2", tableName + "_temp_file");
         map7.put("id", recId);
         danganguanliService.wsajmlTempCount(map7);
+        
         AjaxJson json = AjaxJson.returnInfo("成功上传数successNum，失败上传数falseNum");
         json.put("successNum", successNum);
         json.put("falseNum", falseNum);
@@ -408,6 +443,388 @@ public class AnjuanAndJuanneiController {
         return json;
     }
     
+    /**
+     * NIO方式上传文件，避免OOM问题
+     */
+    private AjaxJson uploadFilesByPathAnjuanNIO(MultipartFile file, String fondscode, String dir, Integer recId, String tableName, HttpServletRequest request) {
+        AjaxJson json = null;
+        Path targetPath = null;
+        InputStream inputStream = null;
+        FileChannel fileChannel = null;
+        
+        try {
+            String originalFilename = file.getOriginalFilename();
+            if (StringUtil.isEmpty(originalFilename)) {
+                return AjaxJson.returnExceptionInfo("文件名为空");
+            }
+            
+            int index = originalFilename.lastIndexOf(".") + 1;
+            if (index <= 0 || index >= originalFilename.length()) {
+                return AjaxJson.returnExceptionInfo("文件格式不正确");
+            }
+            
+            String fileType = originalFilename.substring(index);
+            String file_name_server = StringUtil.generaterUUID() + "." + fileType;
+
+            long fileLen = file.getSize() / 1024;
+
+            Map<String, Object> map5 = new HashMap<>();
+            map5.put("tableName", tableName + "_temp_file");
+            map5.put("conditionSql", "rec_id= '" + recId + "'  and file_status=1 ");
+            int pageNo = danganguanliService.selectObjectCount(map5) + 1;
+            
+            // 使用NIO Path和Files API
+            targetPath = Paths.get(dir, file_name_server);
+            
+            // 检查目标文件是否已存在
+            if (Files.exists(targetPath)) {
+                logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath());
+                Files.deleteIfExists(targetPath);
+            }
+            
+            // 验证父目录是否存在且可写
+            Path parentDir = targetPath.getParent();
+            if (parentDir == null || !Files.exists(parentDir)) {
+                return AjaxJson.returnExceptionInfo("父目录不存在");
+            }
+            if (!Files.isWritable(parentDir)) {
+                logger.error("目录无写权限: {}", parentDir.toAbsolutePath());
+                return AjaxJson.returnExceptionInfo("目录无写权限");
+            }
+            
+            // 使用NIO方式写入文件，避免内存OOM
+            inputStream = file.getInputStream();
+            fileChannel = FileChannel.open(targetPath, 
+                java.nio.file.StandardOpenOption.CREATE, 
+                java.nio.file.StandardOpenOption.WRITE,
+                java.nio.file.StandardOpenOption.TRUNCATE_EXISTING);
+            
+            ReadableByteChannel readableByteChannel = Channels.newChannel(inputStream);
+            
+            // 使用transferFrom进行高效文件传输
+            long transferred = fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
+            
+            // 验证文件是否成功写入
+            if (!Files.exists(targetPath) || Files.size(targetPath) == 0) {
+                logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath());
+                return AjaxJson.returnExceptionInfo("文件传输失败");
+            }
+            
+            logger.info("文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath());
+
+            String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
+            String fieldName =
+                    "file_name," +
+                            "rec_id," +
+                            "file_type," +
+                            "file_len," +
+                            "file_path," +
+                            "page_no," +
+                            "file_status," +
+                            "is_divided," +
+                            "file_des," +
+                            "file_name_server";
+            String valueName = "'" + originalFilename + "'" + ","
+                    + "'" + recId + "'" + ","
+                    + "'" + fileType + "'" + ","
+                    + "'" + fileLen + "'" + ","
+                    + "'" + file_path + "'" + ","
+                    + "'" + pageNo + "'"
+                    + ",1,-1,"
+                    + "'" + dir + "'" + ","
+                    + "'" + file_name_server + "'";
+            Map<String, Object> map = new HashMap<String, Object>();
+            map.put("tableName", tableName + "_temp_file");
+            map.put("fieldName", fieldName);
+            map.put("valueName", valueName);
+            danganguanliService.saveObject(map);
+
+            if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
+                //生成一份pdf文件，用于归档章的操作 - 使用NIO
+                String newName_pdf = file_name_server.replace("." + fileType, ".pdf");
+                Path sourcePath = targetPath;
+                Path pdfPath = Paths.get(dir, newName_pdf);
+                
+                boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString());
+                if (!pdfCreated) {
+                    logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath);
+                } else {
+                    // 只有PDF生成成功才复制原始文件
+                    String newName_pdf_original = newName_pdf.replace(".pdf", "_original.pdf");
+                    Path originalPath = Paths.get(dir, newName_pdf_original);
+                    try {
+                        Files.copy(pdfPath, originalPath, StandardCopyOption.REPLACE_EXISTING);
+                    } catch (IOException e) {
+                        logger.warn("复制PDF原始文件失败: {}", e.getMessage());
+                    }
+                }
+            }
+            
+            //mxf格式的文件需要转换一份mp4给前端展示
+            if (fileType.equalsIgnoreCase("mxf")) {
+                String replaceMp4;
+                if ("MXF".equals(fileType)) {
+                    replaceMp4 = targetPath.toString().replace(".MXF", ".mp4");
+                } else {
+                    replaceMp4 = targetPath.toString().replace(".mxf", ".mp4");
+                }
+                VideoConvertUtil.convert(targetPath.toString(), replaceMp4);
+            }
+            
+            json = AjaxJson.returnInfo("上传文件成功");
+            json.put("file", targetPath.toFile());
+            json.put("file_name_server", file_name_server);
+            
+        } catch (Exception e) {
+            logger.error("上传文件失败: {}", file.getOriginalFilename(), e);
+            json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
+            
+            // 清理失败的文件
+            if (targetPath != null && Files.exists(targetPath)) {
+                try {
+                    Files.deleteIfExists(targetPath);
+                    logger.info("清理失败文件成功: {}", targetPath.toAbsolutePath());
+                } catch (Exception deleteEx) {
+                    logger.warn("清理失败文件时出错: {}", targetPath.toAbsolutePath(), deleteEx);
+                }
+            }
+        } finally {
+            // 确保资源被正确释放
+            try {
+                if (fileChannel != null) {
+                    fileChannel.close();
+                }
+            } catch (IOException e) {
+                logger.warn("关闭FileChannel时出错: {}", e.getMessage());
+            }
+            try {
+                if (inputStream != null) {
+                    inputStream.close();
+                }
+            } catch (IOException e) {
+                logger.warn("关闭InputStream时出错: {}", e.getMessage());
+            }
+        }
+        return json;
+    }
+    
+    /**
+     * 零拷贝方式上传文件，最大程度优化内存使用，防止OOM
+     */
+    private AjaxJson uploadFilesByPathAnjuanZeroCopy(MultipartFile file, String fondscode, String dir, Integer recId, String tableName, HttpServletRequest request) {
+        AjaxJson json = null;
+        Path targetPath = null;
+        ReadableByteChannel readableByteChannel = null;
+        FileChannel fileChannel = null;
+        
+        try {
+            String originalFilename = file.getOriginalFilename();
+            if (StringUtil.isEmpty(originalFilename)) {
+                return AjaxJson.returnExceptionInfo("文件名为空");
+            }
+            
+            int index = originalFilename.lastIndexOf(".") + 1;
+            if (index <= 0 || index >= originalFilename.length()) {
+                return AjaxJson.returnExceptionInfo("文件格式不正确");
+            }
+            
+            String fileType = originalFilename.substring(index);
+            String file_name_server = StringUtil.generaterUUID() + "." + fileType;
+
+            long fileLen = file.getSize() / 1024;
+
+            Map<String, Object> map5 = new HashMap<>();
+            map5.put("tableName", tableName + "_temp_file");
+            map5.put("conditionSql", "rec_id= '" + recId + "'  and file_status=1 ");
+            int pageNo = danganguanliService.selectObjectCount(map5) + 1;
+            
+            // 使用NIO Path和Files API
+            targetPath = Paths.get(dir, file_name_server);
+            
+            // 检查目标文件是否已存在
+            if (Files.exists(targetPath)) {
+                logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath());
+                Files.deleteIfExists(targetPath);
+            }
+            
+            // 验证父目录是否存在且可写
+            Path parentDir = targetPath.getParent();
+            if (parentDir == null || !Files.exists(parentDir)) {
+                return AjaxJson.returnExceptionInfo("父目录不存在");
+            }
+            if (!Files.isWritable(parentDir)) {
+                logger.error("目录无写权限: {}", parentDir.toAbsolutePath());
+                return AjaxJson.returnExceptionInfo("目录无写权限");
+            }
+            
+            // 零拷贝实现：直接使用FileChannel进行传输，避免数据在用户空间的拷贝
+            try (InputStream inputStream = file.getInputStream()) {
+                readableByteChannel = Channels.newChannel(inputStream);
+                fileChannel = FileChannel.open(targetPath, 
+                    StandardOpenOption.CREATE, 
+                    StandardOpenOption.WRITE,
+                    StandardOpenOption.TRUNCATE_EXISTING);
+                
+                // 使用transferFrom实现零拷贝，数据直接从内核缓冲区传输到文件
+                long transferred = 0;
+                long position = 0;
+                long count = Long.MAX_VALUE;
+                
+                // 分块传输大文件，避免单次传输过大导致的问题
+                final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks
+                while (position < file.getSize()) {
+                    long chunkSize = Math.min(CHUNK_SIZE, file.getSize() - position);
+                    long transferredChunk = fileChannel.transferFrom(readableByteChannel, position, chunkSize);
+                    if (transferredChunk == 0) {
+                        break; // 传输完成
+                    }
+                    position += transferredChunk;
+                    transferred += transferredChunk;
+                }
+                
+                // 强制写入磁盘，确保数据持久化
+                fileChannel.force(true);
+                
+                // 验证文件是否成功写入
+                if (!Files.exists(targetPath) || Files.size(targetPath) == 0) {
+                    logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath());
+                    return AjaxJson.returnExceptionInfo("文件传输失败");
+                }
+                
+                logger.info("零拷贝文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath());
+            }
+
+            String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
+            String fieldName =
+                    "file_name," +
+                            "rec_id," +
+                            "file_type," +
+                            "file_len," +
+                            "file_path," +
+                            "page_no," +
+                            "file_status," +
+                            "is_divided," +
+                            "file_des," +
+                            "file_name_server";
+            String valueName = "'" + originalFilename + "'" + ","
+                    + "'" + recId + "'" + ","
+                    + "'" + fileType + "'" + ","
+                    + "'" + fileLen + "'" + ","
+                    + "'" + file_path + "'" + ","
+                    + "'" + pageNo + "'"
+                    + ",1,-1,"
+                    + "'" + dir + "'" + ","
+                    + "'" + file_name_server + "'";
+            Map<String, Object> map = new HashMap<String, Object>();
+            map.put("tableName", tableName + "_temp_file");
+            map.put("fieldName", fieldName);
+            map.put("valueName", valueName);
+            danganguanliService.saveObject(map);
+
+            if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
+                //生成一份pdf文件，用于归档章的操作 - 使用零拷贝
+                String newName_pdf = file_name_server.replace("." + fileType, ".pdf");
+                Path sourcePath = targetPath;
+                Path pdfPath = Paths.get(dir, newName_pdf);
+                
+                boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString());
+                if (!pdfCreated) {
+                    logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath);
+                } else {
+                    // 只有PDF生成成功才复制原始文件 - 使用零拷贝
+                    String newName_pdf_original = newName_pdf.replace(".pdf", "_original.pdf");
+                    Path originalPath = Paths.get(dir, newName_pdf_original);
+                    try {
+                        // 使用零拷贝复制文件
+                        copyFileZeroCopy(pdfPath, originalPath);
+                    } catch (IOException e) {
+                        logger.warn("零拷贝复制PDF原始文件失败: {}", e.getMessage());
+                        // 降级到普通拷贝
+                        try {
+                            Files.copy(pdfPath, originalPath, StandardCopyOption.REPLACE_EXISTING);
+                        } catch (IOException fallbackEx) {
+                            logger.warn("复制PDF原始文件失败: {}", fallbackEx.getMessage());
+                        }
+                    }
+                }
+            }
+            
+            //mxf格式的文件需要转换一份mp4给前端展示
+            if (fileType.equalsIgnoreCase("mxf")) {
+                String replaceMp4;
+                if ("MXF".equals(fileType)) {
+                    replaceMp4 = targetPath.toString().replace(".MXF", ".mp4");
+                } else {
+                    replaceMp4 = targetPath.toString().replace(".mxf", ".mp4");
+                }
+                VideoConvertUtil.convert(targetPath.toString(), replaceMp4);
+            }
+            
+            json = AjaxJson.returnInfo("上传文件成功");
+            json.put("file", targetPath.toFile());
+            json.put("file_name_server", file_name_server);
+            
+        } catch (Exception e) {
+            logger.error("零拷贝上传文件失败: {}", file.getOriginalFilename(), e);
+            json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
+            
+            // 清理失败的文件
+            if (targetPath != null && Files.exists(targetPath)) {
+                try {
+                    Files.deleteIfExists(targetPath);
+                    logger.info("清理失败文件成功: {}", targetPath.toAbsolutePath());
+                } catch (Exception deleteEx) {
+                    logger.warn("清理失败文件时出错: {}", targetPath.toAbsolutePath(), deleteEx);
+                }
+            }
+        } finally {
+            // 确保资源被正确释放
+            try {
+                if (fileChannel != null) {
+                    fileChannel.close();
+                }
+            } catch (IOException e) {
+                logger.warn("关闭FileChannel时出错: {}", e.getMessage());
+            }
+            try {
+                if (readableByteChannel != null) {
+                    readableByteChannel.close();
+                }
+            } catch (IOException e) {
+                logger.warn("关闭ReadableByteChannel时出错: {}", e.getMessage());
+            }
+        }
+        return json;
+    }
+    
+    /**
+     * 零拷贝文件复制方法
+     */
+    private void copyFileZeroCopy(Path source, Path target) throws IOException {
+        try (FileChannel sourceChannel = FileChannel.open(source, StandardOpenOption.READ);
+             FileChannel targetChannel = FileChannel.open(target, 
+                 StandardOpenOption.CREATE, 
+                 StandardOpenOption.WRITE,
+                 StandardOpenOption.TRUNCATE_EXISTING)) {
+            
+            long size = sourceChannel.size();
+            long position = 0;
+            final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks
+            
+            while (position < size) {
+                long chunkSize = Math.min(CHUNK_SIZE, size - position);
+                long transferred = targetChannel.transferFrom(sourceChannel, position, chunkSize);
+                if (transferred == 0) {
+                    break;
+                }
+                position += transferred;
+            }
+            
+            // 强制写入磁盘
+            targetChannel.force(true);
+        }
+    }
+
     //==================================================================================