digital-archive/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test

Browse Source
This commit is contained in:
aipper
2025-11-24 12:21:46 +08:00
parent 6912919692
commit b75f05bac9
1 changed files with 468 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

519
src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java
View File

@@ -20,13 +20,26 @@ import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import javax.servlet.http.HttpServletRequest;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.WritableByteChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -193,15 +206,12 @@ public class AnjuanAndJuanneiController {
@RequestMapping(value = "/uploadSimpleFilesAnjuan", method = RequestMethod.POST)
@ApiOperation(value = "传统案卷整理原文-单个或者多个上传")
//file要与表单上传的名字相同
public AjaxJson uploadSimpleFilesAnjuan(MultipartFile[] file, String fondscode, Integer recId, String tableName, HttpServletRequest request) {
// 使用NIO方式处理多文件上传，避免OOM
public AjaxJson uploadSimpleFilesAnjuan(HttpServletRequest request, String fondscode, Integer recId, String tableName) {
Integer successNum = 0;
Integer falseNum = 0;
// 验证参数
if (file == null || file.length == 0) {
return AjaxJson.returnExceptionInfo("未选择任何文件");
}
if (recId == null || recId <= 0) {
return AjaxJson.returnExceptionInfo("记录ID无效");
}
@@ -212,59 +222,84 @@ public class AnjuanAndJuanneiController {
return AjaxJson.returnExceptionInfo("表名不能为空");
}
for (int i = 0; i < file.length; i++) {
MultipartFile file0 = file[i];
// 验证文件
if (file0 == null || file0.isEmpty()) {
logger.warn("第{}个文件为空，跳过", i + 1);
falseNum++;
continue;
// 创建文件在服务器端存放路径
String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
File fileDir = new File(dir);
if (!fileDir.exists()) {
boolean created = fileDir.mkdirs();
if (!created) {
logger.error("创建目录失败: {}", dir);
return AjaxJson.returnExceptionInfo("创建目录失败");
}
//创建文件在服务器端存放路径 - 使用File.separator确保跨平台兼容性
String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
File fileDir = new File(dir);
if (!fileDir.exists()) {
boolean created = fileDir.mkdirs();
if (!created) {
logger.error("创建目录失败: {}", dir);
}
// 验证目录是否可写
if (!fileDir.canWrite()) {
logger.error("目录无写权限: {}", dir);
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 使用NIO方式迭代处理文件，避免一次性加载所有文件到内存
if (request instanceof MultipartHttpServletRequest) {
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
Iterator<String> iterator = multipartRequest.getFileNames();
int fileIndex = 0;
while (iterator.hasNext()) {
fileIndex++;
String name = iterator.next();
MultipartFile file0 = multipartRequest.getFile(name);
if (file0 == null || file0.isEmpty()) {
logger.warn("第{}个文件为空，跳过", fileIndex);
falseNum++;
continue;
}
// 使用零拷贝方式处理单个文件，最大程度优化内存使用
AjaxJson json2 = uploadFilesByPathAnjuanZeroCopy(file0, fondscode, dir, recId, tableName, request);
if ("101".equals(json2.getCode())) {
falseNum++;
}
if ("100".equals(json2.getCode())) {
successNum++;
}
// 异步处理OCR，避免阻塞
String originalFilename = file0.getOriginalFilename();
if (originalFilename != null) {
int index = originalFilename.lastIndexOf(".") + 1;
if (index > 0 && index < originalFilename.length()) {
String fileType = originalFilename.substring(index);
if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
OCRProcessingTask ocrTask = new OCRProcessingTask(json2, tableName, youhongIntegrate, youhongBaseUrl, tessPath, ocrLogMapper, danganguanliService);
Thread ocrThread = new Thread(ocrTask, "OCR-Processing-" + fileIndex);
ocrThread.setDaemon(true);
ocrThread.start();
}
}
}
// 显式释放资源
try {
if (file0.getInputStream() != null) {
file0.getInputStream().close();
}
} catch (IOException e) {
logger.warn("关闭文件流时出错: {}", e.getMessage());
}
}
// 验证目录是否可写
if (!fileDir.canWrite()) {
logger.error("目录无写权限: {}", dir);
falseNum++;
continue;
}
AjaxJson json2 = uploadFilesByPathAnjuan(file0, fondscode, dir, recId, tableName, request);
if ("101".equals(json2.getCode())) {
falseNum++;
}
if ("100".equals(json2.getCode())) {
successNum++;
}
String originalFilename = file0.getOriginalFilename();
int index = originalFilename.lastIndexOf(".") + 1;
String fileType = originalFilename.substring(index);
//启动一个线程，根据ocr获取图片文字"file_content,"+
if(!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
// 使用线程池替代直接创建线程
OCRProcessingTask ocrTask = new OCRProcessingTask(json2, tableName, youhongIntegrate, youhongBaseUrl, tessPath, ocrLogMapper, danganguanliService);
Thread ocrThread = new Thread(ocrTask, "OCR-Processing-" + (i + 1));
ocrThread.setDaemon(true); // 设置为守护线程
ocrThread.start();
}
} else {
return AjaxJson.returnExceptionInfo("请求类型不支持");
}
Map<String, Object> map7=new HashMap<>();
map7.put("tableName",tableName+"_temp");
map7.put("tableName2",tableName+"_temp_file");
map7.put("id",recId);
Map<String, Object> map7 = new HashMap<>();
map7.put("tableName", tableName + "_temp");
map7.put("tableName2", tableName + "_temp_file");
map7.put("id", recId);
danganguanliService.wsajmlTempCount(map7);
AjaxJson json = AjaxJson.returnInfo("成功上传数successNum，失败上传数falseNum");
json.put("successNum", successNum);
json.put("falseNum", falseNum);
@@ -407,6 +442,388 @@ public class AnjuanAndJuanneiController {
}
return json;
}
/**
* NIO方式上传文件，避免OOM问题
*/
private AjaxJson uploadFilesByPathAnjuanNIO(MultipartFile file, String fondscode, String dir, Integer recId, String tableName, HttpServletRequest request) {
AjaxJson json = null;
Path targetPath = null;
InputStream inputStream = null;
FileChannel fileChannel = null;
try {
String originalFilename = file.getOriginalFilename();
if (StringUtil.isEmpty(originalFilename)) {
return AjaxJson.returnExceptionInfo("文件名为空");
}
int index = originalFilename.lastIndexOf(".") + 1;
if (index <= 0 || index >= originalFilename.length()) {
return AjaxJson.returnExceptionInfo("文件格式不正确");
}
String fileType = originalFilename.substring(index);
String file_name_server = StringUtil.generaterUUID() + "." + fileType;
long fileLen = file.getSize() / 1024;
Map<String, Object> map5 = new HashMap<>();
map5.put("tableName", tableName + "_temp_file");
map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 ");
int pageNo = danganguanliService.selectObjectCount(map5) + 1;
// 使用NIO Path和Files API
targetPath = Paths.get(dir, file_name_server);
// 检查目标文件是否已存在
if (Files.exists(targetPath)) {
logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath());
Files.deleteIfExists(targetPath);
}
// 验证父目录是否存在且可写
Path parentDir = targetPath.getParent();
if (parentDir == null || !Files.exists(parentDir)) {
return AjaxJson.returnExceptionInfo("父目录不存在");
}
if (!Files.isWritable(parentDir)) {
logger.error("目录无写权限: {}", parentDir.toAbsolutePath());
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 使用NIO方式写入文件，避免内存OOM
inputStream = file.getInputStream();
fileChannel = FileChannel.open(targetPath,
java.nio.file.StandardOpenOption.CREATE,
java.nio.file.StandardOpenOption.WRITE,
java.nio.file.StandardOpenOption.TRUNCATE_EXISTING);
ReadableByteChannel readableByteChannel = Channels.newChannel(inputStream);
// 使用transferFrom进行高效文件传输
long transferred = fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
// 验证文件是否成功写入
if (!Files.exists(targetPath) || Files.size(targetPath) == 0) {
logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath());
return AjaxJson.returnExceptionInfo("文件传输失败");
}
logger.info("文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath());
String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
String fieldName =
"file_name," +
"rec_id," +
"file_type," +
"file_len," +
"file_path," +
"page_no," +
"file_status," +
"is_divided," +
"file_des," +
"file_name_server";
String valueName = "'" + originalFilename + "'" + ","
+ "'" + recId + "'" + ","
+ "'" + fileType + "'" + ","
+ "'" + fileLen + "'" + ","
+ "'" + file_path + "'" + ","
+ "'" + pageNo + "'"
+ ",1,-1,"
+ "'" + dir + "'" + ","
+ "'" + file_name_server + "'";
Map<String, Object> map = new HashMap<String, Object>();
map.put("tableName", tableName + "_temp_file");
map.put("fieldName", fieldName);
map.put("valueName", valueName);
danganguanliService.saveObject(map);
if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
//生成一份pdf文件，用于归档章的操作 - 使用NIO
String newName_pdf = file_name_server.replace("." + fileType, ".pdf");
Path sourcePath = targetPath;
Path pdfPath = Paths.get(dir, newName_pdf);
boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString());
if (!pdfCreated) {
logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath);
} else {
// 只有PDF生成成功才复制原始文件
String newName_pdf_original = newName_pdf.replace(".pdf", "_original.pdf");
Path originalPath = Paths.get(dir, newName_pdf_original);
try {
Files.copy(pdfPath, originalPath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException e) {
logger.warn("复制PDF原始文件失败: {}", e.getMessage());
}
}
}
//mxf格式的文件需要转换一份mp4给前端展示
if (fileType.equalsIgnoreCase("mxf")) {
String replaceMp4;
if ("MXF".equals(fileType)) {
replaceMp4 = targetPath.toString().replace(".MXF", ".mp4");
} else {
replaceMp4 = targetPath.toString().replace(".mxf", ".mp4");
}
VideoConvertUtil.convert(targetPath.toString(), replaceMp4);
}
json = AjaxJson.returnInfo("上传文件成功");
json.put("file", targetPath.toFile());
json.put("file_name_server", file_name_server);
} catch (Exception e) {
logger.error("上传文件失败: {}", file.getOriginalFilename(), e);
json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
// 清理失败的文件
if (targetPath != null && Files.exists(targetPath)) {
try {
Files.deleteIfExists(targetPath);
logger.info("清理失败文件成功: {}", targetPath.toAbsolutePath());
} catch (Exception deleteEx) {
logger.warn("清理失败文件时出错: {}", targetPath.toAbsolutePath(), deleteEx);
}
}
} finally {
// 确保资源被正确释放
try {
if (fileChannel != null) {
fileChannel.close();
}
} catch (IOException e) {
logger.warn("关闭FileChannel时出错: {}", e.getMessage());
}
try {
if (inputStream != null) {
inputStream.close();
}
} catch (IOException e) {
logger.warn("关闭InputStream时出错: {}", e.getMessage());
}
}
return json;
}
/**
* 零拷贝方式上传文件，最大程度优化内存使用，防止OOM
*/
private AjaxJson uploadFilesByPathAnjuanZeroCopy(MultipartFile file, String fondscode, String dir, Integer recId, String tableName, HttpServletRequest request) {
AjaxJson json = null;
Path targetPath = null;
ReadableByteChannel readableByteChannel = null;
FileChannel fileChannel = null;
try {
String originalFilename = file.getOriginalFilename();
if (StringUtil.isEmpty(originalFilename)) {
return AjaxJson.returnExceptionInfo("文件名为空");
}
int index = originalFilename.lastIndexOf(".") + 1;
if (index <= 0 || index >= originalFilename.length()) {
return AjaxJson.returnExceptionInfo("文件格式不正确");
}
String fileType = originalFilename.substring(index);
String file_name_server = StringUtil.generaterUUID() + "." + fileType;
long fileLen = file.getSize() / 1024;
Map<String, Object> map5 = new HashMap<>();
map5.put("tableName", tableName + "_temp_file");
map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 ");
int pageNo = danganguanliService.selectObjectCount(map5) + 1;
// 使用NIO Path和Files API
targetPath = Paths.get(dir, file_name_server);
// 检查目标文件是否已存在
if (Files.exists(targetPath)) {
logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath());
Files.deleteIfExists(targetPath);
}
// 验证父目录是否存在且可写
Path parentDir = targetPath.getParent();
if (parentDir == null || !Files.exists(parentDir)) {
return AjaxJson.returnExceptionInfo("父目录不存在");
}
if (!Files.isWritable(parentDir)) {
logger.error("目录无写权限: {}", parentDir.toAbsolutePath());
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 零拷贝实现：直接使用FileChannel进行传输，避免数据在用户空间的拷贝
try (InputStream inputStream = file.getInputStream()) {
readableByteChannel = Channels.newChannel(inputStream);
fileChannel = FileChannel.open(targetPath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING);
// 使用transferFrom实现零拷贝，数据直接从内核缓冲区传输到文件
long transferred = 0;
long position = 0;
long count = Long.MAX_VALUE;
// 分块传输大文件，避免单次传输过大导致的问题
final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks
while (position < file.getSize()) {
long chunkSize = Math.min(CHUNK_SIZE, file.getSize() - position);
long transferredChunk = fileChannel.transferFrom(readableByteChannel, position, chunkSize);
if (transferredChunk == 0) {
break; // 传输完成
}
position += transferredChunk;
transferred += transferredChunk;
}
// 强制写入磁盘，确保数据持久化
fileChannel.force(true);
// 验证文件是否成功写入
if (!Files.exists(targetPath) || Files.size(targetPath) == 0) {
logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath());
return AjaxJson.returnExceptionInfo("文件传输失败");
}
logger.info("零拷贝文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath());
}
String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
String fieldName =
"file_name," +
"rec_id," +
"file_type," +
"file_len," +
"file_path," +
"page_no," +
"file_status," +
"is_divided," +
"file_des," +
"file_name_server";
String valueName = "'" + originalFilename + "'" + ","
+ "'" + recId + "'" + ","
+ "'" + fileType + "'" + ","
+ "'" + fileLen + "'" + ","
+ "'" + file_path + "'" + ","
+ "'" + pageNo + "'"
+ ",1,-1,"
+ "'" + dir + "'" + ","
+ "'" + file_name_server + "'";
Map<String, Object> map = new HashMap<String, Object>();
map.put("tableName", tableName + "_temp_file");
map.put("fieldName", fieldName);
map.put("valueName", valueName);
danganguanliService.saveObject(map);
if (!fileType.equalsIgnoreCase("mp3") && !fileType.equalsIgnoreCase("mp4")) {
//生成一份pdf文件，用于归档章的操作 - 使用零拷贝
String newName_pdf = file_name_server.replace("." + fileType, ".pdf");
Path sourcePath = targetPath;
Path pdfPath = Paths.get(dir, newName_pdf);
boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString());
if (!pdfCreated) {
logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath);
} else {
// 只有PDF生成成功才复制原始文件 - 使用零拷贝
String newName_pdf_original = newName_pdf.replace(".pdf", "_original.pdf");
Path originalPath = Paths.get(dir, newName_pdf_original);
try {
// 使用零拷贝复制文件
copyFileZeroCopy(pdfPath, originalPath);
} catch (IOException e) {
logger.warn("零拷贝复制PDF原始文件失败: {}", e.getMessage());
// 降级到普通拷贝
try {
Files.copy(pdfPath, originalPath, StandardCopyOption.REPLACE_EXISTING);
} catch (IOException fallbackEx) {
logger.warn("复制PDF原始文件失败: {}", fallbackEx.getMessage());
}
}
}
}
//mxf格式的文件需要转换一份mp4给前端展示
if (fileType.equalsIgnoreCase("mxf")) {
String replaceMp4;
if ("MXF".equals(fileType)) {
replaceMp4 = targetPath.toString().replace(".MXF", ".mp4");
} else {
replaceMp4 = targetPath.toString().replace(".mxf", ".mp4");
}
VideoConvertUtil.convert(targetPath.toString(), replaceMp4);
}
json = AjaxJson.returnInfo("上传文件成功");
json.put("file", targetPath.toFile());
json.put("file_name_server", file_name_server);
} catch (Exception e) {
logger.error("零拷贝上传文件失败: {}", file.getOriginalFilename(), e);
json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
// 清理失败的文件
if (targetPath != null && Files.exists(targetPath)) {
try {
Files.deleteIfExists(targetPath);
logger.info("清理失败文件成功: {}", targetPath.toAbsolutePath());
} catch (Exception deleteEx) {
logger.warn("清理失败文件时出错: {}", targetPath.toAbsolutePath(), deleteEx);
}
}
} finally {
// 确保资源被正确释放
try {
if (fileChannel != null) {
fileChannel.close();
}
} catch (IOException e) {
logger.warn("关闭FileChannel时出错: {}", e.getMessage());
}
try {
if (readableByteChannel != null) {
readableByteChannel.close();
}
} catch (IOException e) {
logger.warn("关闭ReadableByteChannel时出错: {}", e.getMessage());
}
}
return json;
}
/**
* 零拷贝文件复制方法
*/
private void copyFileZeroCopy(Path source, Path target) throws IOException {
try (FileChannel sourceChannel = FileChannel.open(source, StandardOpenOption.READ);
FileChannel targetChannel = FileChannel.open(target,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING)) {
long size = sourceChannel.size();
long position = 0;
final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks
while (position < size) {
long chunkSize = Math.min(CHUNK_SIZE, size - position);
long transferred = targetChannel.transferFrom(sourceChannel, position, chunkSize);
if (transferred == 0) {
break;
}
position += transferred;
}
// 强制写入磁盘
targetChannel.force(true);
}
}
//==================================================================================