digital-archive/server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test

Browse Source
This commit is contained in:
aipper
2025-11-24 16:14:12 +08:00
parent e604bacfa6
commit 85953347db
1 changed files with 79 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

153
src/main/java/com/point/strategy/originBatchUpload/AnjuanAndJuanneiController.java
View File

@@ -239,7 +239,7 @@ public class AnjuanAndJuanneiController {
public AjaxJson uploadSimpleFilesAnjuan(HttpServletRequest request, String fondscode, Integer recId, String tableName) {
Integer successNum = 0;
Integer falseNum = 0;
// 验证参数
if (recId == null || recId <= 0) {
return AjaxJson.returnExceptionInfo("记录ID无效");
@@ -250,7 +250,7 @@ public class AnjuanAndJuanneiController {
if (StringUtil.isEmpty(tableName)) {
return AjaxJson.returnExceptionInfo("表名不能为空");
}
// 创建文件在服务器端存放路径
String dir = uploadPath + "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
File fileDir = new File(dir);
@@ -261,44 +261,49 @@ public class AnjuanAndJuanneiController {
return AjaxJson.returnExceptionInfo("创建目录失败");
}
}
// 验证目录是否可写
if (!fileDir.canWrite()) {
logger.error("目录无写权限: {}", dir);
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 使用NIO方式迭代处理文件，避免一次性加载所有文件到内存
if (request instanceof MultipartHttpServletRequest) {
MultipartHttpServletRequest multipartRequest = (MultipartHttpServletRequest) request;
Iterator<String> iterator = multipartRequest.getFileNames();
int fileIndex = 0;
while (iterator.hasNext()) {
// 获取所有文件，包括同名字段的多个文件
Map<String, MultipartFile> fileMap = multipartRequest.getFileMap();
logger.info("接收到文件数量: {}", fileMap.size());
int fileIndex = 0;
for (Map.Entry<String, MultipartFile> entry : fileMap.entrySet()) {
boolean acquired = false;
try {
uploadSemaphore.acquire(); // 最多并行处理3个文件
acquired = true;
fileIndex++;
String name = iterator.next();
MultipartFile file0 = multipartRequest.getFile(name);
String fieldName = entry.getKey();
MultipartFile file0 = entry.getValue();
logger.info("处理第{}个文件: 字段名={}, 文件名={}", fileIndex, fieldName, file0.getOriginalFilename());
if (file0 == null || file0.isEmpty()) {
logger.warn("第{}个文件为空，跳过", fileIndex);
falseNum++;
continue;
}
// 使用零拷贝方式处理单个文件，最大程度优化内存使用
AjaxJson json2 = uploadFilesByPathAnjuanZeroCopy(file0, fondscode, dir, recId, tableName, request);
if ("101".equals(json2.getCode())) {
falseNum++;
}
if ("100".equals(json2.getCode())) {
successNum++;
}
// 异步处理OCR，避免阻塞
String originalFilename = file0.getOriginalFilename();
if (originalFilename != null) {
@@ -315,7 +320,7 @@ public class AnjuanAndJuanneiController {
}
}
}
// 显式释放资源
try {
if (file0.getInputStream() != null) {
@@ -337,14 +342,14 @@ public class AnjuanAndJuanneiController {
} else {
return AjaxJson.returnExceptionInfo("请求类型不支持");
}
Map<String, Object> map7 = new HashMap<>();
map7.put("tableName", tableName + "_temp");
map7.put("tableName2", tableName + "_temp_file");
map7.put("id", recId);
danganguanliService.wsajmlTempCount(map7);
AjaxJson json = AjaxJson.returnInfo("成功上传数successNum，失败上传数falseNum");
AjaxJson json = AjaxJson.returnInfo("成功上传数:"+successNum+"，失败上传数:"+falseNum);
json.put("successNum", successNum);
json.put("falseNum", falseNum);
return json;
@@ -354,18 +359,18 @@ public class AnjuanAndJuanneiController {
private AjaxJson uploadFilesByPathAnjuan(MultipartFile file,String fondscode, String dir, Integer recId,String tableName, HttpServletRequest request) {
AjaxJson json = null;
File files = null;
try {
String originalFilename = file.getOriginalFilename();
if (StringUtil.isEmpty(originalFilename)) {
return AjaxJson.returnExceptionInfo("文件名为空");
}
int index = originalFilename.lastIndexOf(".") + 1;
if (index <= 0 || index >= originalFilename.length()) {
return AjaxJson.returnExceptionInfo("文件格式不正确");
}
String fileType = originalFilename.substring(index);
String file_name_server=StringUtil.generaterUUID()+"."+fileType;
@@ -375,10 +380,10 @@ public class AnjuanAndJuanneiController {
map5.put("tableName",tableName + "_temp_file");
map5.put("conditionSql","rec_id= '"+recId+"' and file_status=1 ");
int pageNo =danganguanliService.selectObjectCount(map5)+1;
// 使用File.separator确保跨平台兼容性
files = new File(dir + File.separator + file_name_server);
// 检查目标文件是否已存在
if (files.exists()) {
logger.warn("目标文件已存在，将覆盖: {}", files.getAbsolutePath());
@@ -388,7 +393,7 @@ public class AnjuanAndJuanneiController {
return AjaxJson.returnExceptionInfo("无法删除已存在的文件");
}
}
// 验证目录是否存在且可写
File parentDir = files.getParentFile();
if (parentDir == null || !parentDir.exists()) {
@@ -398,10 +403,10 @@ public class AnjuanAndJuanneiController {
logger.error("目录无写权限: {}", parentDir.getAbsolutePath());
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 文件传输
file.transferTo(files);
// 验证文件是否成功写入
if (!files.exists() || files.length() == 0) {
logger.error("文件传输失败或文件为空: {}", files.getAbsolutePath());
@@ -441,7 +446,7 @@ public class AnjuanAndJuanneiController {
String newName_pdf=file_name_server.replace("."+fileType,".pdf");
String sourcePath = dir + File.separator + file_name_server;
String targetPath = dir + File.separator + newName_pdf;
boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath, targetPath);
if (!pdfCreated) {
logger.warn("PDF文件生成失败: {} -> {}", sourcePath, targetPath);
@@ -452,7 +457,7 @@ public class AnjuanAndJuanneiController {
FileTool.copyFile(targetPath, originalPath);
}
}
//mxf格式的文件需要转换一份mp4给前端展示
if (fileType.equalsIgnoreCase("mxf")) {
String replaceMp4;
@@ -463,15 +468,15 @@ public class AnjuanAndJuanneiController {
}
VideoConvertUtil.convert(files.getPath(), replaceMp4);
}
json = AjaxJson.returnInfo("上传文件成功");
json.put("file",files);
json.put("file_name_server",file_name_server);
} catch (Exception e) {
logger.error("上传文件失败: {}", file.getOriginalFilename(), e);
json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
// 清理失败的文件
if (files != null && files.exists()) {
try {
@@ -486,7 +491,7 @@ public class AnjuanAndJuanneiController {
}
return json;
}
/**
* NIO方式上传文件，避免OOM问题
*/
@@ -495,18 +500,18 @@ public class AnjuanAndJuanneiController {
Path targetPath = null;
InputStream inputStream = null;
FileChannel fileChannel = null;
try {
String originalFilename = file.getOriginalFilename();
if (StringUtil.isEmpty(originalFilename)) {
return AjaxJson.returnExceptionInfo("文件名为空");
}
int index = originalFilename.lastIndexOf(".") + 1;
if (index <= 0 || index >= originalFilename.length()) {
return AjaxJson.returnExceptionInfo("文件格式不正确");
}
String fileType = originalFilename.substring(index);
String file_name_server = StringUtil.generaterUUID() + "." + fileType;
@@ -516,16 +521,16 @@ public class AnjuanAndJuanneiController {
map5.put("tableName", tableName + "_temp_file");
map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 ");
int pageNo = danganguanliService.selectObjectCount(map5) + 1;
// 使用NIO Path和Files API
targetPath = Paths.get(dir, file_name_server);
// 检查目标文件是否已存在
if (Files.exists(targetPath)) {
logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath());
Files.deleteIfExists(targetPath);
}
// 验证父目录是否存在且可写
Path parentDir = targetPath.getParent();
if (parentDir == null || !Files.exists(parentDir)) {
@@ -535,25 +540,25 @@ public class AnjuanAndJuanneiController {
logger.error("目录无写权限: {}", parentDir.toAbsolutePath());
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 使用NIO方式写入文件，避免内存OOM
inputStream = file.getInputStream();
fileChannel = FileChannel.open(targetPath,
java.nio.file.StandardOpenOption.CREATE,
fileChannel = FileChannel.open(targetPath,
java.nio.file.StandardOpenOption.CREATE,
java.nio.file.StandardOpenOption.WRITE,
java.nio.file.StandardOpenOption.TRUNCATE_EXISTING);
ReadableByteChannel readableByteChannel = Channels.newChannel(inputStream);
// 使用transferFrom进行高效文件传输
long transferred = fileChannel.transferFrom(readableByteChannel, 0, Long.MAX_VALUE);
// 验证文件是否成功写入
if (!Files.exists(targetPath) || Files.size(targetPath) == 0) {
logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath());
return AjaxJson.returnExceptionInfo("文件传输失败");
}
logger.info("文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath());
String file_path = "uploadFile" + File.separator + tableName + "_temp_file" + File.separator + fondscode + File.separator + recId;
@@ -588,7 +593,7 @@ public class AnjuanAndJuanneiController {
String newName_pdf = file_name_server.replace("." + fileType, ".pdf");
Path sourcePath = targetPath;
Path pdfPath = Paths.get(dir, newName_pdf);
boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString());
if (!pdfCreated) {
logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath);
@@ -603,7 +608,7 @@ public class AnjuanAndJuanneiController {
}
}
}
//mxf格式的文件需要转换一份mp4给前端展示
if (fileType.equalsIgnoreCase("mxf")) {
String replaceMp4;
@@ -614,15 +619,15 @@ public class AnjuanAndJuanneiController {
}
VideoConvertUtil.convert(targetPath.toString(), replaceMp4);
}
json = AjaxJson.returnInfo("上传文件成功");
json.put("file", targetPath.toFile());
json.put("file_name_server", file_name_server);
} catch (Exception e) {
logger.error("上传文件失败: {}", file.getOriginalFilename(), e);
json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
// 清理失败的文件
if (targetPath != null && Files.exists(targetPath)) {
try {
@@ -651,7 +656,7 @@ public class AnjuanAndJuanneiController {
}
return json;
}
/**
* 零拷贝方式上传文件，最大程度优化内存使用，防止OOM
*/
@@ -660,18 +665,18 @@ public class AnjuanAndJuanneiController {
Path targetPath = null;
ReadableByteChannel readableByteChannel = null;
FileChannel fileChannel = null;
try {
String originalFilename = file.getOriginalFilename();
if (StringUtil.isEmpty(originalFilename)) {
return AjaxJson.returnExceptionInfo("文件名为空");
}
int index = originalFilename.lastIndexOf(".") + 1;
if (index <= 0 || index >= originalFilename.length()) {
return AjaxJson.returnExceptionInfo("文件格式不正确");
}
String fileType = originalFilename.substring(index);
String file_name_server = StringUtil.generaterUUID() + "." + fileType;
@@ -681,16 +686,16 @@ public class AnjuanAndJuanneiController {
map5.put("tableName", tableName + "_temp_file");
map5.put("conditionSql", "rec_id= '" + recId + "' and file_status=1 ");
int pageNo = danganguanliService.selectObjectCount(map5) + 1;
// 使用NIO Path和Files API
targetPath = Paths.get(dir, file_name_server);
// 检查目标文件是否已存在
if (Files.exists(targetPath)) {
logger.warn("目标文件已存在，将覆盖: {}", targetPath.toAbsolutePath());
Files.deleteIfExists(targetPath);
}
// 验证父目录是否存在且可写
Path parentDir = targetPath.getParent();
if (parentDir == null || !Files.exists(parentDir)) {
@@ -700,20 +705,20 @@ public class AnjuanAndJuanneiController {
logger.error("目录无写权限: {}", parentDir.toAbsolutePath());
return AjaxJson.returnExceptionInfo("目录无写权限");
}
// 零拷贝实现：直接使用FileChannel进行传输，避免数据在用户空间的拷贝
try (InputStream inputStream = file.getInputStream()) {
readableByteChannel = Channels.newChannel(inputStream);
fileChannel = FileChannel.open(targetPath,
StandardOpenOption.CREATE,
fileChannel = FileChannel.open(targetPath,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING);
// 使用transferFrom实现零拷贝，数据直接从内核缓冲区传输到文件
long transferred = 0;
long position = 0;
long count = Long.MAX_VALUE;
// 分块传输大文件，避免单次传输过大导致的问题
final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks
while (position < file.getSize()) {
@@ -725,16 +730,16 @@ public class AnjuanAndJuanneiController {
position += transferredChunk;
transferred += transferredChunk;
}
// 强制写入磁盘，确保数据持久化
fileChannel.force(true);
// 验证文件是否成功写入
if (!Files.exists(targetPath) || Files.size(targetPath) == 0) {
logger.error("文件传输失败或文件为空: {}", targetPath.toAbsolutePath());
return AjaxJson.returnExceptionInfo("文件传输失败");
}
logger.info("零拷贝文件传输完成: {} bytes, 目标: {}", transferred, targetPath.toAbsolutePath());
}
@@ -770,7 +775,7 @@ public class AnjuanAndJuanneiController {
String newName_pdf = file_name_server.replace("." + fileType, ".pdf");
Path sourcePath = targetPath;
Path pdfPath = Paths.get(dir, newName_pdf);
boolean pdfCreated = PdfFileHelper.image2Pdf(sourcePath.toString(), pdfPath.toString());
if (!pdfCreated) {
logger.warn("PDF文件生成失败: {} -> {}", sourcePath, pdfPath);
@@ -792,7 +797,7 @@ public class AnjuanAndJuanneiController {
}
}
}
//mxf格式的文件需要转换一份mp4给前端展示
if (fileType.equalsIgnoreCase("mxf")) {
String replaceMp4;
@@ -803,15 +808,15 @@ public class AnjuanAndJuanneiController {
}
VideoConvertUtil.convert(targetPath.toString(), replaceMp4);
}
json = AjaxJson.returnInfo("上传文件成功");
json.put("file", targetPath.toFile());
json.put("file_name_server", file_name_server);
} catch (Exception e) {
logger.error("零拷贝上传文件失败: {}", file.getOriginalFilename(), e);
json = AjaxJson.returnExceptionInfo("上传文件失败: " + e.getMessage());
// 清理失败的文件
if (targetPath != null && Files.exists(targetPath)) {
try {
@@ -840,21 +845,21 @@ public class AnjuanAndJuanneiController {
}
return json;
}
/**
* 零拷贝文件复制方法
*/
private void copyFileZeroCopy(Path source, Path target) throws IOException {
try (FileChannel sourceChannel = FileChannel.open(source, StandardOpenOption.READ);
FileChannel targetChannel = FileChannel.open(target,
StandardOpenOption.CREATE,
FileChannel targetChannel = FileChannel.open(target,
StandardOpenOption.CREATE,
StandardOpenOption.WRITE,
StandardOpenOption.TRUNCATE_EXISTING)) {
long size = sourceChannel.size();
long position = 0;
final long CHUNK_SIZE = 8 * 1024 * 1024; // 8MB chunks
while (position < size) {
long chunkSize = Math.min(CHUNK_SIZE, size - position);
long transferred = targetChannel.transferFrom(sourceChannel, position, chunkSize);
@@ -863,7 +868,7 @@ public class AnjuanAndJuanneiController {
}
position += transferred;
}
// 强制写入磁盘
targetChannel.force(true);
}