From 3c0568cf940c522648a1c4d7240700da707eea08 Mon Sep 17 00:00:00 2001 From: ygs Date: Fri, 11 Jul 2025 15:34:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=87=E4=BB=B6=E4=B8=8A=E4=BC=A0=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E5=8A=9F=E8=83=BD=20=E9=85=8D=E7=BD=AE=20=20#?= =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=20kafka=20=E9=85=8D=E7=BD=AE=20#?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=9D=97=E5=AD=98=E5=82=A8=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../attachment/entity/dao/OssObjectDao.java | 24 + .../attachment/entity/dto/OssFinderDTO.java | 12 + .../attachment/entity/po/AttachmentPO.java | 19 + .../entity/vo/FeedbackMessageVO.java | 32 ++ .../attachment/entity/vo/SnowFlakerVO.java | 12 + .../system/attachment/entity/vo/UploadVO.java | 27 + .../attachment/handler/OssStorageHandler.java | 12 + .../handler/impl/OssStorageHandlerImpl.java | 63 +++ .../files/controller/FileController.java | 157 +----- .../service/impl/SysStorageServiceImpl.java | 59 +- .../ebtp/system/tus/constant/TusConstant.java | 46 ++ .../system/tus/controller/OssController.java | 79 +++ .../tus/controller/TusUploadController.java | 379 +++++++++++++ .../ebtp/system/tus/entity/ApiResultVO.java | 23 + .../tus/entity/TusUploadFileEntity.java | 38 ++ .../tus/exception/ControllerErrorHandler.java | 37 ++ .../system/tus/exception/TusException.java | 41 ++ .../system/tus/service/CosClientService.java | 25 + .../system/tus/service/OssClientService.java | 40 ++ .../tus/service/TusUploadFileService.java | 30 + .../service/impl/CosClientServiceImpl.java | 282 ++++++++++ .../service/impl/OssClientServiceImpl.java | 524 ++++++++++++++++++ .../ebtp/system/tus/service/impl/S3Test.java | 86 +++ .../ebtp/system/tus/util/TusUtils.java | 134 +++++ .../background/service/CurdDataService.java | 15 + .../background/service/NoticeTaskService.java | 16 + .../service/impl/CurdDataServiceImpl.java | 42 ++ .../service/impl/NoticeTaskServiceImpl.java | 113 ++++ .../updownload/client/RespFeignClient.java | 19 + .../client/RespFeignFallbackFactory.java | 28 + .../system/updownload/common/Constant.java | 22 + .../updownload/config/CacheConfiguration.java | 31 ++ .../config/DigitalSecurityConfiguration.java | 22 + .../config/KafkaProducerConfigDev.java | 62 +++ .../config/KafkaProducerConfigPro.java | 63 +++ .../updownload/entity/dao/RpcDataDAO.java | 23 + .../updownload/entity/dao/UploadDataDAO.java | 41 ++ .../entity/dto/NotificationDTO.java | 48 ++ .../system/updownload/entity/dto/PartTag.java | 17 + .../updownload/entity/dto/PartUploadDTO.java | 40 ++ .../system/updownload/entity/dto/TaskDTO.java | 46 ++ .../system/updownload/entity/dto/Tfile.java | 23 + .../updownload/entity/dto/UploadParamDTO.java | 64 +++ .../updownload/entity/po/ResourcePO.java | 42 ++ .../updownload/entity/vo/DownloadVO.java | 14 + .../updownload/kafka/entity/MessageState.java | 15 + .../kafka/maneger/MessageManager.java | 26 + .../controller/HulkDownloadController.java | 191 +++++++ .../web/controller/HulkPreviewController.java | 129 +++++ .../web/controller/HulkUploadController.java | 89 +++ .../updownload/web/dto/MixDataItem.java | 16 + .../updownload/web/dto/MixDownloadParam.java | 14 + .../handler/checker/ChunkExistsChecker.java | 132 +++++ .../web/handler/remover/ChunkFileRemover.java | 44 ++ .../handler/storage/ChunkStorageHandler.java | 10 + .../storage/impl/ChunkStorageHandlerImpl.java | 174 ++++++ .../web/service/FileUploadService.java | 10 + .../service/impl/FileUploadServiceImpl.java | 35 ++ .../updownload/web/utils/RegexpUtil.java | 33 ++ src/main/resources/application-master.yml | 33 +- 60 files changed, 3709 insertions(+), 214 deletions(-) create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dao/OssObjectDao.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dto/OssFinderDTO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/entity/po/AttachmentPO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/FeedbackMessageVO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/SnowFlakerVO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/UploadVO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/handler/OssStorageHandler.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/attachment/handler/impl/OssStorageHandlerImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/constant/TusConstant.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/controller/OssController.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/controller/TusUploadController.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/entity/ApiResultVO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/entity/TusUploadFileEntity.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/exception/ControllerErrorHandler.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/exception/TusException.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/service/CosClientService.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/service/OssClientService.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/service/TusUploadFileService.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/CosClientServiceImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/OssClientServiceImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/S3Test.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/tus/util/TusUtils.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/CurdDataService.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/NoticeTaskService.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/CurdDataServiceImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/NoticeTaskServiceImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignClient.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignFallbackFactory.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/common/Constant.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/config/CacheConfiguration.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/config/DigitalSecurityConfiguration.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigDev.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigPro.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/RpcDataDAO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/UploadDataDAO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/NotificationDTO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartTag.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartUploadDTO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/TaskDTO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/Tfile.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/UploadParamDTO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/po/ResourcePO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/entity/vo/DownloadVO.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/entity/MessageState.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/maneger/MessageManager.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkDownloadController.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkPreviewController.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkUploadController.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDataItem.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDownloadParam.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/checker/ChunkExistsChecker.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/remover/ChunkFileRemover.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/ChunkStorageHandler.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/impl/ChunkStorageHandlerImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/FileUploadService.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/impl/FileUploadServiceImpl.java create mode 100644 src/main/java/com/coscoshipping/ebtp/system/updownload/web/utils/RegexpUtil.java diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dao/OssObjectDao.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dao/OssObjectDao.java new file mode 100644 index 0000000..21a7733 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dao/OssObjectDao.java @@ -0,0 +1,24 @@ +package com.coscoshipping.ebtp.system.attachment.entity.dao; + + +import com.baomidou.mybatisplus.annotation.TableId; +import com.baomidou.mybatisplus.annotation.TableName; +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@TableName(value = "biz_oss_object_tbl") +@Accessors(chain = true) +public class OssObjectDao { + + /* 对象ID */ + @TableId + private String id; + + /* 业务Id */ + private String bid; + + /* 文件名称 */ + private String filename; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dto/OssFinderDTO.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dto/OssFinderDTO.java new file mode 100644 index 0000000..59acfab --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/dto/OssFinderDTO.java @@ -0,0 +1,12 @@ +package com.coscoshipping.ebtp.system.attachment.entity.dto; + +import lombok.Data; + +import java.util.List; + +@Data +public class OssFinderDTO { + + private List bidList; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/po/AttachmentPO.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/po/AttachmentPO.java new file mode 100644 index 0000000..e09c751 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/po/AttachmentPO.java @@ -0,0 +1,19 @@ +package com.coscoshipping.ebtp.system.attachment.entity.po; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class AttachmentPO { + + /* 附件ID */ + private String id; + + /* 业务ID */ + private String bid; + + /* 附件名称 */ + private String filename; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/FeedbackMessageVO.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/FeedbackMessageVO.java new file mode 100644 index 0000000..047f62c --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/FeedbackMessageVO.java @@ -0,0 +1,32 @@ +package com.coscoshipping.ebtp.system.attachment.entity.vo; + +import lombok.Data; +import lombok.experimental.Accessors; + +import java.util.Optional; + +@Data +@Accessors(chain = true) +public class FeedbackMessageVO { + + private Boolean success; + private String oid; + private String message; + + public FeedbackMessageVO(String oid) { + this.success = Optional.ofNullable(oid).isPresent(); + + if (!this.success) { + this.message = "文档中心异常或无效的文件类型"; + } + this.oid = oid; + } + + public FeedbackMessageVO(Boolean success) { + if (!this.success) { + this.message = "文档中心异常或无效的文件类型"; + } + this.success = success; + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/SnowFlakerVO.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/SnowFlakerVO.java new file mode 100644 index 0000000..83483ce --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/SnowFlakerVO.java @@ -0,0 +1,12 @@ +package com.coscoshipping.ebtp.system.attachment.entity.vo; + +import lombok.Data; +import lombok.experimental.Accessors; + +@Data +@Accessors(chain = true) +public class SnowFlakerVO { + + private String id; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/UploadVO.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/UploadVO.java new file mode 100644 index 0000000..39847a9 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/entity/vo/UploadVO.java @@ -0,0 +1,27 @@ +package com.coscoshipping.ebtp.system.attachment.entity.vo; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import org.springframework.web.multipart.MultipartFile; + +import javax.validation.constraints.NotNull; + +@Data +public class UploadVO { + + @NotNull(message = "业务id不嫩为空") + private String businessId; + + /** + * 文件流 + */ + @JsonIgnore + @NotNull(message = "附件不能为空") + private MultipartFile file; + + /** + * bid下是否唯一 + */ + private boolean only; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/handler/OssStorageHandler.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/handler/OssStorageHandler.java new file mode 100644 index 0000000..059f83e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/handler/OssStorageHandler.java @@ -0,0 +1,12 @@ +package com.coscoshipping.ebtp.system.attachment.handler; + +import com.coscoshipping.ebtp.system.attachment.entity.dao.OssObjectDao; + +import java.util.Optional; + +@FunctionalInterface +public interface OssStorageHandler { + + Optional save(T t); + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/attachment/handler/impl/OssStorageHandlerImpl.java b/src/main/java/com/coscoshipping/ebtp/system/attachment/handler/impl/OssStorageHandlerImpl.java new file mode 100644 index 0000000..906b5db --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/attachment/handler/impl/OssStorageHandlerImpl.java @@ -0,0 +1,63 @@ +package com.coscoshipping.ebtp.system.attachment.handler.impl; + +import com.chinaunicom.ebtp.mall.cloud.attachment.sdk.api.AttachmentClient; +import com.chinaunicom.ebtp.mall.cloud.attachment.sdk.model.UploadObject; +import com.coscoshipping.ebtp.system.attachment.entity.dao.OssObjectDao; +import com.coscoshipping.ebtp.system.attachment.entity.vo.UploadVO; +import com.coscoshipping.ebtp.system.attachment.handler.OssStorageHandler; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; +import org.springframework.web.multipart.MultipartFile; + +import java.util.Optional; + +/** + * 文件对象存储到OSS + * + * @author Ajaxfan + */ +@Slf4j +@Component +public class OssStorageHandlerImpl implements OssStorageHandler { + + /* 附件工具 */ + private @Autowired + AttachmentClient attachmentClient; + + @Override + public Optional save(UploadVO vo) { + return Optional.of(vo).map(this::pushObjectToOss); + } + + /** + * 对象推入到 OSS + * + * @param vo + * @return + */ + private OssObjectDao pushObjectToOss(UploadVO vo) { + try { + MultipartFile multipartFile = vo.getFile(); + Assert.notNull(multipartFile, "无效的附件"); + + String businessId = vo.getBusinessId(); + Assert.notNull(businessId, "业务id不能为空"); + + String filename = multipartFile.getOriginalFilename(); + + // 资源上传， 需要提供业务ID和待上传文件对象 + Optional op = attachmentClient.upload(businessId, filename, multipartFile.getBytes(), vo.isOnly()); + + // 检验结果的有效性 + if (op.isPresent()) { + return new OssObjectDao().setId(op.get().getId()).setFilename(filename).setBid(businessId); + } + } catch (Exception e) { + log.error(e.getMessage()); + } + return null; + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/files/controller/FileController.java b/src/main/java/com/coscoshipping/ebtp/system/files/controller/FileController.java index 7312556..f5b0595 100644 --- a/src/main/java/com/coscoshipping/ebtp/system/files/controller/FileController.java +++ b/src/main/java/com/coscoshipping/ebtp/system/files/controller/FileController.java @@ -2,7 +2,6 @@ package com.coscoshipping.ebtp.system.files.controller; import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse; -//import com.coscoshipping.ebtp.system.cloud.auth.MallUser; import com.coscoshipping.ebtp.system.common.base.BaseController; import com.coscoshipping.ebtp.system.common.framework.enums.ResponseEnum; import com.coscoshipping.ebtp.system.common.framework.exception.FileException; @@ -14,9 +13,6 @@ import com.coscoshipping.ebtp.system.files.service.impl.SysStorageServiceImpl; import com.coscoshipping.ebtp.system.files.vo.FilePageVO; import com.coscoshipping.ebtp.system.files.vo.FileQuery; import com.coscoshipping.ebtp.system.files.vo.SysStorageStatVO; -//import com.coscoshipping.ebtp.system.framework.utils.UnifastContext; -//import com.coscoshipping.ebtp.system.openPlatformNew.business.watermark.model.vo.WaterMarkFileVo; -//import com.coscoshipping.ebtp.system.openPlatformNew.business.watermark.service.WatermarkService; import io.swagger.annotations.Api; import io.swagger.annotations.ApiImplicitParam; import io.swagger.annotations.ApiImplicitParams; @@ -56,13 +52,6 @@ public class FileController extends BaseController { private final SysStorageService fileService; -// @Resource -// private WatermarkService watermarkService; -// -// -// @Autowired -// private UnifastContext unifastContext; - public FileController(SysStorageServiceImpl fileService) { this.fileService = fileService; } @@ -140,7 +129,7 @@ public class FileController extends BaseController { */ @ApiOperation(value = "单文件逻辑删除，不删除磁盘文件") @PostMapping("/logicDelete") - public BaseResponse deleteFile(@RequestParam("fileId") String fileId) throws Exception { + public BaseResponse deleteFile(@RequestParam("fileId") String fileId) throws Exception { if (null == fileId) { return new BaseResponse<>(400, false, ResponseEnum.FILEID_NULL.getMsg(), null); } @@ -165,129 +154,6 @@ public class FileController extends BaseController { return ok(this.fileService.deleteFileOnDisk(fileId)); } - /** - * 下载文件的接口。根据请求参数决定是否添加水印。 - * - * @param fileId 文件ID，用于定位要下载的文件。 - * @param content 可选参数，用于指定添加水印的内容。 - * @param waterMarkFlag 可选参数，指示是否添加水印。 true的表示不用水印 - * @return 包含文件信息的SysStorageVO对象，用于下载文件。 - * @throws FileException 如果文件操作发生错误。 - * @throws IOException 如果读取文件发生错误。 - */ -// @ApiOperation(value = "单文件下载") -// @GetMapping(value = "/download") -// public SysStorageVO downloadFile(@RequestParam("fileId") String fileId, -// @RequestParam(value = "content", required = false) String content, -// @RequestParam(value = "waterMarkFlag", required = false) Boolean waterMarkFlag) -// throws Exception { -// // 记录进入下载接口的日志 -// log.info("进入单文件下载接口,{},{},{}", fileId, content, waterMarkFlag); -// -// // 判断是否需要添加水印 -// if (waterMarkFlag != null && waterMarkFlag) { -// // 获取当前用户信息，用于添加水印 -// // @Modify 20240717 增加水印功能 -// MallUser user = unifastContext.getUser(); -// // 检查用户信息是否有效 -// if (ObjectUtil.isNull(user)) { -// throw new NullPointerException("用户身份信息无效，请重新登录系统"); -// } -// try { -// // 根据文件ID、内容和用户信息生成带水印的文件信息 -// WaterMarkFileVo waterMarkFileVo = watermarkService.getWaterMarkFileVo(fileId, content, user.getStaffId()); -// // 根据文件ID获取文件基本信息 -// SysStoragePO filePO = fileService.getById(fileId); -// // 创建带水印文件信息的对象 -// SysStorageVO fileVO = new SysStorageVO(); -// // 复制文件基本信息到带水印文件信息的对象 -// BeanUtils.copyProperties(filePO, fileVO); -// // 设置带水印的文件流 -// fileVO.setFileStream(waterMarkFileVo.getFileStream()); -// // 返回带水印的文件信息 -// return fileVO; -// } catch (Exception e) { -// // 如果添加水印失败，记录日志并返回原文件 -// log.info("下载带水印文件报错，重新下载原文件"); -// SysStorageVO fileVO = this.fileService.downloadFile(fileId); -// return fileVO; -// } -// } else { -// // 如果不需要添加水印，直接返回原文件信息 -// SysStorageVO fileVO = this.fileService.downloadFile(fileId); -// return fileVO; -// } -// } - - -// @ApiOperation(value = "单文件下载") -// @PostMapping(value = "/download4POST") -// public SysStorageVO download4POST(@RequestParam("fileId") String fileId, -// @RequestParam(value = "content", required = false) String content, -// @RequestParam(value = "waterMarkFlag", required = false) Boolean waterMarkFlag) -// throws Exception { -// // 记录进入下载接口的日志 -// log.info("进入单文件下载接口,{},{},{}", fileId, content, waterMarkFlag); -// -// // 判断是否需要添加水印 -// if (waterMarkFlag != null && waterMarkFlag) { -// // 获取当前用户信息，用于添加水印 -// // @Modify 20240717 增加水印功能 -// MallUser user = unifastContext.getUser(); -// // 检查用户信息是否有效 -// if (ObjectUtil.isNull(user)) { -// throw new NullPointerException("用户身份信息无效，请重新登录系统"); -// } -// try { -// // 根据文件ID、内容和用户信息生成带水印的文件信息 -// WaterMarkFileVo waterMarkFileVo = watermarkService.getWaterMarkFileVo(fileId, content, user.getStaffId()); -// // 根据文件ID获取文件基本信息 -// SysStoragePO filePO = fileService.getById(fileId); -// // 创建带水印文件信息的对象 -// SysStorageVO fileVO = new SysStorageVO(); -// // 复制文件基本信息到带水印文件信息的对象 -// BeanUtils.copyProperties(filePO, fileVO); -// // 设置带水印的文件流 -// fileVO.setFileStream(waterMarkFileVo.getFileStream()); -// // 返回带水印的文件信息 -// return fileVO; -// } catch (Exception e) { -// // 如果添加水印失败，记录日志并返回原文件 -// log.info("下载带水印文件报错，重新下载原文件"); -// SysStorageVO fileVO = this.fileService.downloadFile(fileId); -// return fileVO; -// } -// } else { -// // 如果不需要添加水印，直接返回原文件信息 -// SysStorageVO fileVO = this.fileService.downloadFile(fileId); -// return fileVO; -// } -// } - - -// @ApiOperation(value = "文件下载") -// @PostMapping("/testDownload") -// public void testDownload(@RequestParam("fileId") String fileId, @RequestParam(value = "content", required = false) String content, -// @RequestParam(value = "waterMarkFlag", required = false) Boolean waterMarkFlag) throws Exception { -// -// try { -// SysStorageVO sysStorageVO = this.downloadFile(fileId, content, waterMarkFlag); -// resp.reset(); -// String originalName = sysStorageVO.getOriginalName(); -// String mimeType = request.getServletContext().getMimeType(originalName); -// resp.setContentType(mimeType); -// resp.setHeader("Content-Disposition", "attachment;filename=" -// + URLEncoder.encode(originalName, "utf-8")); -// OutputStream os = resp.getOutputStream(); -// os.write(sysStorageVO.getFileStream()); -// os.flush(); -// os.close(); -// } catch (Exception e) { -// e.printStackTrace(); -// throw new NullPointerException(UniStorageProperties.FAST_DFS_ERROR); -// } -// } - /** * 功能描述:下载原图加规格图 @@ -308,13 +174,12 @@ public class FileController extends BaseController { /** * 根据文件Id，返回文件服务器上文件的下载地址。可接收多个文件Id 仅支持FastDfs和Ceph存储。" - * ceph地址有效期1h。 * * @param fileId 文件Id * @return 文件下载URL */ - @ApiOperation(value = "返回文件下载地址（仅支持FastDfs和Ceph存储。文件下载|在线预览 建议使用已提供的接口）", - notes = "根据fileId，返回文件服务器上文件的下载地址，仅支持FastDfs和Ceph存储。" + + @ApiOperation(value = "返回文件下载地址（仅支持Ceph存储。文件下载|在线预览 建议使用已提供的接口）", + notes = "根据fileId，返回文件服务器上文件的下载地址，仅支持Ceph存储。" + "可接收多个fileId，多fileId之间请以 , 隔开") @PostMapping("/url") public BaseResponse> downloadURL(@RequestBody String... fileId) throws Exception { @@ -464,7 +329,7 @@ public class FileController extends BaseController { if (copy != null) { return ok(copy); } - return new BaseResponse<>(400,false,"复制失败",null); + return new BaseResponse<>(400, false, "复制失败", null); } /** @@ -512,22 +377,10 @@ public class FileController extends BaseController { public BaseResponse saveCutover(@RequestBody List sysStoragePOList) { Boolean aBoolean = fileService.saveCutover(sysStoragePOList); if (aBoolean == null || aBoolean == false) { - return new BaseResponse<>(400,false,"文档割接失败",aBoolean); + return new BaseResponse<>(400, false, "文档割接失败", aBoolean); } return ok(aBoolean); } - /** - * 描述: 下载大附件 - * - * @author: MengHaoHao - * @Date 创建时间: 2021/6/24 - */ -// @GetMapping("/getDownload") -// public BaseResponse getDownload(@RequestParam("fileId") String fileId, HttpServletResponse httpServletResponse, @RequestParam("documentSecretKey") String documentSecretKey) { -// return fileService.getDownload(fileId, httpServletResponse, documentSecretKey); -// } - - } diff --git a/src/main/java/com/coscoshipping/ebtp/system/files/service/impl/SysStorageServiceImpl.java b/src/main/java/com/coscoshipping/ebtp/system/files/service/impl/SysStorageServiceImpl.java index 4c436b3..5580f1d 100644 --- a/src/main/java/com/coscoshipping/ebtp/system/files/service/impl/SysStorageServiceImpl.java +++ b/src/main/java/com/coscoshipping/ebtp/system/files/service/impl/SysStorageServiceImpl.java @@ -58,8 +58,6 @@ public class SysStorageServiceImpl extends ServiceImpl annexSysStorageVOList = new ArrayList<>(); List sysStorageAnnexPOS = sysStorageAnnexService.getBaseMapper().downloadFileAll(fileId); - AnnexSysStorageVO annexSysStorageVO = new AnnexSysStorageVO(); for (SysStorageAnnexPO sysStorageAnnexPO : sysStorageAnnexPOS) { byte[] fileAnnexStream = new byte[0]; try { @@ -415,6 +412,7 @@ public class SysStorageServiceImpl extends ServiceImpl getDownload(String fileId, HttpServletResponse httpServletResponse, String documentSecretKey) { -// -// Object redisDocumentSecretKey = redisUtils.get("documentSecretKey"); -// if (redisDocumentSecretKey == null || !documentSecretKey.equals(redisDocumentSecretKey.toString())) { -// return new BaseResponse<>(400, false, "密钥不正确", null); -// } -// redisUtils.deleteByIndistinctKey("documentSecretKey"); -// -// SysStoragePO sysStoragePO = this.baseMapper.selectFileById(fileId); -// if (sysStoragePO == null) { -// return new BaseResponse<>(400, false, "fileId不存在", null); -// } -// try { -// httpServletResponse.setHeader("Content-Disposition", "attachment;filename=" -// + URLEncoder.encode(sysStoragePO.getOriginalName(), "utf-8")); //originalName:这个下载附件名字 比如：abc.jpg -// -// } catch (Exception e) { -// e.printStackTrace(); -// } -// -// if ("0".equals(sysStoragePO.getCutoverStatus()) && sysStoragePO.getLogoFilePath() != null) { -// try { -// URL url = new URL(sysStoragePO.getLogoFilePath()); -// InputStream inputStream = url.openStream(); -// ServletOutputStream outputStream = httpServletResponse.getOutputStream(); -// byte[] buffer = new byte[1024]; -// int length; -// while ((length = inputStream.read(buffer)) > 0) { -// outputStream.write(buffer, 0, length); -// } -// } catch (Exception e) { -// e.printStackTrace(); -// return new BaseResponse<>(400, false, "下载割接附件失败", null); -// } -// } -// try { -// InputStream inputStream = uniStorageService.downloadFileAsStream(sysStoragePO.getFilePath()); -// ServletOutputStream outputStream = httpServletResponse.getOutputStream(); -// byte[] buffer = new byte[1024]; -// int length; -// while ((length = inputStream.read(buffer)) > 0) { -// outputStream.write(buffer, 0, length); -// outputStream.flush(); -// } -// outputStream.close(); -// } catch (Exception e) { -// e.printStackTrace(); -// return new BaseResponse<>(400, false, "下载3.0附件失败", null); -// } -// return new BaseResponse<>(400, true, "下载附件成功", null); -// -// } - - //链接url下载图片 private static byte[] downloadPicture(String urlList) { URL url = null; diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/constant/TusConstant.java b/src/main/java/com/coscoshipping/ebtp/system/tus/constant/TusConstant.java new file mode 100644 index 0000000..ca80f13 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/constant/TusConstant.java @@ -0,0 +1,46 @@ +package com.coscoshipping.ebtp.system.tus.constant; + +/** + * TUS HEADER所需环境变量 + * + * @author Neo + */ +public class TusConstant { + + public static final String TUS_RESUMABLE_HEADER = "Tus-Resumable"; + public static final String TUS_RESUMABLE_VALUE = "1.0.0"; + + public static final String TUS_VERSION_HEADER = "Tus-Version"; + public static final String TUS_VERSION_VALUE = "1.0.0,0.2.2,0.2.1"; + + public static final String TUS_EXTENTION_HEADER = "Tus-Extension"; + public static final String TUS_EXTENTION_VALUE = "creation,expiration"; + + public static final String TUS_MAX_SIZE_HEADER = "Tus-Max-Size"; + + public static final String UPLOAD_OFFSET_HEADER = "Upload-Offset"; + + public static final String UPLOAD_LENGTH_HEADER = "Upload-Length"; + + public static final String LOCATION_HEADER = "Location"; + + public static final String ACCESS_STREAM_TYPE = "application/offset+octet-stream"; + + public static final String ACCESS_CONTROL_ALLOW_ORIGIN_HEADER = "Access-Control-Allow-Origin"; + public static final String ACCESS_CONTROL_ALLOW_ORIGIN_VALUE = "*"; + + public static final String ACCESS_CONTROL_ALLOW_METHIDS_HEADER = "Access-Control-Allow-Methods"; + public static final String ACCESS_CONTROL_ALLOW_METHIDS_VALUE = "GET,PUT,POST,DELETE"; + + public static final String ACCESS_CONTROL_EXPOSE_HEADER = "Access-Control-Expose-Headers"; + public static final String ACCESS_CONTROL_EXPOSE_OPTIONS_VALUE = "Tus-Resumable, Tus-Version, Tus-Max-Size, Tus-Extension"; + public static final String ACCESS_CONTROL_EXPOSE_POST_VALUE = "Location, Tus-Resumable"; + public static final String ACCESS_CONTROL_EXPOSE_HEAD_VALUE = "Upload-Offset, Upload-Length, Tus-Resumable"; + public static final String ACCESS_CONTROL_EXPOSE_PATCH_VALUE = "Upload-Offset, Tus-Resumable"; + + public static final String URL_PREFIX = "tus/"; + + public static final String OSS_SPILT = ":"; + public static final String OSS_OBJECT_PREFIX = "oss"; + public static final String OSS_TUS_PREFIX = "tus"; +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/controller/OssController.java b/src/main/java/com/coscoshipping/ebtp/system/tus/controller/OssController.java new file mode 100644 index 0000000..099578a --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/controller/OssController.java @@ -0,0 +1,79 @@ +package com.coscoshipping.ebtp.system.tus.controller; + +import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse; +import com.coscoshipping.ebtp.system.tus.constant.TusConstant; +import com.coscoshipping.ebtp.system.tus.service.OssClientService; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.*; + +import javax.annotation.Resource; +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; + +@Slf4j +@RestController +@RequestMapping("/v1/oss") +public class OssController { + + @Resource + private OssClientService ossClientService; + + @ApiOperation("附件上传") + @PostMapping("/attachment/uploadFile") + public BaseResponse attachmentUpload(@ApiParam(value = "字节", required = true) @RequestParam byte[] bytes, + @ApiParam(value = "业务id", required = true) @RequestParam String id, + @ApiParam(value = "文件类型", required = true) @RequestParam String fileType) { + return BaseResponse.success(ossClientService.attachmentUpload(bytes, id, fileType)); + } + + @ApiOperation("附件下载") + @GetMapping("/attachment/downloadFile") + public BaseResponse attachmentDownload(@ApiParam(value = "文件名", required = true) @RequestParam String fileName, HttpServletResponse httpServletResponse) { + return BaseResponse.success(ossClientService.attachmentDownload(fileName, httpServletResponse)); + } + + @ApiOperation("文件下载") + @GetMapping("/file/downloadFile") + public void fileDownload(@ApiParam(value = "文件名", required = true) @RequestParam String fileName, HttpServletResponse httpServletResponse) { + InputStream inputStream = null; + ServletOutputStream out = null; + try { + String[] split = fileName.split(TusConstant.OSS_SPILT); + String s = split[1].replaceAll("/", "_"); + httpServletResponse.setHeader("content-Type","application/octet-stream"); + httpServletResponse.setContentType("application/octet-stream"); + httpServletResponse.setHeader("Content-Disposition","attachment;filename=" + URLEncoder.encode(s,"UTF-8")); + inputStream = ossClientService.downloadStream(split[0], split[1]); + // 配置文件下载 + out = httpServletResponse.getOutputStream(); + byte[] b = new byte[1024]; + int len; + while ((len = inputStream.read(b)) > 0){ + out.write(b,0,len); + } + + }catch (IOException e){ + e.printStackTrace(); + }finally{ + try{ + if(inputStream!=null){ + inputStream.close(); + } + if(out!=null){ + out.close(); + out.flush(); + } + }catch (IOException e) { + e.printStackTrace(); + } + + } + } + + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/controller/TusUploadController.java b/src/main/java/com/coscoshipping/ebtp/system/tus/controller/TusUploadController.java new file mode 100644 index 0000000..f2ac429 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/controller/TusUploadController.java @@ -0,0 +1,379 @@ +//package com.coscoshipping.ebtp.system.tus.controller; +// +// +//import cn.hutool.core.util.RandomUtil; +//import com.chinaunicom.mall.ebtp.common.util.PropertyUtils; +//import com.chinaunicom.mall.ebtp.tus.constant.TusConstant; +//import com.chinaunicom.mall.ebtp.tus.entity.TusUploadFileEntity; +//import com.chinaunicom.mall.ebtp.tus.exception.TusException; +//import com.chinaunicom.mall.ebtp.tus.service.CosClientService; +//import com.chinaunicom.mall.ebtp.tus.service.OssClientService; +//import com.chinaunicom.mall.ebtp.tus.service.TusUploadFileService; +//import com.chinaunicom.mall.ebtp.updownload.background.service.CurdDataService; +//import com.chinaunicom.mall.ebtp.updownload.common.Constant; +//import com.chinaunicom.mall.ebtp.updownload.entity.dao.UploadDataDAO; +//import com.chinaunicom.mall.ebtp.updownload.kafka.maneger.MessageManager; +//import com.fasterxml.jackson.core.JsonProcessingException; +//import io.swagger.annotations.ApiOperation; +//import io.swagger.v3.oas.annotations.Operation; +//import org.slf4j.Logger; +//import org.slf4j.LoggerFactory; +//import org.springframework.beans.factory.annotation.Autowired; +//import org.springframework.beans.factory.annotation.Qualifier; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.core.io.Resource; +//import org.springframework.http.HttpHeaders; +//import org.springframework.http.RequestEntity; +//import org.springframework.http.ResponseEntity; +//import org.springframework.web.bind.annotation.*; +//import org.springframework.web.util.UriComponentsBuilder; +// +//import javax.servlet.http.HttpServletResponse; +//import java.io.IOException; +//import java.io.InputStream; +//import java.text.DateFormat; +//import java.text.SimpleDateFormat; +//import java.util.*; +// +// +///** +// *

+// * ＜p＞＜br＞ +// * +// * @author liuh +// * Tus所需的Contriller +// * 常量中的 URL_PREFIX 必须和当前 Controller 的 RequestMapping 映射一致 +// */ +//@CrossOrigin(origins = "*") +//@RestController +//@RequestMapping("/tus") +//public class TusUploadController { +// private Logger logger = LoggerFactory.getLogger(this.getClass()); +// +// @Value("${tus.max-size}") +// private Long tusMaxSize; +// @javax.annotation.Resource +// @Qualifier("kafkaManager") +// MessageManager kafkaManager; +// @Value("${oss.bucketName}") +// private String bucketName; +// +// @Autowired +// private TusUploadFileService tusUploadFileService; +// @Autowired +// private OssClientService ossClientService; +// @Autowired +// private CosClientService cosClientService; +// @Autowired +// private CurdDataService curdDataService; +// /** +// * 获取Tus支持的信息 +// * +// * @param response +// * @return +// */ +// @Operation(summary = "获取Tus支持的信息") +// @RequestMapping(method = RequestMethod.OPTIONS) +// public RequestEntity optionRequest(HttpServletResponse response) { +// logger.info("1 - OPTIONS:"); +// logger.info("1 - OPTIONS END"); +// +// response.setHeader(TusConstant.ACCESS_CONTROL_ALLOW_ORIGIN_HEADER, TusConstant.ACCESS_CONTROL_ALLOW_ORIGIN_VALUE); +// response.setHeader(TusConstant.ACCESS_CONTROL_EXPOSE_HEADER, TusConstant.ACCESS_CONTROL_EXPOSE_OPTIONS_VALUE); +// response.setHeader(TusConstant.TUS_RESUMABLE_HEADER, TusConstant.TUS_RESUMABLE_VALUE); +// response.setHeader(TusConstant.TUS_VERSION_HEADER, TusConstant.TUS_VERSION_VALUE); +// response.setHeader(TusConstant.TUS_MAX_SIZE_HEADER, tusMaxSize.toString()); +// response.setHeader(TusConstant.TUS_EXTENTION_HEADER, TusConstant.TUS_EXTENTION_VALUE); +// response.setHeader(TusConstant.ACCESS_CONTROL_ALLOW_METHIDS_HEADER, TusConstant.ACCESS_CONTROL_ALLOW_METHIDS_VALUE); +// return null; +// } +// +// /** +// * 获取Tus支持的信息 +// * +// * @param response +// * @return +// */ +// @Operation(summary = "获取Tus支持的信息 guid") +// @RequestMapping(method = RequestMethod.OPTIONS, value = "/{guid}") +// public RequestEntity optionRequest(@PathVariable String guid, HttpServletResponse response) throws JsonProcessingException { +// logger.info("2 - OPTIONS: {}", guid); +// TusUploadFileEntity tusUploadFileEntity = this.tusUploadFileService.getFileInfoByOssObjectName(guid); +// if (tusUploadFileEntity == null) { +// throw TusException.FILE_NOT_FOUND; +// } +// logger.info("file offset: {}", tusUploadFileEntity.getOffset()); +// logger.info("2 - OPTIONS END"); +// +// response.setHeader(TusConstant.ACCESS_CONTROL_ALLOW_ORIGIN_HEADER, TusConstant.ACCESS_CONTROL_ALLOW_ORIGIN_VALUE); +// response.setHeader(TusConstant.ACCESS_CONTROL_EXPOSE_HEADER, TusConstant.ACCESS_CONTROL_EXPOSE_OPTIONS_VALUE); +// response.setHeader(TusConstant.TUS_RESUMABLE_HEADER, TusConstant.TUS_RESUMABLE_VALUE); +// response.setHeader(TusConstant.TUS_VERSION_HEADER, TusConstant.TUS_VERSION_VALUE); +// response.setHeader(TusConstant.TUS_MAX_SIZE_HEADER, tusMaxSize.toString()); +// response.setHeader(TusConstant.TUS_EXTENTION_HEADER, TusConstant.TUS_EXTENTION_VALUE); +// response.setHeader(TusConstant.ACCESS_CONTROL_ALLOW_METHIDS_HEADER, TusConstant.ACCESS_CONTROL_ALLOW_METHIDS_VALUE); +// response.setHeader(TusConstant.UPLOAD_OFFSET_HEADER, Long.toString(tusUploadFileEntity.getOffset())); +// response.setStatus(204); +// return null; +// } +// +// /** +// * 创建新的文件 +// * +// * @param uploadLength +// * @param metadata +// * @param tfile +// * @param uriComponentsBuilder +// * @param response +// * @return +// */ +// @Operation(summary = "创建新的文件") +// @RequestMapping(method = RequestMethod.POST) +// public RequestEntity postRequest(@RequestHeader("Upload-Length") Long uploadLength, +// @RequestHeader("Upload-Metadata") String metadata, +// @RequestHeader("Upload-Tfile") String tfile, +// UriComponentsBuilder uriComponentsBuilder, +// HttpServletResponse response) throws IOException { +// logger.info("3 - POST START"); +// logger.info("Final-Length header value: {}", uploadLength); +// if (uploadLength < 1) { +// throw TusException.FILE_SIZE_ERROR; +// } +// +// if (uploadLength > tusMaxSize) { +// throw TusException.FILE_BEYOUND_SIZE; +// } +// DateFormat yearMonthPath = new SimpleDateFormat("yyyy/MM/dd"); +// String filePathPrefix = TusConstant.OSS_OBJECT_PREFIX + Constant.SEPARATOR + +// TusConstant.OSS_TUS_PREFIX + Constant.SEPARATOR + yearMonthPath.format(new Date((System.currentTimeMillis()))); +// +// byte[] nameByte = Base64.getDecoder().decode(metadata.split(",")[0].split(" ")[1]); +// String fileName = new String(nameByte); +// //截取文件后缀 +// String fileType = fileName.substring(fileName.lastIndexOf(".") + 1); +// //创建对象名 +// String objectName = filePathPrefix + Constant.SEPARATOR + RandomUtil.randomString(32).toUpperCase(Locale.ROOT) + "." + fileType; +// +// TusUploadFileEntity tusUploadFileEntity = new TusUploadFileEntity(); +// tusUploadFileEntity.setFileName(fileName); +// tusUploadFileEntity.setUploadSize(uploadLength); +// tusUploadFileEntity.setOffset(0L); +// tusUploadFileEntity.setCompleted(false); +// //取当前桶名称 +// tusUploadFileEntity.setOssObjectName(bucketName + TusConstant.OSS_SPILT + objectName); +// tusUploadFileEntity.setTfile(tfile); +// +// tusUploadFileEntity = this.tusUploadFileService.createFile(tusUploadFileEntity); +// this.tusUploadFileService.saveTusUploadFile(tusUploadFileEntity); +// String location = uriComponentsBuilder.path("/" + TusConstant.URL_PREFIX + tusUploadFileEntity.getGuid()).build().toString(); +// +// response.setHeader(TusConstant.ACCESS_CONTROL_EXPOSE_HEADER, TusConstant.ACCESS_CONTROL_EXPOSE_POST_VALUE); +// response.setHeader(TusConstant.LOCATION_HEADER, location); +// response.setHeader(TusConstant.TUS_RESUMABLE_HEADER, TusConstant.TUS_RESUMABLE_VALUE); +// response.setStatus(201); +// +// logger.info(location); +// logger.info("3 - POST END"); +// return null; +// } +// +// /** +// * 查询文件信息 +// * +// * @param guid +// * @param response +// * @return +// */ +// @Operation(summary = "查询文件信息") +// @RequestMapping(method = RequestMethod.HEAD, value = "/{guid}") +// public ResponseEntity headRequest(@PathVariable String guid, HttpServletResponse response) throws JsonProcessingException { +// logger.info("4 - HEAD START"); +// logger.info("objectName value: {}" + guid); +// +// TusUploadFileEntity tusUploadFileEntity = this.tusUploadFileService.getFileInfoByOssObjectName(guid); +// if (tusUploadFileEntity == null) { +// throw TusException.FILE_NOT_FOUND; +// } +// +// logger.info("file offset: {}", tusUploadFileEntity.getOffset()); +// logger.info("4 - HEAD END"); +// +// response.setHeader(TusConstant.ACCESS_CONTROL_EXPOSE_HEADER, TusConstant.ACCESS_CONTROL_EXPOSE_HEAD_VALUE); +// response.setHeader(TusConstant.UPLOAD_OFFSET_HEADER, Long.toString(tusUploadFileEntity.getOffset())); +// response.setHeader(TusConstant.UPLOAD_LENGTH_HEADER, Long.toString(tusUploadFileEntity.getUploadSize())); +// response.setHeader(TusConstant.TUS_RESUMABLE_HEADER, TusConstant.TUS_RESUMABLE_VALUE); +// response.setStatus(200); +// return null; +// } +// +// /** +// * 上传文件 +// * +// * @param uploadOffset +// * @param contentLength +// * @param contentType +// * @param guid +// * @param inputStream +// * @param response +// * @return +// * @throws Exception +// */ +// @Operation(summary = "上传文件") +// @RequestMapping(method = RequestMethod.PATCH, value = "/{guid}") +// public RequestEntity patchRequest(@RequestHeader("Upload-Offset") Long uploadOffset, +// @RequestHeader("Content-Length") Long contentLength, +// @RequestHeader("Content-Type") String contentType, +// @PathVariable String guid, +// InputStream inputStream, +// HttpServletResponse response) throws Exception { +// logger.info("5 - PATCH START"); +// logger.info("uuid value: {}", guid); +// logger.info("Upload-Offset: {}", uploadOffset); +// logger.info("Content-Length: {}", contentLength); +// logger.info("Content-Type: {}", contentType); +// +// if (uploadOffset == null || uploadOffset < 0) { +// throw TusException.HEADER_OFFSET_NEED; +// } +// +// if (contentLength == null || contentLength < 0) { +// throw TusException.HEADER_CONTENT_LENGTH_NEED; +// } +// +// if (!contentType.equals(TusConstant.ACCESS_STREAM_TYPE)) { +// throw TusException.HEADER_CONTENT_TYPE_NEED; +// } +// +// TusUploadFileEntity tusUploadFileEntity = this.tusUploadFileService.getFileInfoByOssObjectName(guid); +// if (tusUploadFileEntity == null) { +// throw TusException.FILE_NOT_FOUND; +// } +// +// logger.info("Offset: {}", tusUploadFileEntity.getOffset()); +// logger.info("FinalLength: {}", tusUploadFileEntity.getUploadSize()); +// +// if (!Objects.equals(uploadOffset, tusUploadFileEntity.getOffset())) { +// throw TusException.FILE_UPLOAD_OFFSET_ERROR; +// } +// +// if (tusUploadFileEntity.getUploadSize() < tusUploadFileEntity.getOffset()) { +// throw TusException.FILE_SIZE_ERROR; +// } +// +// if (Objects.equals(tusUploadFileEntity.getUploadSize(), tusUploadFileEntity.getOffset())) { +// logger.info("Upload-length == Offset"); +// logger.info("5 - PATCH END"); +// if (!tusUploadFileEntity.isCompleted()) { +// tusUploadFileEntity.setCompleted(true); +// this.tusUploadFileService.updateTusUploadFile(tusUploadFileEntity); +// } +// response.setStatus(200); +// return null; +// } +// tusUploadFileEntity = this.tusUploadFileService.processStream(tusUploadFileEntity, inputStream); +// if (tusUploadFileEntity.isCompleted()) { +// UploadDataDAO task = new UploadDataDAO(); +// task.setFilename(tusUploadFileEntity.getFileName()); +// task.setFileRealName(tusUploadFileEntity.getFileName()); +// task.setTotalSize(Math.toIntExact(tusUploadFileEntity.getUploadSize())); +// task.setPath(tusUploadFileEntity.getBucketName() + TusConstant.OSS_SPILT +tusUploadFileEntity.getOssObjectName()); +// task.setRelativePath(tusUploadFileEntity.getOssObjectName()); +// task.setTfile(tusUploadFileEntity.getTfile()); +// task.setLastUpdate(Calendar.getInstance().getTimeInMillis()); +// task.setTuid(PropertyUtils.getSnowflakeId()); +// task.setUuid(PropertyUtils.getSnowflakeId()); +// curdDataService.add(task); +// } +// this.tusUploadFileService.updateTusUploadFile(tusUploadFileEntity); +// +// logger.info("5 - PATCH END"); +// +// response.setHeader(TusConstant.ACCESS_CONTROL_EXPOSE_HEADER, TusConstant.ACCESS_CONTROL_EXPOSE_PATCH_VALUE); +// response.setHeader(TusConstant.TUS_RESUMABLE_HEADER, TusConstant.TUS_RESUMABLE_VALUE); +// response.setHeader(TusConstant.UPLOAD_OFFSET_HEADER, Long.toString(tusUploadFileEntity.getOffset())); +// response.setStatus(204); +// return null; +// } +// +// /** +// * 文件下载，根据 Guid +// * +// * @return +// */ +// @Operation(summary = "文件下载，根据 Guid") +// @RequestMapping(method = RequestMethod.GET, path = "/{guid}") +// public ResponseEntity getRequest(@PathVariable String guid) throws Exception { +// logger.info("6 - GET START"); +// logger.info("GUID value: {}", guid); +// +// TusUploadFileEntity tusUploadFileEntity = this.tusUploadFileService.getFileInfoByOssObjectName(guid); +// if (tusUploadFileEntity == null) { +// throw TusException.FILE_NOT_FOUND; +// } +// +// if (!tusUploadFileEntity.isCompleted() && Objects.equals(tusUploadFileEntity.getUploadSize(), tusUploadFileEntity.getOffset())) { +// logger.info("Upload-length == Offset"); +// tusUploadFileEntity.setCompleted(true); +// this.tusUploadFileService.updateTusUploadFile(tusUploadFileEntity); +// } +// +// if (!tusUploadFileEntity.isCompleted()) { +// throw TusException.FILE_UPLOAD_UNCOMPLETED; +// } +// +// Resource fileResource = tusUploadFileService.loadResource(guid); +// +// logger.info("6 - GET END"); +// return ResponseEntity.ok().header(HttpHeaders.CONTENT_DISPOSITION, +// "attachment; filename=\"" + tusUploadFileEntity.getFileName() + "\"").body(fileResource); +// } +// +// @Operation(summary = "上传文件") +// @RequestMapping(method = RequestMethod.PATCH, value = "/ossTest/{guid}") +// public RequestEntity patchOssRequest( +// @PathVariable String guid, +// InputStream inputStream, +// HttpServletResponse response) { +// +// ossClientService.uploadTest(inputStream, guid); +// +// +// response.setStatus(204); +// return null; +// } +// +// +// @Operation(summary = "上传文件") +// @RequestMapping(method = RequestMethod.PATCH, value = "/cosTest/{guid}") +// public RequestEntity patchCosRequest( +// @PathVariable String guid, +// InputStream inputStream, +// HttpServletResponse response) { +// +// cosClientService.uploadTest(inputStream, guid); +// +// +// response.setStatus(204); +// return null; +// } +// @ApiOperation("测试") +// @GetMapping("/test/{type}/{num}") +// public void test(@PathVariable String type, @PathVariable Integer num) throws Exception { +// for (int i = 0; i < num; i++) { +// logger.info("发送消息:{}",i); +// kafkaManager.send("jl-cos-uat", type); +// } +// } +// +// @ApiOperation("测试") +// @GetMapping("/copyFile") +// public void copyFile(@RequestParam(value = "count") Integer count){ +// cosClientService.copyFile(count); +// } +// +// public static void main(String[] args) { +// String metadata = "filename 5rKz5Y2X5oGS5Y2O56eR5oqA5pyJ6ZmQ5YWs5Y+4X1hZOTI2MDIwMTAxX+aKpeS7t+aKleagh+aWh+S7ti5maWxl"; +// byte[] nameByte = Base64.getDecoder().decode(metadata.split(",")[0].split(" ")[1]); +// System.out.println(new String(nameByte)); +// } +//} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/entity/ApiResultVO.java b/src/main/java/com/coscoshipping/ebtp/system/tus/entity/ApiResultVO.java new file mode 100644 index 0000000..4b228db --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/entity/ApiResultVO.java @@ -0,0 +1,23 @@ +package com.coscoshipping.ebtp.system.tus.entity; + +import lombok.Data; + +/** + *

+ * ＜p＞＜br＞ + * + * @author liuh + */ +@Data +public class ApiResultVO { + private Integer code; + private String message; + private Object data; + + public ApiResultVO(Integer code, String message, String s) { + } + + public static ApiResultVO success() { + return new ApiResultVO(0, "成功", null); + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/entity/TusUploadFileEntity.java b/src/main/java/com/coscoshipping/ebtp/system/tus/entity/TusUploadFileEntity.java new file mode 100644 index 0000000..145201e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/entity/TusUploadFileEntity.java @@ -0,0 +1,38 @@ +package com.coscoshipping.ebtp.system.tus.entity; + +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartTag; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; +import java.util.UUID; + +/** + *

+ * ＜p＞＜br＞ + * + * @author liuh + */ +@Setter +@Getter +@ToString +public class TusUploadFileEntity { + private String guid; + private String ossObjectName; + private String bucketName; + private String fileName; + private Long uploadSize; + private Long offset; + private String uploadId; + private Integer partNumber; + private List partTags; + private boolean isCompleted; + + private String tfile; + + + public TusUploadFileEntity() { + this.guid = UUID.randomUUID().toString().replace("-", ""); + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/exception/ControllerErrorHandler.java b/src/main/java/com/coscoshipping/ebtp/system/tus/exception/ControllerErrorHandler.java new file mode 100644 index 0000000..0431f06 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/exception/ControllerErrorHandler.java @@ -0,0 +1,37 @@ +package com.coscoshipping.ebtp.system.tus.exception; + +import com.coscoshipping.ebtp.system.tus.entity.ApiResultVO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; + +import java.io.IOException; + +/** + * 全局错误处理 + */ +@ControllerAdvice +@ResponseStatus(HttpStatus.OK) +public class ControllerErrorHandler { + + // 日志记录 + private static final Logger logger = LoggerFactory.getLogger(ControllerErrorHandler.class); + + @ResponseBody + @ExceptionHandler(value = TusException.class) + public ApiResultVO handleISSException(TusException e) { + logger.error(e.getMessage()); + return new ApiResultVO(e.getCode(), e.getMessage(), ""); + } + + @ResponseBody + @ExceptionHandler(value = IOException.class) + public ApiResultVO handleIOException(IOException e) { + logger.error(e.getMessage()); + return new ApiResultVO(999, e.getMessage(), ""); + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/exception/TusException.java b/src/main/java/com/coscoshipping/ebtp/system/tus/exception/TusException.java new file mode 100644 index 0000000..c400951 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/exception/TusException.java @@ -0,0 +1,41 @@ +package com.coscoshipping.ebtp.system.tus.exception; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * 错误信息 + */ +@Getter +@Setter +@ToString +@AllArgsConstructor +public class TusException extends RuntimeException { + protected Integer code; + protected String message; + + public static final TusException UNKNOWN_ERROR = new TusException(1000, "未知错误"); + public static final TusException FILE_PREVIEW_FAIL = new TusException( + 1001, "不支持的文件类型!"); + public static final TusException FILE_NOT_FOUND = new TusException( + 1002, "文件不存在!"); + public static final TusException FILE_READ_IO_FAIL = new TusException( + 1003, "读取过程中出现错误!"); + public static final TusException FILE_UPLOAD_UNCOMPLETED = new TusException( + 1004, "文件未上传完成!"); + public static final TusException FILE_BEYOUND_SIZE = new TusException( + 1005, "文件尺寸大于原始尺寸!"); + public static final TusException FILE_UPLOAD_OFFSET_ERROR = new TusException( + 1007, "Upload Offsets 不一致!"); + public static final TusException HEADER_CONTENT_TYPE_NEED = new TusException( + 1008, "需要 Content-Type Header!"); + public static final TusException HEADER_OFFSET_NEED = new TusException( + 1009, "需要 Offset Header Header!"); + public static final TusException HEADER_CONTENT_LENGTH_NEED = new TusException( + 5010, "需要 Content-Length Header!"); + public static final TusException FILE_SIZE_ERROR = new TusException( + 5011, "文件长度错误!"); + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/service/CosClientService.java b/src/main/java/com/coscoshipping/ebtp/system/tus/service/CosClientService.java new file mode 100644 index 0000000..599a601 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/service/CosClientService.java @@ -0,0 +1,25 @@ +package com.coscoshipping.ebtp.system.tus.service; + +import com.coscoshipping.ebtp.system.tus.entity.TusUploadFileEntity; +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +/** + *

+ * ＜p＞＜br＞ + * + * @author liuh + */ +public interface CosClientService { + + TusUploadFileEntity createFile(TusUploadFileEntity tusUploadFileEntity); + + boolean fileAppend(TusUploadFileEntity tusUploadFileEntity, InputStream inputStream); + + boolean uploadFile(MultipartFile file); + + boolean uploadTest(InputStream inputStream, String guid); + + void copyFile( Integer count); +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/service/OssClientService.java b/src/main/java/com/coscoshipping/ebtp/system/tus/service/OssClientService.java new file mode 100644 index 0000000..955e41e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/service/OssClientService.java @@ -0,0 +1,40 @@ +package com.coscoshipping.ebtp.system.tus.service; + + +import com.aliyun.oss.model.PutObjectResult; +import com.coscoshipping.ebtp.system.tus.entity.TusUploadFileEntity; +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartUploadDTO; + +import javax.servlet.http.HttpServletResponse; +import java.io.InputStream; + +/** + *

+ * ＜p＞＜br＞ + * + * @author liuh + */ +public interface OssClientService { + + TusUploadFileEntity createMutiPartFile(TusUploadFileEntity tusUploadFileEntity); + + TusUploadFileEntity fileAppend(TusUploadFileEntity tusUploadFileEntity, InputStream inputStream); + + boolean uploadTest(InputStream inputStream, String guid); + + PartUploadDTO initMultipartUpload(String objectName, InputStream inputStream, int fileSize, boolean isComplete); + + PartUploadDTO mutiPartUpload(String objectName, String uploadId, InputStream inputStream, int partNum, int fileSize, boolean isComplete); + + PutObjectResult onceUpload(String objectName, InputStream inputStream); + + InputStream downloadStream(String bucketName, String objectName); + + Long queryObjectOffset(String objectName); + + String attachmentUpload(byte[] bytes,String id, String fileType); + + boolean attachmentDownload(String fileName, HttpServletResponse httpServletResponse); + + boolean fileDownload(String fileName, HttpServletResponse httpServletResponse); +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/service/TusUploadFileService.java b/src/main/java/com/coscoshipping/ebtp/system/tus/service/TusUploadFileService.java new file mode 100644 index 0000000..25d0c28 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/service/TusUploadFileService.java @@ -0,0 +1,30 @@ +package com.coscoshipping.ebtp.system.tus.service; + +import com.coscoshipping.ebtp.system.tus.entity.TusUploadFileEntity; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.core.io.Resource; + +import java.io.IOException; +import java.io.InputStream; + +/** + *

+ * ＜p＞＜br＞ + * + * @author liuh + */ +public interface TusUploadFileService { + + TusUploadFileEntity processStream(TusUploadFileEntity tusUploadFileEntity,InputStream inputStream); + + Resource loadResource(String uuid) throws Exception; + + TusUploadFileEntity createFile(TusUploadFileEntity tusUploadFileEntity) throws IOException; + + boolean saveTusUploadFile(TusUploadFileEntity tusUploadFileEntity) throws JsonProcessingException; + + boolean updateTusUploadFile(TusUploadFileEntity tusUploadFileEntity) throws JsonProcessingException; + + TusUploadFileEntity getFileInfoByOssObjectName(String guid) throws JsonProcessingException; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/CosClientServiceImpl.java b/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/CosClientServiceImpl.java new file mode 100644 index 0000000..159a0e4 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/CosClientServiceImpl.java @@ -0,0 +1,282 @@ +package com.coscoshipping.ebtp.system.tus.service.impl; + +import cn.hutool.core.util.RandomUtil; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.SdkClientException; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.*; +import com.coscoshipping.ebtp.system.tus.entity.TusUploadFileEntity; +import com.coscoshipping.ebtp.system.tus.exception.TusException; +import com.coscoshipping.ebtp.system.tus.service.CosClientService; +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartTag; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +/** + * + *

+ * ＜p＞＜br＞ + * + * + * @author liuh + */ +@Slf4j +@Service +public class CosClientServiceImpl implements CosClientService { + private static String endpoint = "http://cos.gz-tst.cos.tg.unicom.local/"; + private static String accessKeyId = "BB0XPOYYXVEKXRY6G1ED"; + private static String accessKeySecret = "2aPitaVMGweGPR6N41AHTM3N7SAYRTSAZmA3vT7G "; + private static String bucketName = "349553515466:tender-file"; + private final DateFormat YMD_PATH = new SimpleDateFormat("yyyy/MM/dd"); + + @Override + public TusUploadFileEntity createFile(TusUploadFileEntity tusUploadFileEntity) { + List partTags = new ArrayList<>(); + String guid = tusUploadFileEntity.getGuid(); + + AmazonS3 conn = this.initAmazonS3Object(); + + //初始化分片上传 + InitiateMultipartUploadRequest initRequest = new InitiateMultipartUploadRequest(bucketName, guid); + InitiateMultipartUploadResult initResponse = conn.initiateMultipartUpload(initRequest); + String uploadId = initResponse.getUploadId(); + File file = new File("D:\\download\\s3browser-10-3-1.exe"); + String fileData = ""; + ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(fileData.getBytes(StandardCharsets.UTF_8)); + Integer contentLength = fileData.length(); + // 文件上传 + UploadPartRequest uploadPartRequest = new UploadPartRequest() + .withBucketName(bucketName) + .withKey(guid) + .withUploadId(uploadId) + .withPartNumber(1) + .withPartSize(file.length()) + .withFile(file); +// .withInputStream(byteArrayInputStream); + UploadPartResult uploadPartResult = conn.uploadPart(uploadPartRequest); + + try { + byteArrayInputStream.close(); + } catch (IOException e) { + throw TusException.FILE_UPLOAD_UNCOMPLETED; + } + PartTag partTag = new PartTag(); + partTag.setPartNumber(uploadPartResult.getPartETag().getPartNumber()); + partTag.setETag(uploadPartResult.getPartETag().getETag()); + partTags.add(partTag); + Integer partNumber = uploadPartResult.getPartNumber(); + S3Object object = null; + if (conn.doesObjectExist(bucketName, guid)) { + object = conn.getObject(bucketName, guid); + } + log.info("Storage s3 api, get object result :{}", object); + log.info("Storage s3 api, upload appender file end, fileName: {}, partNumber:{},uploadId:{}", guid, partNumber, uploadId); + tusUploadFileEntity.setPartTags(partTags); + tusUploadFileEntity.setUploadId(uploadId); + tusUploadFileEntity.setPartNumber(partNumber); + return tusUploadFileEntity; + } + + @Override + public boolean fileAppend(TusUploadFileEntity tusUploadFileEntity, InputStream inputStream) { + List partTags = tusUploadFileEntity.getPartTags(); + log.info("Storage s3 api, upload chunk file start"); + + String fileName = tusUploadFileEntity.getGuid(); + + Integer partNumber = tusUploadFileEntity.getPartNumber(); + String uploadId = tusUploadFileEntity.getUploadId(); + + //储存空间名 + + AmazonS3 conn = initAmazonS3Object(); + Long contentLength = tusUploadFileEntity.getUploadSize(); + + UploadPartRequest uploadPartRequest = new UploadPartRequest() + .withBucketName(bucketName) + .withKey(fileName) + .withUploadId(uploadId) + .withPartNumber(partNumber + 1) + .withPartSize(contentLength) + .withInputStream(inputStream); + + UploadPartResult uploadPartResult = conn.uploadPart(uploadPartRequest); + partNumber = uploadPartResult.getPartNumber(); + + try { + inputStream.close(); + } catch (IOException e) { + throw TusException.FILE_UPLOAD_UNCOMPLETED; + } + List partETags = new ArrayList<>(); + partETags.add(uploadPartResult.getPartETag()); + partTags.forEach(f -> partETags.add(new PartETag(f.getPartNumber(), f.getETag()))); +// if (tusUploadFileEntity.isCompleted()) { + //完成分片上传，生成储存对象 + CompleteMultipartUploadRequest compRequest = new CompleteMultipartUploadRequest(bucketName, fileName, + uploadId, partETags); + conn.completeMultipartUpload(compRequest); +// } + + log.info("Storage s3 api, upload chunk file end"); + + return true; + } + + @Override + public boolean uploadFile(MultipartFile file) { + + String originalName = file.getOriginalFilename(); + assert originalName != null; + String fileType = originalName.substring(originalName.lastIndexOf(".") + 1); + String storageName = RandomUtil.randomString(32).toUpperCase(Locale.ROOT) + "." + fileType; + //路径修改 + String filePath = YMD_PATH.format(new Date((System.currentTimeMillis()))) + "/" + storageName; + AmazonS3 amazonS3 = initAmazonS3Object(); + try { + //文件名：filePath， 文件流：file.getInputStream() + amazonS3.putObject(bucketName, filePath, file.getInputStream(), new ObjectMetadata()); + //文件名：filePath，CannedAccessControlList.Private：私有 + amazonS3.setObjectAcl(bucketName, filePath, CannedAccessControlList.Private); + } catch (IOException e) { + log.error("IOException: {}", e.getMessage()); + } + return true; + } + + @Override + public boolean uploadTest(InputStream inputStream, String guid) { + String storageName = RandomUtil.randomString(32).toUpperCase(Locale.ROOT); + //路径修改 + String filePath = YMD_PATH.format(new Date((System.currentTimeMillis()))) + "/" + storageName; + AmazonS3 amazonS3 = initAmazonS3Object(); + //文件名：filePath， 文件流：file.getInputStream() + amazonS3.putObject(bucketName, filePath, inputStream, new ObjectMetadata()); + //文件名：filePath，CannedAccessControlList.Private：私有 + amazonS3.setObjectAcl(bucketName, filePath, CannedAccessControlList.Private); + return true; + } + + @Override + public void copyFile(Integer count) { + try { + AmazonS3 s3Client = initAmazonS3Object(); + List fileList = new ArrayList<>(); + fileList.add("1C18520430B4426FB7B28BBFBBEB7840.file"); + fileList.add("D39AAFB7B6C34223826B3ED1A20755EE.file"); + fileList.add("EFF10E3757184A3C83BE5EA91566FA93.file"); + + String sourcePath = "2022/02/23/"; + String destinationKey = "2022/02/30/"; + String cfsDestinationKey = "/storage/files/cosTest/upload/"; + for (int i = 50; i < 101; i++) { + int finalI = i; + fileList.forEach(f -> { + String sourceKey = sourcePath + f; + //cos copy + long getStartTime = System.currentTimeMillis(); + CopyObjectRequest copyObjRequest = new CopyObjectRequest(bucketName, sourceKey, bucketName, destinationKey + finalI + "/" + f); + s3Client.copyObject(copyObjRequest); + log.info("copycost：{}", System.currentTimeMillis() - getStartTime); + //cfs copy +// File newFile = new File(cfsDestinationKey + count + "/" + f); +// s3Client.getObject(new com.amazonaws.services.s3.model.GetObjectRequest(bucketName, sourceKey), newFile); + }); + } + + + + + } catch (SdkClientException e) { + // The call was transmitted successfully, but Amazon S3 couldn't process + // it, so it returned an error response. + e.printStackTrace(); + } + } + + private AmazonS3 initAmazonS3Object(){ + // 新建一个凭证 + AWSCredentials credentials = new BasicAWSCredentials(accessKeyId,accessKeySecret); + ClientConfiguration clientConfig = new ClientConfiguration(); + clientConfig.setProtocol(Protocol.HTTP); + AmazonS3 amazonS3Client = new AmazonS3Client(credentials, clientConfig); + // 对象网关地址 + amazonS3Client.setEndpoint(endpoint); + + return amazonS3Client; + } + + private int deleteFile(String filepath){ + if (filepath == null) { + throw TusException.FILE_NOT_FOUND; + } else { + AmazonS3 amazonS3 = initAmazonS3Object(); + try{ + amazonS3.deleteObject(bucketName, filepath); + } catch (SdkClientException e) { + e.printStackTrace(); + } + return 0; + } + } + + private void listObject(String filePath) { + try { + AmazonS3 amazonS3 = initAmazonS3Object(); + ListObjectsV2Request req = new ListObjectsV2Request() + .withBucketName(bucketName) + .withPrefix(filePath); + ListObjectsV2Result result; + do { + result = amazonS3.listObjectsV2(req); + + for (S3ObjectSummary objectSummary : result.getObjectSummaries()) { + System.out.printf(" - %s (size: %d)\n", objectSummary.getKey(), objectSummary.getSize()); + } + // If there are more than maxKeys keys in the bucket, get a continuation token + // and list the next objects. + String token = result.getNextContinuationToken(); + System.out.println("Next Continuation Token: " + token); + req.setContinuationToken(token); + } while (result.isTruncated()); + } catch (AmazonServiceException e) { + // The call was transmitted successfully, but Amazon S3 couldn't process + // it, so it returned an error response. + e.printStackTrace(); + } catch (SdkClientException e) { + // Amazon S3 couldn't be contacted for a response, or the client + // couldn't parse the response from Amazon S3. + e.printStackTrace(); + } + } + +// public static void main(String[] args) throws FileNotFoundException { +// File file = new File("C:\\Users\\Neo\\Desktop\\上传测试文件\\浪潮天元通信信息系统有限公司_XY926020101.zip"); +// String storageName = RandomUtil.randomString(32).toUpperCase(Locale.ROOT); +// //路径修改 +// String filePath = YMD_PATH.format(new Date((System.currentTimeMillis()))) + "/" + storageName; +// AmazonS3 amazonS3 = initAmazonS3Object(); +// //文件名：filePath， 文件流：file.getInputStream() +// amazonS3.putObject(bucketName, filePath, new FileInputStream(file), new ObjectMetadata()); +// //文件名：filePath，CannedAccessControlList.Private：私有 +// amazonS3.setObjectAcl(bucketName, filePath, CannedAccessControlList.Private); +// } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/OssClientServiceImpl.java b/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/OssClientServiceImpl.java new file mode 100644 index 0000000..842b9a8 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/OssClientServiceImpl.java @@ -0,0 +1,524 @@ +package com.coscoshipping.ebtp.system.tus.service.impl; + +import cn.hutool.core.net.URLEncoder; +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.internal.OSSConstants; +import com.aliyun.oss.model.*; +import com.coscoshipping.ebtp.system.tus.constant.TusConstant; +import com.coscoshipping.ebtp.system.tus.entity.TusUploadFileEntity; +import com.coscoshipping.ebtp.system.tus.service.OssClientService; +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartTag; +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartUploadDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import javax.servlet.ServletOutputStream; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.UUID; + +/** + *

+ * ＜p＞＜br＞ + * + * @author liuh + */ +@Slf4j +@Service +public class OssClientServiceImpl implements OssClientService { + @Value("${oss.endpoint}") + private String endpoint; + @Value("${oss.accessKeyId}") + private String accessKeyId; + @Value("${oss.accessKeySecret}") + private String accessKeySecret; + @Value("${oss.bucketName}") + private String bucketName; + + @Override + public TusUploadFileEntity createMutiPartFile(TusUploadFileEntity tusUploadFileEntity) { + OSS client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + ObjectMetadata meta = new ObjectMetadata(); + //过期时间一天 + Date expiration = new Date(System.currentTimeMillis() + 3600L * 1000 * 24); + meta.setExpirationTime(expiration); + String content = ""; + AppendObjectRequest appendObjectRequest = new AppendObjectRequest(bucketName, tusUploadFileEntity.getOssObjectName(), new ByteArrayInputStream(content.getBytes()), meta); + appendObjectRequest.setPosition(0L); + AppendObjectResult appendObjectResult = client.appendObject(appendObjectRequest); + // 文件的64位CRC值。此值根据ECMA-182标准计算得出。 + tusUploadFileEntity.setBucketName(bucketName); + log.info("文件地址{},crc{}", tusUploadFileEntity.getOssObjectName(), appendObjectResult.getObjectCRC()); + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message: {}", oe.getErrorMessage()); + log.info("Error Code: {}", oe.getErrorCode()); + log.info("Request ID: {}", oe.getRequestId()); + log.info("Host ID: {}", oe.getHostId()); + } catch (ClientException ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message: {}", ce.getMessage()); + } catch (Throwable throwable) { + log.info("Error Message: {}", throwable.getMessage()); + throwable.printStackTrace(); + } finally { + client.shutdown(); + } + return tusUploadFileEntity; + } + + @Override + public TusUploadFileEntity fileAppend(TusUploadFileEntity tusUploadFileEntity, InputStream inputStream) { + OSS client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + ObjectMetadata metadata = new ObjectMetadata(); + metadata.setContentEncoding(OSSConstants.DEFAULT_CHARSET_NAME); + String objectName = tusUploadFileEntity.getOssObjectName(); + AppendObjectRequest appendObjectRequest; + boolean contains = objectName.contains(TusConstant.OSS_SPILT) && objectName.contains(TusConstant.OSS_OBJECT_PREFIX); + if (contains) { + String[] split = objectName.split(TusConstant.OSS_SPILT); + appendObjectRequest = new AppendObjectRequest(split[0], split[1], inputStream, metadata); + } else { + appendObjectRequest = new AppendObjectRequest(bucketName, objectName, inputStream, metadata); + } + appendObjectRequest.setPosition(tusUploadFileEntity.getOffset()); + AppendObjectResult appendObjectResult = client.appendObject(appendObjectRequest); + tusUploadFileEntity.setOffset(appendObjectResult.getNextPosition()); + if (tusUploadFileEntity.getUploadSize().equals(tusUploadFileEntity.getOffset())) { + tusUploadFileEntity.setCompleted(true); + } + log.info("文件地址{},crc{}", tusUploadFileEntity.getOssObjectName(), appendObjectResult.getObjectCRC()); + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message: {}", oe.getErrorMessage()); + log.info("Error Code: {}", oe.getErrorCode()); + log.info("Request ID: {}", oe.getRequestId()); + log.info("Host ID: {}", oe.getHostId()); + } catch (ClientException ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message: {}", ce.getMessage()); + } catch (Throwable throwable) { + log.info("Error Message: {}", throwable.getMessage()); + throwable.printStackTrace(); + } finally { + client.shutdown(); + } + return tusUploadFileEntity; + } + + @Override + public boolean uploadTest(InputStream inputStream, String guid) { +// long t = System.currentTimeMillis(); +// OSS client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); +// try { +// ObjectMetadata meta = new ObjectMetadata(); +// +// AppendObjectRequest appendObjectRequest = new AppendObjectRequest(bucketName, PropertyUtils.getSnowflakeId(), inputStream, meta); +// appendObjectRequest.setPosition(0L); +// AppendObjectResult appendObjectResult = client.appendObject(appendObjectRequest); +// log.info("文件crc：{},nextPosition:{}", appendObjectResult.getObjectCRC(), appendObjectResult.getNextPosition()); +// +// log.info("cost time ({}",(System.currentTimeMillis() - t) + "ms)."); + // +// +// ObjectListing objectListing = client.listObjects(bucketName); +// List sums = objectListing.getObjectSummaries(); +// for (OSSObjectSummary s : sums) { +// log.info("\t{}",s.getKey()); + // +// // 删除文件。如需删除文件夹，请将ObjectName设置为对应的文件夹名称。如果文件夹非空，则需要将文件夹下的所有object删除后才能删除该文件夹。 +//// client.deleteObject(bucketName, s.getKey()); +//// client.getObject(new GetObjectRequest(bucket Name, key), new File("C:\\下载文件")); +// } +// } catch (OSSException oe) { +// log.info("Caught an OSSException, which means your request made it to OSS, " +// + "but was rejected with an error response for some reason."); +// log.info("Error Message: {}" , oe.getErrorMessage()); +// log.info("Error Code: {}" , oe.getErrorCode()); +// log.info("Request ID: {}" , oe.getRequestId()); +// log.info("Host ID: {}" , oe.getHostId()); +// } catch (ClientException ce) { +// log.info("Caught an ClientException, which means the client encountered " +// + "a serious internal problem while trying to communicate with OSS, " +// + "such as not being able to access the network."); +// log.info("Error Message: {}" , ce.getMessage()); +// } catch (Throwable throwable) { +// throwable.printStackTrace(); +// } finally { +// client.shutdown(); +// } + return true; + } + + @Override + public PartUploadDTO initMultipartUpload(String objectName, InputStream inputStream, int fileSize, boolean isComplete) { + // 创建OSSClient实例。 + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + // 创建InitiateMultipartUploadRequest对象。 + InitiateMultipartUploadRequest request = new InitiateMultipartUploadRequest(bucketName, objectName); + + // 初始化分片。 + InitiateMultipartUploadResult upresult = ossClient.initiateMultipartUpload(request); + // 返回uploadId，它是分片上传事件的唯一标识。您可以根据该uploadId发起相关的操作，例如取消分片上传、查询分片上传等。 + String uploadId = upresult.getUploadId(); + + // partETags是PartETag的集合。PartETag由分片的ETag和分片号组成。 + List partETags = new ArrayList<>(); + + + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(bucketName); + uploadPartRequest.setKey(objectName); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(inputStream); + // 设置分片大小。除了最后一个分片没有大小限制，其他的分片最小为100 KB。 + uploadPartRequest.setPartSize(fileSize); + // 设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出此范围，OSS将返回InvalidArgument错误码。 + uploadPartRequest.setPartNumber(1); + // 每个分片不需要按顺序上传，甚至可以在不同客户端上传，OSS会按照分片号排序组成完整的文件。 + UploadPartResult uploadPartResult = ossClient.uploadPart(uploadPartRequest); + // 每次上传分片之后，OSS的返回结果包含PartETag。PartETag将被保存在partETags中。 + partETags.add(uploadPartResult.getPartETag()); + if (isComplete) { + // 创建CompleteMultipartUploadRequest对象。 + // 在执行完成分片上传操作时，需要提供所有有效的partETags。OSS收到提交的partETags后，会逐一验证每个分片的有效性。当所有的数据分片验证通过后，OSS将把这些分片组合成一个完整的文件。 + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, partETags); + // 完成分片上传。 + CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest); + log.info(completeMultipartUploadResult.getETag()); + } + PartUploadDTO partUploadDTO = new PartUploadDTO(); + partUploadDTO.setUploadId(uploadId); + partUploadDTO.setFullPath(objectName); + partUploadDTO.setPartNumber(1); + List partTags = new ArrayList<>(); + PartTag partTag = new PartTag(); + partTag.setPartNumber(uploadPartResult.getPartETag().getPartNumber()); + partTag.setETag(uploadPartResult.getPartETag().getETag()); + partTags.add(partTag); + partUploadDTO.setPartTags(partTags); + return partUploadDTO; + + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message:{}", oe.getErrorMessage()); + log.info("Error Code:{}", oe.getErrorCode()); + log.info("Request ID:{}", oe.getRequestId()); + log.info("Host ID:{}", oe.getHostId()); + } catch (ClientException ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message:{}", ce.getMessage()); + } finally { + if (ossClient != null) { + ossClient.shutdown(); + } + } + return null; + } + + @Override + public PartUploadDTO mutiPartUpload(String objectName, String uploadId, InputStream inputStream, int partNum, int fileSize, boolean isComplete) { + log.info("OssClient分片上传objectName：{}，uploadId：{}，partNum：{}，fileSize：{}，isComplete：{}", objectName, uploadId, partNum, fileSize, isComplete); + // 创建OSSClient实例。 + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + + UploadPartRequest uploadPartRequest = new UploadPartRequest(); + uploadPartRequest.setBucketName(bucketName); + uploadPartRequest.setKey(objectName); + uploadPartRequest.setUploadId(uploadId); + uploadPartRequest.setInputStream(inputStream); + // 设置分片大小。除了最后一个分片没有大小限制，其他的分片最小为100 KB。 + uploadPartRequest.setPartSize(fileSize); + // 设置分片号。每一个上传的分片都有一个分片号，取值范围是1~10000，如果超出此范围，OSS将返回InvalidArgument错误码。 + uploadPartRequest.setPartNumber(partNum); + // 每个分片不需要按顺序上传，甚至可以在不同客户端上传，OSS会按照分片号排序组成完整的文件。 + ossClient.uploadPart(uploadPartRequest); + // 每次上传分片之后，OSS的返回结果包含PartETag。PartETag将被保存在partETags中。 + // partETags是PartETag的集合。PartETag由分片的ETag和分片号组成。 + List partETags = new ArrayList<>(); + // 分页列举已上传的分片。 + PartListing partListing; + ListPartsRequest listPartsRequest = new ListPartsRequest(bucketName, objectName, uploadId); + do { + partListing = ossClient.listParts(listPartsRequest); + for (PartSummary part : partListing.getParts()) { + PartETag partETag = new PartETag(part.getPartNumber(), part.getETag()); +// part.getLastModified(); + partETags.add(partETag); + } + + listPartsRequest.setPartNumberMarker(partListing.getNextPartNumberMarker()); + + } while (partListing.isTruncated()); + + if (isComplete && partETags.size() == partNum) { + log.info("分片全部上传，等待合并：{}", objectName); + // 创建CompleteMultipartUploadRequest对象。 + // 在执行完成分片上传操作时，需要提供所有有效的partETags。OSS收到提交的partETags后，会逐一验证每个分片的有效性。当所有的数据分片验证通过后，OSS将把这些分片组合成一个完整的文件。 + CompleteMultipartUploadRequest completeMultipartUploadRequest = + new CompleteMultipartUploadRequest(bucketName, objectName, uploadId, partETags); + // 完成分片上传。 + CompleteMultipartUploadResult completeMultipartUploadResult = ossClient.completeMultipartUpload(completeMultipartUploadRequest); + log.info("完成分片上传{}",completeMultipartUploadResult.getETag()); + } + PartUploadDTO partUploadDTO = new PartUploadDTO(); + partUploadDTO.setUploadId(uploadId); + partUploadDTO.setFullPath(objectName); + partUploadDTO.setPartNumber(partNum); + partUploadDTO.setPartTags(this.convertPartTag(partETags)); + return partUploadDTO; + + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message:{}", oe.getErrorMessage()); + log.info("Error Code:{}", oe.getErrorCode()); + log.info("Request ID:{}", oe.getRequestId()); + log.info("Host ID:{}", oe.getHostId()); + } catch (ClientException ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message:{}", ce.getMessage()); + } finally { + if (ossClient != null) { + ossClient.shutdown(); + } + } + return null; + } + + @Override + public PutObjectResult onceUpload(String objectName, InputStream inputStream) { + OSS client = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + return client.putObject(new PutObjectRequest(bucketName, objectName, inputStream, new ObjectMetadata())); + } + + @Override + public InputStream downloadStream(String bucketName,String objectName) { + // 创建OSSClient实例。 + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + OSSObject ossObject = null; + try { + // ossObject包含文件所在的存储空间名称、文件名称、文件元信息以及一个输入流。 + ossObject = ossClient.getObject(bucketName, objectName); + log.info("bucketName:{}, objectName:{}", bucketName, objectName); + log.info("ossObject:{}", ossObject.toString()); + //返回文件流 + return ossObject.getObjectContent(); + + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message:{}", oe.getErrorMessage()); + log.info("Error Code:{}", oe.getErrorCode()); + log.info("Request ID:{}", oe.getRequestId()); + log.info("Host ID:{}", oe.getHostId()); + } catch (Throwable ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message:{}", ce.getMessage()); + } + return null; + } + + @Override + public Long queryObjectOffset(String objectName) { + // 创建OSSClient实例。 + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + boolean contains = objectName.contains(TusConstant.OSS_SPILT) && objectName.contains(TusConstant.OSS_OBJECT_PREFIX); + if (contains) { + String[] split = objectName.split(TusConstant.OSS_SPILT); + OSSObject ossObject = ossClient.getObject(new GetObjectRequest(split[0], split[1])); + return (long) ossObject.getObjectContent().available(); + } else { + OSSObject ossObject = ossClient.getObject(new GetObjectRequest(bucketName, objectName)); + return (long) ossObject.getObjectContent().available(); + } + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message:{}", oe.getErrorMessage()); + log.info("Error Code:{}", oe.getErrorCode()); + log.info("Request ID:{}", oe.getRequestId()); + log.info("Host ID:{}", oe.getHostId()); + } catch (Throwable ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message:{}", ce.getMessage()); + } finally { + if (ossClient != null) { + ossClient.shutdown(); + } + } + return null; + } + + @Override + public String attachmentUpload(byte[] bytes, String id, String fileType) { + // 创建OSSClient实例。 + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + + String objectName = "oss/attachment" + File.separator + id + File.separator + "resources/tb"+ File.separator + + UUID.randomUUID().toString().replace("-", "") + "." + fileType; + try { + PutObjectRequest putObjectRequest = new PutObjectRequest(bucketName, objectName, new ByteArrayInputStream(bytes)); + ossClient.putObject(putObjectRequest); + + } catch (OSSException oe) { + log.info("文件上传失败：{}",bucketName + TusConstant.OSS_SPILT + objectName); + System.out.println("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + System.out.println("Error Message:" + oe.getErrorMessage()); + System.out.println("Error Code:" + oe.getErrorCode()); + System.out.println("Request ID:" + oe.getRequestId()); + System.out.println("Host ID:" + oe.getHostId()); + } catch (ClientException ce) { + log.info("文件上传失败：{}",bucketName + TusConstant.OSS_SPILT + objectName); + System.out.println("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + System.out.println("Error Message:" + ce.getMessage()); + } finally { + if (ossClient != null) { + ossClient.shutdown(); + } + } + log.info("文件上传成功：{}", bucketName + TusConstant.OSS_SPILT + objectName); + return bucketName + TusConstant.OSS_SPILT + objectName; + } + + @Override + public boolean attachmentDownload(String fileName, HttpServletResponse httpServletResponse) { + boolean contains = fileName.contains(TusConstant.OSS_SPILT) && fileName.contains(TusConstant.OSS_OBJECT_PREFIX); + if (contains) { + String[] split = fileName.split(TusConstant.OSS_SPILT); + InputStream inputStream = null; + try { + inputStream = this.downloadStream(split[0], split[1]); + ServletOutputStream outputStream = httpServletResponse.getOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, length); + } + outputStream.close(); + }catch (IOException e) { + log.error(e.getMessage()); + }finally { + try { + assert inputStream != null; + inputStream.close(); + } catch (Exception e) { + log.info("流关闭失败"); + e.printStackTrace(); + } + } + } else { + return false; + } + return false; + } + + @Override + public boolean fileDownload(String fileName, HttpServletResponse httpServletResponse) { + boolean contains = fileName.contains(TusConstant.OSS_SPILT) && fileName.contains(TusConstant.OSS_OBJECT_PREFIX); + if (contains) { + String[] split = fileName.split(TusConstant.OSS_SPILT); + String s = split[1].replaceAll("/", "_"); + httpServletResponse.setHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", + URLEncoder.ALL.encode(s, StandardCharsets.UTF_8))); + InputStream inputStream = null; + try { + inputStream = this.downloadStream(split[0], split[1]); + ServletOutputStream outputStream = httpServletResponse.getOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = inputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, length); + outputStream.flush(); + } + outputStream.close(); + }catch (IOException e) { + log.error(e.getMessage()); + }finally { + try { + assert inputStream != null; + inputStream.close(); + } catch (Exception e) { + log.info("流关闭失败"); + e.printStackTrace(); + } + } + } else { + return false; + } + return false; + } + + private List convertPartTag(List partETags) { + List returnList = new ArrayList<>(); + partETags.forEach(partETag -> { + PartTag partTag = new PartTag(); + partTag.setETag(partETag.getETag()); + partTag.setPartNumber(partETag.getPartNumber()); + returnList.add(partTag); + }); + return returnList; + } + public static void main(String[] args) { + +// OSS client = new OSSClientBuilder().build("10.238.9.62", "TgQXRdxaaqwaDXDN", "4ld16kglZ1aTa8BOsJAPHFcVMjRwSF"); +// OSS client = new OSSClientBuilder().build("10.242.0.95", "wikrNkei48gp4buV", "LD8jDhf0qow4iel8vO7wno6bp4ZM7q"); +// try { +// String objectName = "test-mall3:oss/file/2022/05/25/18cf4ac631144fb2be27b06f194fbf29.file"; +// String[] split = objectName.split(":"); +// String targetName = "D:\\download\\" + split[1].replaceAll("/", "_"); +// client.getObject(new GetObjectRequest(split[0], split[1]), new File(targetName)); +// } catch (OSSException oe) { +// log.info("Caught an OSSException, which means your request made it to OSS, " +// + "but was rejected with an error response for some reason."); +// log.info("Error Message: {}" , oe.getErrorMessage()); +// log.info("Error Code: {}" , oe.getErrorCode()); +// log.info("Request ID: {}" , oe.getRequestId()); +// log.info("Host ID: {}" , oe.getHostId()); +// } catch (ClientException ce) { +// log.info("Caught an ClientException, which means the client encountered " +// + "a serious internal problem while trying to communicate with OSS, " +// + "such as not being able to access the network."); +// log.info("Error Message: {}" , ce.getMessage()); +// } catch (Throwable throwable) { +// throwable.printStackTrace(); +// } finally { +// client.shutdown(); +// } + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/S3Test.java b/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/S3Test.java new file mode 100644 index 0000000..0eaf983 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/service/impl/S3Test.java @@ -0,0 +1,86 @@ +package com.coscoshipping.ebtp.system.tus.service.impl; + +import com.amazonaws.ClientConfiguration; +import com.amazonaws.Protocol; +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.model.*; +import com.amazonaws.util.StringUtils; + +import java.io.ByteArrayInputStream; +import java.io.File; + +/** + * S3Test + */ +public class S3Test { + + static String bucket = "tender-file"; + static String accessKey = "BB0XPOYYXVEKXRY6G1ED"; + static String secretKey = "2aPitaVMGweGPR6N41AHTM3N7SAYRTSAZmA3vT7G"; + static String serviceEndpoint = "cos.gz-tst.cos.tg.unicom.local"; // e.g., "cos.gz-tst.cos.tg.unicom.local" + static String region = "gz-tst"; // e.g., "gz-tst" + + static AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); + static AWSStaticCredentialsProvider awsStaticCredentialsProvider = new AWSStaticCredentialsProvider(credentials); + static ClientConfiguration config = new ClientConfiguration(); + static EndpointConfiguration endpointConfiguration = new EndpointConfiguration(serviceEndpoint, region); + static AmazonS3 conn = AmazonS3ClientBuilder.standard() + .withCredentials(awsStaticCredentialsProvider) + .withClientConfiguration(config.withProtocol(Protocol.HTTP)) + .withEndpointConfiguration(endpointConfiguration).build(); + + public static String upload(String key, String content) { + PutObjectResult result = conn.putObject(bucket, key, content); + return key; + } + + public static void main(String[] args) { + + // 列出所有桶列表 +// List buckets = conn.listBuckets(); +// for (Bucket bucket : buckets) { +// System.out.println(bucket.getName() + "\t" + StringUtils.fromDate(bucket.getCreationDate())); +// } +// +// // 创建桶 + Bucket bucket1 = conn.createBucket(bucket); +// System.out.println(bucket.getName()); + + // 列出桶内对象 + ObjectListing objects = conn.listObjects(bucket); + do { + for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) { + System.out.println(objectSummary.getKey() + "\t" + + objectSummary.getSize() + "\t" + + StringUtils.fromDate(objectSummary.getLastModified())); + } + objects = conn.listNextBatchOfObjects(objects); + } while (objects.isTruncated()); + + // 创建对象 + ByteArrayInputStream input = new ByteArrayInputStream("Hello World!".getBytes()); + conn.putObject(bucket, "hello.txt", input, new ObjectMetadata()); + + // 修改对象的访问控制权限 + conn.setObjectAcl(bucket, "hello.txt", CannedAccessControlList.PublicRead); + + // 下载一个对象(到本地文件) + conn.getObject(new GetObjectRequest(bucket, "hello.txt"), new File("./hello.txt")); + + // 生成对象下载链接（带签名） + GeneratePresignedUrlRequest request = new GeneratePresignedUrlRequest(bucket, "hello.txt"); + System.out.println(conn.generatePresignedUrl(request)); + + // 删除一个对象 + conn.deleteObject(bucket, "hello.txt"); + + //删除桶 +// conn.deleteBucket(bucket); + + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/tus/util/TusUtils.java b/src/main/java/com/coscoshipping/ebtp/system/tus/util/TusUtils.java new file mode 100644 index 0000000..4dfb268 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/tus/util/TusUtils.java @@ -0,0 +1,134 @@ +package com.coscoshipping.ebtp.system.tus.util; + +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.io.*; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.FileLock; +import java.nio.file.Path; +import java.util.LinkedList; +import java.util.List; + +import static java.nio.file.StandardOpenOption.*; + +/** + * Utility class that contains various static helper methods + */ +public class TusUtils { + + private static final Logger log = LoggerFactory.getLogger(TusUtils.class); + private static final int LOCK_FILE_RETRY_COUNT = 3; + private static final long LOCK_FILE_SLEEP_TIME = 500; + + private TusUtils() { + //This is a utility class that only holds static utility methods + } + + public static String getHeader(HttpServletRequest request, String header) { + return StringUtils.trimToEmpty(request.getHeader(header)); + } + + public static Long getLongHeader(HttpServletRequest request, String header) { + try { + return Long.valueOf(getHeader(request, header)); + } catch (NumberFormatException ex) { + return null; + } + } + + + + public static List parseConcatenationIDsFromHeader(String uploadConcatValue) { + List output = new LinkedList<>(); + + String idString = StringUtils.substringAfter(uploadConcatValue, ";"); + for (String id : StringUtils.trimToEmpty(idString).split("\\s")) { + output.add(id); + } + + return output; + } + + public static T readSerializable(Path path, Class clazz) throws IOException { + T info = null; + if (path != null) { + try (FileChannel channel = FileChannel.open(path, READ)) { + //Lock will be released when the channel is closed + if (lockFileShared(channel) != null) { + + try (ObjectInputStream ois = new ObjectInputStream(Channels.newInputStream(channel))) { + info = clazz.cast(ois.readObject()); + } catch (ClassNotFoundException e) { + //This should not happen + info = null; + } + } else { + throw new IOException("Unable to lock file " + path); + } + } + } + return info; + } + + + public static void writeSerializable(Serializable object, Path path) throws IOException { + if (path != null) { + try (FileChannel channel = FileChannel.open(path, WRITE, CREATE, TRUNCATE_EXISTING)) { + //Lock will be released when the channel is closed + if (lockFileExclusively(channel) != null) { + + try (OutputStream buffer = new BufferedOutputStream(Channels.newOutputStream(channel)); + ObjectOutput output = new ObjectOutputStream(buffer)) { + + output.writeObject(object); + } + } else { + throw new IOException("Unable to lock file " + path); + } + } + } + } + + public static FileLock lockFileExclusively(FileChannel channel) throws IOException { + return lockFile(channel, false); + } + + public static FileLock lockFileShared(FileChannel channel) throws IOException { + return lockFile(channel, true); + } + + /** + * Sleep the specified number of milliseconds + * @param sleepTimeMillis The time to sleep in milliseconds + */ + public static void sleep(long sleepTimeMillis) { + try { + Thread.sleep(sleepTimeMillis); + } catch (InterruptedException e) { + log.warn("Sleep was interrupted"); + // Restore interrupted state... + Thread.currentThread().interrupt(); + } + } + + private static FileLock lockFile(FileChannel channel, boolean shared) throws IOException { + int i = 0; + FileLock lock = null; + do { + if (i > 0) { + sleep(LOCK_FILE_SLEEP_TIME); + } + + lock = channel.tryLock(0L, Long.MAX_VALUE, shared); + + i++; + } while (lock == null && i < LOCK_FILE_RETRY_COUNT); + + return lock; + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/CurdDataService.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/CurdDataService.java new file mode 100644 index 0000000..677d0a0 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/CurdDataService.java @@ -0,0 +1,15 @@ +package com.coscoshipping.ebtp.system.updownload.background.service; + +import com.coscoshipping.ebtp.system.updownload.entity.dao.UploadDataDAO; + +/** + * 上传任务状态管理服务 + * + * @author Ajaxfan + */ +public interface CurdDataService{ + + UploadDataDAO add(UploadDataDAO task); + + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/NoticeTaskService.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/NoticeTaskService.java new file mode 100644 index 0000000..c335ce9 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/NoticeTaskService.java @@ -0,0 +1,16 @@ +package com.coscoshipping.ebtp.system.updownload.background.service; + +import com.coscoshipping.ebtp.system.updownload.entity.dto.TaskDTO; + +/** + * + *

+ * ＜p＞＜br＞ + * + * + * @author liuh + */ +public interface NoticeTaskService { + + boolean noticeStep(TaskDTO dto); +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/CurdDataServiceImpl.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/CurdDataServiceImpl.java new file mode 100644 index 0000000..d929652 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/CurdDataServiceImpl.java @@ -0,0 +1,42 @@ +package com.coscoshipping.ebtp.system.updownload.background.service.impl; + +import com.chinaunicom.mall.ebtp.common.log.enums.EbtpLogBusinessModule; +import com.chinaunicom.mall.ebtp.common.log.enums.EbtpLogType; +import com.chinaunicom.mall.ebtp.common.log.service.OperationLogService; +import com.coscoshipping.ebtp.system.updownload.background.service.CurdDataService; +import com.coscoshipping.ebtp.system.updownload.background.service.NoticeTaskService; +import com.coscoshipping.ebtp.system.updownload.entity.dao.UploadDataDAO; +import com.coscoshipping.ebtp.system.updownload.entity.dto.TaskDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.BeanUtils; +import org.springframework.stereotype.Service; +import org.springframework.validation.annotation.Validated; + +import javax.annotation.Resource; + +@Slf4j +@Service +public class CurdDataServiceImpl implements CurdDataService { + + + @Resource + private NoticeTaskService noticeTaskService; + + @Resource + private OperationLogService operationLogService; + + + @Override + public UploadDataDAO add(@Validated UploadDataDAO task) { + + TaskDTO dto = new TaskDTO(); + BeanUtils.copyProperties(task, dto); + dto.setFilename(task.getPath()); + log.info("noticeTask发送kafka前：{}", dto.toString()); + boolean noticeStep = noticeTaskService.noticeStep(dto); + operationLogService.addOperationLog("noticeTask发送kafka：", noticeStep, EbtpLogBusinessModule.TENDER_UPLOAD_RECORD, EbtpLogType.INSERT); + return task; + } + + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/NoticeTaskServiceImpl.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/NoticeTaskServiceImpl.java new file mode 100644 index 0000000..788f0f0 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/background/service/impl/NoticeTaskServiceImpl.java @@ -0,0 +1,113 @@ +package com.coscoshipping.ebtp.system.updownload.background.service.impl; + +import cn.hutool.core.util.StrUtil; +import com.coscoshipping.ebtp.system.updownload.background.service.NoticeTaskService; +import com.coscoshipping.ebtp.system.updownload.entity.dto.NotificationDTO; +import com.coscoshipping.ebtp.system.updownload.entity.dto.TaskDTO; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.producer.ProducerRecord; +import org.springframework.beans.BeanUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.SendResult; +import org.springframework.stereotype.Service; +import org.springframework.util.concurrent.ListenableFuture; + +/** + * + *

+ * ＜p＞＜br＞ + * + * + * @author liuh + */ +@Slf4j +@Service +public class NoticeTaskServiceImpl implements NoticeTaskService { + + /* kafka工具类 */ + private @Autowired + KafkaTemplate kafkaTemplate; + + /* Json工具类 */ + private @Autowired + ObjectMapper objectMapper; + + /* 发送消息主题 */ + private @Value("${kafka.topic.request-topic}") + String requestTopic; + + + @Override + public boolean noticeStep(TaskDTO dto) { + log.info("notification body: {}", dto); + + try { + NotificationDTO entity = new NotificationDTO(); + BeanUtils.copyProperties(dto, entity); + + /**************** 调换tuid 和 uuid，为了解决相同文件重复上传uuid不变的问题 *************/ +// swapTuidAndUuid(entity); + + send(entity).get(); + return true; + } catch (Exception e) { + log.error(e.getMessage()); + log.debug("send notification error. reason: maybe the channel blocked or reply unreceived:{}", dto); + +// if (incrementAndGet() <= retryLimit) { +// noticeStep(dto); +// } + } + return true; + } + + + /** + * 调整重试计数器 + * + * @return + */ +// private int incrementAndGet() { +// if (Objects.isNull(threadLocal.get())) { +// threadLocal.set(new AtomicInteger()); +// } +// return threadLocal.get().incrementAndGet(); +// } + + /** + * 发送消息 + * + * @param dto + * @throws JsonProcessingException + */ + private ListenableFuture> send(NotificationDTO dto) + throws JsonProcessingException { + String body = objectMapper.writeValueAsString(dto); + log.debug("notification message: {}", body); + + // 消息实体 + ProducerRecord record = new ProducerRecord<>(requestTopic, body); + + // 发送消息并监听返回 ( 如果指定时间内未收到消息反馈则视为消息失效 ) + return kafkaTemplate.send(record); + } + + /** + * 原来uuid对应客户端上传文件的md5，因文件在未修改状态下多次上传uuid一致，因此新增任务id---tuid + * + * @param entity + */ + private void swapTuidAndUuid(NotificationDTO entity) { + String tuid = entity.getTuid(); + + if (StrUtil.isBlank(tuid)) { + return; + } + entity.setTuid(entity.getUuid()); + entity.setUuid(tuid); + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignClient.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignClient.java new file mode 100644 index 0000000..9a95ec6 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignClient.java @@ -0,0 +1,19 @@ +package com.coscoshipping.ebtp.system.updownload.client; + + +import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse; +import com.coscoshipping.ebtp.system.updownload.entity.dto.Tfile; +import com.coscoshipping.ebtp.system.updownload.web.dto.MixDownloadParam; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; + + +@FeignClient(value = "${mconfig.feign.name.resps}", fallbackFactory = RespFeignFallbackFactory.class) +public interface RespFeignClient { + + + @PostMapping("/v1/quoteSupplier/getSupplierAttachment") + BaseResponse getSupplierAttachment(@RequestBody Tfile tfile); + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignFallbackFactory.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignFallbackFactory.java new file mode 100644 index 0000000..82da55e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/client/RespFeignFallbackFactory.java @@ -0,0 +1,28 @@ +package com.coscoshipping.ebtp.system.updownload.client; + +import cn.hutool.core.exceptions.ExceptionUtil; +import com.chinaunicom.mall.ebtp.common.base.entity.BaseResponse; +import com.coscoshipping.ebtp.system.updownload.entity.dto.Tfile; +import com.coscoshipping.ebtp.system.updownload.web.dto.MixDownloadParam; +import feign.hystrix.FallbackFactory; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class RespFeignFallbackFactory implements FallbackFactory { + @Override + public RespFeignClient create(Throwable throwable) { + RespFeignClient back = new RespFeignClient() { + + + @Override + public BaseResponse getSupplierAttachment(Tfile tfile) { + log.info(tfile.toString()); + return null; + } + }; + log.error(ExceptionUtil.stacktraceToString(throwable)); + return back; + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/common/Constant.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/common/Constant.java new file mode 100644 index 0000000..d8e7ba7 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/common/Constant.java @@ -0,0 +1,22 @@ +package com.coscoshipping.ebtp.system.updownload.common; + +public interface Constant { + + /* 描述文件名称 */ + public static final String PROFILE_NAME = "profile.json"; + + /* 文件编码s */ + public static final String ENCODING = "UTF-8"; + + /* Path Separator */ + public static final String SEPARATOR = "/"; + + /* 应用秘钥 */ + public static final String SECRETE = "Qus$%^423rt&EIURE298(*$234"; + + /* 文件来源 3.0 一期前端*/ + String FILE_SOURCE_FRONT = "front"; + + /* 文件来源 3.0 二期客户端*/ + String FILE_SOURCE_CLIENT = "client"; +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/config/CacheConfiguration.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/CacheConfiguration.java new file mode 100644 index 0000000..c13da16 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/CacheConfiguration.java @@ -0,0 +1,31 @@ +package com.coscoshipping.ebtp.system.updownload.config; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.concurrent.TimeUnit; + +@Configuration +public class CacheConfiguration { + + /** + * 创建缓存工具，缓存内容一小时内没有访问则过期 + * + * @return + */ + @Bean + public LoadingCache createCache(@Value("${mall.cacheExpire:30}") int cacheExpire) { + final CacheLoader loader = new CacheLoader() { + @Override + public String load(String key) throws Exception { + return key.toUpperCase(); + } + }; + return CacheBuilder.newBuilder().maximumSize(Integer.MAX_VALUE).expireAfterWrite(cacheExpire, TimeUnit.MINUTES).build(loader); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/config/DigitalSecurityConfiguration.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/DigitalSecurityConfiguration.java new file mode 100644 index 0000000..4940cd3 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/DigitalSecurityConfiguration.java @@ -0,0 +1,22 @@ +package com.coscoshipping.ebtp.system.updownload.config; + +import cn.hutool.crypto.SecureUtil; +import cn.hutool.crypto.symmetric.DES; +import com.coscoshipping.ebtp.system.updownload.common.Constant; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class DigitalSecurityConfiguration { + + /** + * DES 加解密工具 + * + * @return + */ + @Bean + public DES des() { + return SecureUtil.des(Constant.SECRETE.getBytes()); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigDev.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigDev.java new file mode 100644 index 0000000..05e6018 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigDev.java @@ -0,0 +1,62 @@ +package com.coscoshipping.ebtp.system.updownload.config; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ConcurrentMessageListenerContainer; +import org.springframework.kafka.requestreply.ReplyingKafkaTemplate; + +import java.util.HashMap; +import java.util.Map; + +/** + * Kafka 参数配置 + * + * @author Ajaxfan + */ +@Configuration +@Profile("local") +public class KafkaProducerConfigDev { + + private @Value("${spring.kafka.bootstrap-servers}") + String bootstrapServers; + private @Value("${kafka.topic.request-reply-topic}") + String requestReplyTopic; + private @Value("${kafka.group-id}") + String requestReplyGroupId; + + @Bean + public ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + + return new DefaultKafkaProducerFactory<>(configProps); + } + + /** + * Request-Reply 模式 ( 可以确定消息被正确消费 ) + * + * @param pf + * @param container + * @return + */ + @Bean + public ReplyingKafkaTemplate replyKafkaTemplate(ProducerFactory pf, + ConcurrentKafkaListenerContainerFactory factory) { + // 创建反馈监听 + ConcurrentMessageListenerContainer replyContainer = factory.createContainer(requestReplyTopic); + // 设置监听者组 + replyContainer.getContainerProperties().setGroupId(requestReplyGroupId); + + return new ReplyingKafkaTemplate<>(pf, replyContainer); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigPro.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigPro.java new file mode 100644 index 0000000..eac9a5e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/config/KafkaProducerConfigPro.java @@ -0,0 +1,63 @@ +package com.coscoshipping.ebtp.system.updownload.config; + +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * Kafka 参数配置 + * + * @author Ajaxfan + */ +@Configuration +@Profile({"uat", "test", "pro"}) +public class KafkaProducerConfigPro { + + private @Value("${spring.kafka.bootstrap-servers}") + String bootstrapServers; + private @Value("${kafka.topic.request-reply-topic}") + String requestReplyTopic; + private @Value("${kafka.group-id}") + String requestReplyGroupId; + private @Value("${spring.kafka.producer.properties.security.protocol}") + String securityProtocal; + private @Value("${spring.kafka.producer.properties.sasl.jaas.config}") + String jaasConfig; + private @Value("${spring.kafka.producer.properties.sasl.mechanism}") + String mechanism; + + @Bean + public ProducerFactory producerFactory() { + Map configProps = new HashMap<>(); + configProps.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, bootstrapServers); + configProps.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configProps.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer.class); + configProps.put("security.protocol", securityProtocal); + configProps.put("sasl.jaas.config", jaasConfig); + configProps.put("sasl.mechanism", mechanism); + + return new DefaultKafkaProducerFactory<>(configProps); + } + + /** + * Request-Reply 模式 ( 可以确定消息被正确消费 ) + * + * @param pf + * @param container + * @return + */ + @Bean + public KafkaTemplate kafkaTemplate() { + return new KafkaTemplate<>(producerFactory()); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/RpcDataDAO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/RpcDataDAO.java new file mode 100644 index 0000000..277d0d5 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/RpcDataDAO.java @@ -0,0 +1,23 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dao; + +import lombok.Data; +import lombok.experimental.Accessors; +import org.hibernate.validator.constraints.Length; + +//@TableName(value = "biz_rpc_data", autoResultMap = true) +@Data +@Accessors(chain = true) +public class RpcDataDAO { + + /* 任务标识 */ +// @TableId + private String id; + + /* 附加信息 */ + @Length(max = 8000) + private String content; + + /* 时间戳 */ + private Long lastUpdate; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/UploadDataDAO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/UploadDataDAO.java new file mode 100644 index 0000000..274ec08 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dao/UploadDataDAO.java @@ -0,0 +1,41 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dao; + +import lombok.Data; +import lombok.experimental.Accessors; +import org.hibernate.validator.constraints.Length; + +//@TableName(value = "biz_upload_task", autoResultMap = true) +@Data +@Accessors(chain = true) +public class UploadDataDAO { + + /* 任务标识 */ +// @TableId + private String uuid; + + /* 文件名称 */ + private String filename; + + /* 文件实际名称 */ + private String fileRealName; + + /* 文件实际大小 */ + private Integer totalSize; + + /* 文件绝对路径 */ + private String path; + + /* 文件保存的相对路径 */ + private String relativePath; + + /* 附加信息 */ + @Length(max = 4000) + private String tfile; + + /* 时间戳 */ + private Long lastUpdate; + + /* 任务唯一标识 */ + private String tuid; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/NotificationDTO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/NotificationDTO.java new file mode 100644 index 0000000..a609941 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/NotificationDTO.java @@ -0,0 +1,48 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.experimental.Accessors; + +@Data +@NoArgsConstructor +@AllArgsConstructor +@Accessors(chain = true) +public class NotificationDTO { + + /** + * 对象id + */ + private String uuid; + + /** + * 文件原始路径 + */ + private String path; + + /** + * 文件名称 + */ + private String filename; + + /* 文件实际名称 */ + private String fileRealName; + + + /** + * 文件保存的相对路径 + */ + private String relativePath; + + /** + * 文件描述信息 + */ + private String tfile; + + /** + * 任务唯一标识 + */ + private String tuid; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartTag.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartTag.java new file mode 100644 index 0000000..dea946e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartTag.java @@ -0,0 +1,17 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dto; + +import lombok.Data; + +/** + * + *

+ * ＜p＞＜br＞ + * + * + * @author liuh + */ +@Data +public class PartTag { + private int partNumber; + private String eTag; +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartUploadDTO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartUploadDTO.java new file mode 100644 index 0000000..5aa308a --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/PartUploadDTO.java @@ -0,0 +1,40 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dto; + +import lombok.Data; + +import java.util.List; + +@Data +public class PartUploadDTO { + + /** + * 上传oss任务id（uploadId） + */ + private String uploadId; + + /** + * 上传文件全名带路径（objectName/key） + */ + private String fullPath; + + /** + * 上传文件总长度 + */ + private Integer length; + + /** + * 总的分块数 + */ + private Integer totalParts; + + /** + * 分块索引 + */ + private Integer partNumber; + + + /** + * PartETag的集合。PartETag由分片的ETag和分片号组成。 + */ + private List partTags; +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/TaskDTO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/TaskDTO.java new file mode 100644 index 0000000..5453825 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/TaskDTO.java @@ -0,0 +1,46 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dto; + +import lombok.Data; + +@Data +public class TaskDTO { + + /** + * 对象id + */ + private String uuid; + + /** + * 文件原始路径 + */ + private String path; + + /** + * 文件名称 + */ + private String filename; + + /* 文件实际名称 */ + private String fileRealName; + + /** + * 文件保存的相对路径 + */ + private String relativePath; + + /** + * 文件描述信息 + */ + private String tfile; + + /** + * chunk文件的原始路径 + */ + private String source; + + /** + * 任务id + */ + private String tuid; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/Tfile.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/Tfile.java new file mode 100644 index 0000000..84ed0d5 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/Tfile.java @@ -0,0 +1,23 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dto; + +import lombok.Data; +import lombok.experimental.Accessors; + +import java.io.Serializable; + +/** + * 实体类 Tfile- + * + * @auto.generated + */ +@Data +@Accessors(chain = true) +public class Tfile implements Serializable { + + private static final long serialVersionUID = 1L; + + private String tdocId; + private String tendererId; + private String tendererName; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/UploadParamDTO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/UploadParamDTO.java new file mode 100644 index 0000000..686528e --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/dto/UploadParamDTO.java @@ -0,0 +1,64 @@ +package com.coscoshipping.ebtp.system.updownload.entity.dto; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; +import lombok.ToString; +import org.springframework.web.multipart.MultipartFile; + +/** + * 文件上传参数 + * + * @author ajaxfan + * @date 2020/11/11 + */ +@ToString(exclude = "file") +@Data +public class UploadParamDTO { + + /* 任务标识 (文件md5值)*/ + private String resumableIdentifier; + + /* 分块索引 */ + @JsonIgnore + private Integer resumableChunkNumber; + + /* 分块大小 */ + @JsonIgnore + private Integer resumableChunkSize; + + /* 当前分块大小 */ + @JsonIgnore + private Integer resumableCurrentChunkSize; + + /* 文件总尺寸 */ + private Integer resumableTotalSize; + + /* MIME类型(application/zip) */ + @JsonIgnore + private String resumableType; + + /* 文件名称 */ + private String resumableFilename; + + //文件原名 + private String resumableRelativePath; + + /* 总的分块数 */ + private Integer resumableTotalChunks; + + /* 文件保存的相对路径 */ + private String relativePath; + + /* 文件描述信息 */ + private String tfile; + + /* 任务唯一标识 */ + private String tuid; + + /* 文件流 */ + @JsonIgnore + private MultipartFile file; + + /* 文件来源 front前端页面/client客户端 */ + private String fileSource; +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/po/ResourcePO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/po/ResourcePO.java new file mode 100644 index 0000000..a6064ca --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/po/ResourcePO.java @@ -0,0 +1,42 @@ +package com.coscoshipping.ebtp.system.updownload.entity.po; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class ResourcePO { + + /** + * 任务标识 + */ + private String resumableIdentifier; + + /** + * 文件总尺寸 + */ + private Integer resumableTotalSize; + + /** + * 文件名称 + */ + private String resumableFilename; + + /** + * 总的分块数 + */ + private Integer resumableTotalChunks; + + /** + * 文件保存的相对路径 + */ + private String relativePath; + + /** + * 文件描述信息 + */ + private String tfile; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/vo/DownloadVO.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/vo/DownloadVO.java new file mode 100644 index 0000000..3139334 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/entity/vo/DownloadVO.java @@ -0,0 +1,14 @@ +package com.coscoshipping.ebtp.system.updownload.entity.vo; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DownloadVO { + + private String path; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/entity/MessageState.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/entity/MessageState.java new file mode 100644 index 0000000..50cc475 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/entity/MessageState.java @@ -0,0 +1,15 @@ +package com.coscoshipping.ebtp.system.updownload.kafka.entity; + +public class MessageState { + + private boolean success; + + public boolean isSuccess() { + return success; + } + + public MessageState setSuccess(boolean success) { + this.success = success; + return this; + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/maneger/MessageManager.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/maneger/MessageManager.java new file mode 100644 index 0000000..24ed438 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/kafka/maneger/MessageManager.java @@ -0,0 +1,26 @@ +package com.coscoshipping.ebtp.system.updownload.kafka.maneger; + + + + + +import com.coscoshipping.ebtp.system.updownload.kafka.entity.MessageState; + +import java.util.concurrent.ExecutionException; + +public interface MessageManager { + + MessageState send(String topic, String message) throws ExecutionException, InterruptedException; + + boolean stop(String id); + + boolean start(String id); + + boolean resume(String id); + + boolean pause(String id); + + boolean isRunning(String id); + + boolean isNotRunning(String id); +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkDownloadController.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkDownloadController.java new file mode 100644 index 0000000..ba27467 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkDownloadController.java @@ -0,0 +1,191 @@ +package com.coscoshipping.ebtp.system.updownload.web.controller; + +import cn.hutool.core.net.URLEncoder; +import cn.hutool.crypto.symmetric.DES; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.model.OSSObject; +import com.coscoshipping.ebtp.system.tus.constant.TusConstant; +import com.coscoshipping.ebtp.system.updownload.web.utils.RegexpUtil; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +@Slf4j +@RestController +@RequestMapping("v1/hulk") +public class HulkDownloadController { + + /* 目标文件根目录 */ + private @Value("${mall.merge.target}") + String fileLocation; + + /* DES 解密工具 */ + private @Autowired + DES destool; + @Value("${oss.endpoint}") + private String endpoint; + @Value("${oss.accessKeyId}") + private String accessKeyId; + @Value("${oss.accessKeySecret}") + private String accessKeySecret; + + @Operation(summary = "大文件下载") + @RequestMapping(value = "/download/**", method = RequestMethod.GET, produces = "application/zip; charset=utf-8") + public ResponseEntity download(HttpServletResponse response, + @Parameter(description = "文件路径", required = true) @RequestParam("p") String path, + @Parameter(description = "文件名称") @RequestParam(value = "n", required = false) String name) + throws IOException { + String decryptStr = destool.decryptStr(path); + log.debug("Hulk preview path: {}", decryptStr); + boolean isOssPath = decryptStr.contains(TusConstant.OSS_SPILT) && decryptStr.contains(TusConstant.OSS_OBJECT_PREFIX); + if (isOssPath) { + InputStream inputStream = null; + BufferedInputStream bis = null; + OSSObject ossObject = null; + OSS ossClient = null; + try { + log.info("文件位置：{}", decryptStr); + ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + //如果cfsPath路径种含有冒号，为oss下载 + String[] split = decryptStr.split(":"); + String s = split[0]; + String bucketName = s.substring(s.lastIndexOf("/") + 1); + log.info("bucketName:{}, objectName:{}", bucketName, split[1]); + String suffix = split[1].substring(split[1].lastIndexOf(".")); + response.setHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", + URLEncoder.ALL.encode(getFilename(name + suffix, split[1].substring(split[1].lastIndexOf("/") + 1)), StandardCharsets.UTF_8))); + + ossObject = ossClient.getObject(bucketName, split[1]); + inputStream = ossObject.getObjectContent(); + byte[] buffs = new byte[1024]; + bis = new BufferedInputStream(inputStream, 1024); + int read; + while ((read = bis.read(buffs, 0, 1024)) != -1) { + response.getOutputStream().write(buffs, 0, read); + } + + } catch (IOException e) { + log.error(e.getMessage()); + }finally { + try { + inputStream.close(); + bis.close(); + response.getOutputStream().close(); + ossObject.close(); + if (ossClient != null) { + ossClient.shutdown(); + } + } catch (Exception e) { + log.info("流关闭失败"); + e.printStackTrace(); + } + } + } + File file = new File(path = String.format("%s/%s", fileLocation, decryptStr)); + log.debug("Hulk download path: {}", path); + + if (!file.exists()) { + return new ResponseEntity<>(null, HttpStatus.NOT_FOUND); + } + + if (file.isDirectory()) { + return new ResponseEntity<>(downloadDirectory(response, file, name), HttpStatus.OK); + } + return new ResponseEntity<>(downloadFile(response, file, name), HttpStatus.OK); + } + + /** + * 按目录下载( 压缩包 ) + * + * @param response + * @param file + * @param name + * @return + * @throws IOException + */ + private StreamingResponseBody downloadDirectory(HttpServletResponse response, File file, String name) + throws IOException { + response.setHeader("Content-Disposition", String.format("attachment; filename=\"%s.zip\"", + URLEncoder.ALL.encode(getFilename(name, file.getName()), Charset.forName("UTF-8")))); + + // 根路径匹配模式(用于后面打压缩包是生成层级结构) + final String basePath = file.getCanonicalPath(); + + /** + * StreamingResponseBody 可以提高处理IO请求的处理能力 Controller + * 在处理异步请求的时候，StreamingResponseBody 会直接把流写入到response的输出流中，并且不会占用Servlet容器线程 + */ + // 创建 zip流， 输出位置指向 response.outputStream + return out -> { + try (ZipOutputStream zout = new ZipOutputStream(out)) { + FileUtils.listFiles(file, null, true).forEach(obj -> { + try { + File tmp = (File) obj; + // 创建压缩项 (文件全路径中剔除根路径，形成zip内目录层级结构) + zout.putNextEntry( + new ZipEntry(RegexpUtil.trimPrefixSlash(tmp.getCanonicalPath().replace(basePath, "")))); + // 写入zip流 （org.apache.commons.io.FileUtils 这里使用了apache的工具了， 该工具有很多IO便捷工具） + zout.write(FileUtils.readFileToByteArray(tmp)); + } catch (IOException e) { + log.error(e.getMessage()); + } + }); + } + }; + + } + + /** + * 按文件下载 + * + * @param response + * @param file + * @param name + * @return + * @throws IOException + */ + private StreamingResponseBody downloadFile(HttpServletResponse response, File file, String name) + throws IOException { + response.setHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", + URLEncoder.ALL.encode(getFilename(name, file.getName()), Charset.forName("UTF-8")))); + + /** + * StreamingResponseBody 可以提高处理IO请求的处理能力 Controller + * 在处理异步请求的时候，StreamingResponseBody 会直接把流写入到response的输出流中，并且不会占用Servlet容器线程 + */ + return out -> { + out.write(FileUtils.readFileToByteArray(file)); + }; + } + + /** + * @param name + * @return + */ + private String getFilename(String name, String defaultName) { + return StringUtils.isEmpty(name) ? defaultName : name; + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkPreviewController.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkPreviewController.java new file mode 100644 index 0000000..a211129 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkPreviewController.java @@ -0,0 +1,129 @@ +package com.coscoshipping.ebtp.system.updownload.web.controller; + +import cn.hutool.core.net.URLEncoder; +import cn.hutool.crypto.symmetric.DES; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.model.OSSObject; +import com.coscoshipping.ebtp.system.tus.constant.TusConstant; +import com.coscoshipping.ebtp.system.tus.service.OssClientService; +import com.coscoshipping.ebtp.system.updownload.web.utils.RegexpUtil; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import javax.annotation.Resource; +import javax.servlet.http.HttpServletResponse; +import java.io.BufferedInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.zip.ZipEntry; + +@Slf4j +@RestController +@RequestMapping("v1/hulk") +public class HulkPreviewController { + + /* 目标文件根目录 */ + private @Value("${mall.merge.target}") + String fileLocation; + + /* DES 解密工具 */ + private @Autowired + DES destool; + + + @Value("${oss.endpoint}") + private String endpoint; + @Value("${oss.accessKeyId}") + private String accessKeyId; + @Value("${oss.accessKeySecret}") + private String accessKeySecret; + + @Operation(summary = "文件预览") + @RequestMapping(value = "/preview/**", method = RequestMethod.GET, produces = "application/octet-stream; charset=utf-8") + public ResponseEntity download(HttpServletResponse response, + @Parameter(description = "文件路径") @RequestParam("p") String path, + @Parameter(description = "文件名称") @RequestParam(value = "n", required = false) String name) + throws IOException { + String decryptStr = destool.decryptStr(path); + log.debug("Hulk preview path: {}", decryptStr); + boolean isOssPath = decryptStr.contains(TusConstant.OSS_SPILT) && decryptStr.contains(TusConstant.OSS_OBJECT_PREFIX); + if (isOssPath) { + InputStream inputStream = null; + BufferedInputStream bis = null; + OSSObject ossObject = null; + OSS ossClient = null; + try { + log.info("文件位置：{}", decryptStr); + ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + //如果cfsPath路径种含有冒号，为oss下载 + String[] split = decryptStr.split(":"); + String s = split[0]; + String bucketName = s.substring(s.lastIndexOf("/") + 1); + log.info("bucketName:{}, objectName:{}", bucketName, split[1]); + response.setHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", + URLEncoder.ALL.encode(getFilename(name, split[1].substring(split[1].lastIndexOf("/") + 1)), StandardCharsets.UTF_8))); + + ossObject = ossClient.getObject(bucketName, split[1]); + inputStream = ossObject.getObjectContent(); + byte[] buffs = new byte[1024]; + bis = new BufferedInputStream(inputStream, 1024); + int read; + while ((read = bis.read(buffs, 0, 1024)) != -1) { + response.getOutputStream().write(buffs, 0, read); + } + + } catch (IOException e) { + log.error(e.getMessage()); + }finally { + try { + inputStream.close(); + bis.close(); + response.getOutputStream().close(); + ossObject.close(); + if (ossClient != null) { + ossClient.shutdown(); + } + } catch (Exception e) { + log.info("流关闭失败"); + e.printStackTrace(); + } + } + } + File file = new File(path = String.format("%s/%s", fileLocation, decryptStr)); + + if (!file.exists()) { + return new ResponseEntity<>(null, HttpStatus.NOT_FOUND); + } + response.setHeader("Content-Disposition", String.format("attachment; filename=\"%s\"", + URLEncoder.ALL.encode(getFilename(name, file.getName()), Charset.forName("UTF-8")))); + + return new ResponseEntity<>(out -> { + out.write(FileUtils.readFileToByteArray(file)); + }, HttpStatus.OK); + } + + /** + * @param name + * @return + */ + private String getFilename(String name, String defaultName) { + return StringUtils.isEmpty(name) ? defaultName : name; + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkUploadController.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkUploadController.java new file mode 100644 index 0000000..b9db383 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/controller/HulkUploadController.java @@ -0,0 +1,89 @@ +package com.coscoshipping.ebtp.system.updownload.web.controller; + +import com.coscoshipping.ebtp.system.updownload.entity.dto.UploadParamDTO; +import com.coscoshipping.ebtp.system.updownload.web.handler.checker.ChunkExistsChecker; +import com.coscoshipping.ebtp.system.updownload.web.handler.remover.ChunkFileRemover; +import com.coscoshipping.ebtp.system.updownload.web.service.FileUploadService; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.media.Content; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.*; + +/** + * 投标文件上传controller + * + * @author ajaxfan + * @date 2020/11/11 + */ +@Slf4j +@RestController +@RequestMapping("v1/hulk") +public class HulkUploadController { + + /** + * 文件上传服务 + */ + private @Autowired + FileUploadService fileUploadService; + + /** + * 检测工具 + */ + private @Autowired + ChunkExistsChecker checker; + + /** + * 分片文件清理工具 + */ + private @Autowired + ChunkFileRemover remover; + + + /** + * 文件上传 + * + * @param param 上传元数据 + * @return 文件上传状态 + */ + @Operation(summary = "单文件上传") + @ApiResponses({@ApiResponse(responseCode = "200", description = "大文件上传成功", content = @Content), + @ApiResponse(responseCode = "400", description = "大文件上传失败", content = @Content)}) + @PostMapping(value = "/upload") + public ResponseEntity fileUpload(@RequestBody UploadParamDTO param) { + log.info("Hulk Upload: md5: {}, ChunkNo: {}, tuid: {}", param.getResumableIdentifier(), + param.getResumableChunkNumber(),param.getTuid()); + return ResponseEntity.ok().body(fileUploadService.upload(param)); + } + + /** + * 文件上传 + * + * @param param 上传元数据 + * @return 文件上传状态 + */ + @Operation(summary = "检查分片文件是否已经存在") + @ResponseStatus(code = HttpStatus.OK) + @GetMapping(value = "/upload") + public Boolean check(UploadParamDTO param) { + return checker.isExists(param); + } + + /** + * 文件上传 + * + * @return 文件上传状态 + */ + @Operation(summary = "删除已上传的分片") + @ResponseStatus(code = HttpStatus.OK) + @PostMapping(value = "/upload/delete") + public Boolean delete(@RequestParam("identifierParameterName") String identifier) { + return remover.delete(identifier); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDataItem.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDataItem.java new file mode 100644 index 0000000..758ac50 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDataItem.java @@ -0,0 +1,16 @@ +package com.coscoshipping.ebtp.system.updownload.web.dto; + +import lombok.Data; + +@Data +public class MixDataItem { + + private String oid; + private String path; + + /** + * cfs实际存储路径 + * ＜p＞如果oid为空，cfsPath是解密后文件路径，path是下载需要转换的路径＜br＞ + */ + private String cfsPath; +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDownloadParam.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDownloadParam.java new file mode 100644 index 0000000..7198732 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/dto/MixDownloadParam.java @@ -0,0 +1,14 @@ +package com.coscoshipping.ebtp.system.updownload.web.dto; + +import lombok.Data; + +import java.util.List; + +@Data +public class MixDownloadParam { + + private String path; + private String name; + private List data; + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/checker/ChunkExistsChecker.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/checker/ChunkExistsChecker.java new file mode 100644 index 0000000..4737ce1 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/checker/ChunkExistsChecker.java @@ -0,0 +1,132 @@ +package com.coscoshipping.ebtp.system.updownload.web.handler.checker; + +import com.aliyun.oss.ClientException; +import com.aliyun.oss.OSS; +import com.aliyun.oss.OSSClientBuilder; +import com.aliyun.oss.OSSException; +import com.aliyun.oss.model.ListPartsRequest; +import com.aliyun.oss.model.PartListing; +import com.aliyun.oss.model.PartSummary; +import com.coscoshipping.ebtp.system.tus.constant.TusConstant; +import com.coscoshipping.ebtp.system.updownload.common.Constant; +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartUploadDTO; +import com.coscoshipping.ebtp.system.updownload.entity.dto.UploadParamDTO; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.time.LocalDateTime; +import java.util.concurrent.TimeUnit; + +/** + * 端点续传功能：分片文件检测 + * + * @author ajaxfan + */ +@Slf4j +@Component +public class ChunkExistsChecker { + @Value("${oss.endpoint}") + private String endpoint; + @Value("${oss.accessKeyId}") + private String accessKeyId; + @Value("${oss.accessKeySecret}") + private String accessKeySecret; + @Value("${oss.bucketName}") + private String bucketName; + + @Autowired(required = false) + @Qualifier("idempotentRedisTemplate") + private RedisTemplate redisTemplate; + + /* Json格式化工具 */ + private @Autowired + ObjectMapper objectMapper; + + /* 上传位置 */ + private @Value("${mall.upload-path}") + String uploadPath; + + /** + * 检查分片文件是否存在 + * + * @param dto + * @return + */ + public boolean isExists(UploadParamDTO dto){ + // 根目录 ( uuid -- 由上传组件生成的任务id ) + 文件名称 + String redisKey = dto.getTuid() + Constant.SEPARATOR + dto.getResumableIdentifier(); + + if (redisTemplate.hasKey(redisKey)) { + //查询redis是否记录该上传任务 + String redisValue = (String) redisTemplate.opsForValue().get(redisKey); + //获取缓存的文件信息 + OSS ossClient = new OSSClientBuilder().build(endpoint, accessKeyId, accessKeySecret); + try { + PartUploadDTO partUploadDTO = objectMapper.readValue(redisValue, PartUploadDTO.class); + log.info("分片已经存在,uploadId：{},partUploadDTO:{} ", partUploadDTO.getUploadId(),partUploadDTO.toString()); + String fullPath = partUploadDTO.getFullPath(); + if (fullPath.contains(TusConstant.OSS_SPILT) && fullPath.contains(TusConstant.OSS_OBJECT_PREFIX)) { + String[] split = fullPath.split(TusConstant.OSS_SPILT); + ListPartsRequest listPartsRequest = new ListPartsRequest(split[0], split[1], partUploadDTO.getUploadId()); + // 指定List的起始位置。只有分片号大于此参数值的分片会被列举。 + listPartsRequest.setPartNumberMarker(dto.getResumableChunkNumber() - 1); + PartListing partListing = ossClient.listParts(listPartsRequest); + for (PartSummary part : partListing.getParts()) { + //分片序号与分片大小完全一致，确认该分片存在，无需上传 + if (part.getPartNumber() == dto.getResumableChunkNumber() && + dto.getResumableCurrentChunkSize() == part.getSize()) { + log.info("分片号:{},分片数据大小:{},分片的ETag:{},分片的最后修改时间:{}", + part.getPartNumber(), part.getSize(), part.getETag(), part.getLastModified()); + return true; + } + } + } + return false; + } catch (OSSException oe) { + log.info("Caught an OSSException, which means your request made it to OSS, " + + "but was rejected with an error response for some reason."); + log.info("Error Message:{}", oe.getErrorMessage()); + log.info("Error Code:{}", oe.getErrorCode()); + log.info("Request ID:{}", oe.getRequestId()); + log.info("Host ID:{}", oe.getHostId()); + } catch (ClientException ce) { + log.info("Caught an ClientException, which means the client encountered " + + "a serious internal problem while trying to communicate with OSS, " + + "such as not being able to access the network."); + log.info("Error Message:{}", ce.getMessage()); + } catch (JsonProcessingException ce) { + log.info("Error Message:{}", ce.getMessage()); + } finally { + if (ossClient != null) { + ossClient.shutdown(); + } + } + return false; + } else { + return false; + } + +// /********* 构建本地绝存储路径 ( 系统指定根目录/uuid) ************************/ +// String basePath = uploadPath + Constant.SEPARATOR + +// // 根目录 ( uuid -- 由上传组件生成的任务id ) +// dto.getResumableIdentifier() + Constant.SEPARATOR + +// // 文件名称 +// dto.getResumableFilename() + Constant.SEPARATOR; +// /********************************************************************** ************************/ +// +// File file = FileUtils.getFile(String.format("%s/%s", basePath, dto.getResumableChunkNumber())); +// +// return file.exists() && dto.getResumableCurrentChunkSize() == FileUtils.sizeOf(file); + + + } +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/remover/ChunkFileRemover.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/remover/ChunkFileRemover.java new file mode 100644 index 0000000..483b1c7 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/remover/ChunkFileRemover.java @@ -0,0 +1,44 @@ +package com.coscoshipping.ebtp.system.updownload.web.handler.remover; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.IOException; + +/** + * 用户取消上传，删除已上传的分片数据 + * + * @author ajaxfan + */ +@Slf4j +@Component +public class ChunkFileRemover { + + /* 上传位置 */ + private @Value("${mall.upload-path}") + String uploadPath; + + /** + * 检查分片文件是否存在 + * + * @param dto + * @return + */ + public boolean delete(String id) { + File file = new File(String.format("%s/%s", uploadPath, id)); + + if (file.exists()) { + try { + FileUtils.forceDelete(file); + return true; + } catch (IOException e) { + log.error(e.getMessage()); + } + } + return false; + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/ChunkStorageHandler.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/ChunkStorageHandler.java new file mode 100644 index 0000000..a5bfd08 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/ChunkStorageHandler.java @@ -0,0 +1,10 @@ +package com.coscoshipping.ebtp.system.updownload.web.handler.storage; + +import com.coscoshipping.ebtp.system.updownload.entity.dto.UploadParamDTO; + +@FunctionalInterface +public interface ChunkStorageHandler { + + boolean save(UploadParamDTO t); + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/impl/ChunkStorageHandlerImpl.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/impl/ChunkStorageHandlerImpl.java new file mode 100644 index 0000000..b003010 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/handler/storage/impl/ChunkStorageHandlerImpl.java @@ -0,0 +1,174 @@ +package com.coscoshipping.ebtp.system.updownload.web.handler.storage.impl; + +import cn.hutool.core.util.RandomUtil; +import com.coscoshipping.ebtp.system.tus.constant.TusConstant; +import com.coscoshipping.ebtp.system.tus.service.OssClientService; +import com.coscoshipping.ebtp.system.updownload.common.Constant; +import com.coscoshipping.ebtp.system.updownload.entity.dao.UploadDataDAO; +import com.coscoshipping.ebtp.system.updownload.entity.dto.PartUploadDTO; +import com.coscoshipping.ebtp.system.updownload.entity.dto.UploadParamDTO; +import com.coscoshipping.ebtp.system.updownload.web.handler.storage.ChunkStorageHandler; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; + +import javax.annotation.Resource; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Locale; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** + * 持久化数据到磁盘 + * + * @file: + * @description: + * @author: ajaxfan + * @date: 2020/11/11 + * @version: V1.0 + * @update + */ +@Component +@Slf4j +@ConditionalOnProperty(prefix = "mall", name = "chunk.storage.medium", havingValue = "disk") +public class ChunkStorageHandlerImpl implements ChunkStorageHandler { + @Value("${oss.endpoint}") + private String endpoint; + @Value("${oss.accessKeyId}") + private String accessKeyId; + @Value("${oss.accessKeySecret}") + private String accessKeySecret; + @Value("${oss.bucketName}") + private String bucketName; + + @Autowired(required = false) + @Qualifier("idempotentRedisTemplate") + private RedisTemplate redisTemplate; + + /* Json格式化工具 */ + private @Autowired + ObjectMapper objectMapper; + + @Resource + private OssClientService ossClientService; + + /* 任务数据管理服务 */ + //@Resource + //private CurdDataService dataService; + + /** + * 文件保存 + */ + @Override + public boolean save(UploadParamDTO dto) { + return Optional.of(dto).map(this::writeChunkFileToDisk).isPresent(); + } + + /** + * 写出分块文件到磁盘 + * + * @param dto + * @throws IOException + */ + private UploadParamDTO writeChunkFileToDisk(UploadParamDTO dto){ + String fileType = dto.getResumableFilename().substring(dto.getResumableFilename().lastIndexOf(".") + 1); + DateFormat yearMonthPath = new SimpleDateFormat("yyyy/MM/dd"); + + // 根目录 ( uuid -- 由上传组件生成的任务id ) + 文件名称 + String redisKey = dto.getTuid() + Constant.SEPARATOR + dto.getResumableIdentifier(); + PartUploadDTO partUploadDTO; + if (redisTemplate.hasKey(redisKey)) { + //如果存在key，获取value，更新 + String redisValue = (String) redisTemplate.opsForValue().get(redisKey); + log.info("分片上传，存在key：{},value:{}",redisKey, redisValue); + //获取缓存的文件信息 + try { + partUploadDTO = objectMapper.readValue(redisValue, PartUploadDTO.class); + partUploadDTO.setPartNumber(partUploadDTO.getPartNumber() + 1); + boolean isComplete = partUploadDTO.getPartNumber().equals(dto.getResumableTotalChunks()); + PartUploadDTO newPartUploadDTO = ossClientService.mutiPartUpload(partUploadDTO.getFullPath(), + partUploadDTO.getUploadId(), + dto.getFile().getInputStream(), + partUploadDTO.getPartNumber(), + dto.getResumableCurrentChunkSize(), + isComplete); + newPartUploadDTO.setTotalParts(dto.getResumableTotalChunks()); + newPartUploadDTO.setLength(dto.getResumableTotalSize()); + //redis值覆盖 + redisTemplate.opsForValue().getAndSet(redisKey, objectMapper.writeValueAsString(newPartUploadDTO)); + redisTemplate.expire(redisKey, 1, TimeUnit.DAYS); + if (isComplete && newPartUploadDTO.getPartTags().size() == dto.getResumableTotalChunks()) { + log.info("上传完成，分片数：{}",newPartUploadDTO.getPartTags().size()); + //dataService.add(createUploadDao(dto, partUploadDTO.getFullPath())); + } + return dto; + } catch (Exception e) { + log.error(e.getMessage()); + return null; + } + } else { + //如果没有key，初始化文件、初始化redis + String filePathPrefix = TusConstant.OSS_OBJECT_PREFIX + Constant.SEPARATOR + yearMonthPath.format(new Date((System.currentTimeMillis()))); + String objectName = filePathPrefix + Constant.SEPARATOR + RandomUtil.randomString(32).toUpperCase(Locale.ROOT) + "." + fileType; + + try { + log.info("分片上传，不存在key，初始化：{}",redisKey); + //分块索引 == 总的分块数 且 总的分块数 == 1 不需要分片上传，执行单独上传 + boolean isComplete = dto.getResumableChunkNumber().equals(dto.getResumableTotalChunks()) && dto.getResumableTotalChunks() == 1; + if (isComplete) { + log.info("单文件上传，obName:{}", objectName); + ossClientService.onceUpload(objectName, dto.getFile().getInputStream()); + //dataService.add(createUploadDao(dto, objectName)); + } else { + log.info("分片文件上传，初始化，obName:{}", objectName); + partUploadDTO = ossClientService.initMultipartUpload(objectName, dto.getFile().getInputStream(), dto.getResumableCurrentChunkSize(),false); + partUploadDTO.setLength(dto.getResumableTotalSize()); + partUploadDTO.setTotalParts(dto.getResumableTotalChunks()); + redisTemplate.opsForValue().setIfAbsent(redisKey, objectMapper.writeValueAsString(partUploadDTO), 1, TimeUnit.DAYS); + } + return dto; + } catch (IOException e) { + log.error("IOException:{}",e.getMessage()); + return null; + } + } + } + + + /** + * @param dto + * @param path + * @return + */ + private UploadDataDAO createUploadDao(UploadParamDTO dto, String path) { + UploadDataDAO dao = new UploadDataDAO(); + // 任务标识 + dao.setUuid(dto.getTuid()); + // 文件绝对路径 + dao.setPath(bucketName + TusConstant.OSS_SPILT + path); + // 文件名称 + dao.setFilename(dto.getResumableFilename()); + // 文件实际名称 + dao.setFileRealName(dto.getResumableRelativePath()); + // 文件相对路径 + dao.setRelativePath(dto.getRelativePath()); + // 文件实际尺寸 + dao.setTotalSize(dto.getResumableTotalSize()); + // 前端传递的业务数据(不做处理， 后续透传到kafka) + dao.setTfile(dto.getTfile()); + // 任务唯一标识 + dao.setTuid(dto.getResumableIdentifier()); + + return dao.setLastUpdate(Calendar.getInstance().getTimeInMillis()); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/FileUploadService.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/FileUploadService.java new file mode 100644 index 0000000..81ae931 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/FileUploadService.java @@ -0,0 +1,10 @@ +package com.coscoshipping.ebtp.system.updownload.web.service; + +import com.coscoshipping.ebtp.system.updownload.entity.dto.UploadParamDTO; + +public interface FileUploadService { + + + Boolean upload(UploadParamDTO param); + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/impl/FileUploadServiceImpl.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/impl/FileUploadServiceImpl.java new file mode 100644 index 0000000..7b3d750 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/service/impl/FileUploadServiceImpl.java @@ -0,0 +1,35 @@ +package com.coscoshipping.ebtp.system.updownload.web.service.impl; + +import com.coscoshipping.ebtp.system.updownload.entity.dto.UploadParamDTO; +import com.coscoshipping.ebtp.system.updownload.web.handler.storage.ChunkStorageHandler; +import com.coscoshipping.ebtp.system.updownload.web.service.FileUploadService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 投标文件上传service + * + * @author xuzh + * @date 2018/06/08 + */ +@Service +public class FileUploadServiceImpl implements FileUploadService { + + /** + * 分片文件持久化 + */ + private @Autowired + ChunkStorageHandler storageHandler; + + /** + * 分片文件持久化 + * + * @param param + * @throws Exception + */ + @Override + public Boolean upload(UploadParamDTO param) { + return storageHandler.save(param); + } + +} diff --git a/src/main/java/com/coscoshipping/ebtp/system/updownload/web/utils/RegexpUtil.java b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/utils/RegexpUtil.java new file mode 100644 index 0000000..d578c16 --- /dev/null +++ b/src/main/java/com/coscoshipping/ebtp/system/updownload/web/utils/RegexpUtil.java @@ -0,0 +1,33 @@ +package com.coscoshipping.ebtp.system.updownload.web.utils; + +/** + * 自定义正则工具 + * + * @author Ajaxfan + */ +public final class RegexpUtil { + + private static final String PREFIX_SLASH_PATTERN = "^[/\\\\]*"; + private static final String NEIGHBOUR_SLASH = "[/\\\\]{2,}"; + + /** + * 去除路径描述符中开始位置的斜杠符 + * + * @param source + * @return + */ + public static String trimPrefixSlash(String source) { + return source.replaceAll(PREFIX_SLASH_PATTERN, ""); + } + + /** + * 移除相邻的斜杠 (防止生成zip文件的时候产生错误的层级) + * + * @param source + * @return + */ + public static String trimNeighbourSlash(String source) { + return source.replaceAll(NEIGHBOUR_SLASH, "/"); + } + +} diff --git a/src/main/resources/application-master.yml b/src/main/resources/application-master.yml index 65e5c4f..73c3b7a 100644 --- a/src/main/resources/application-master.yml +++ b/src/main/resources/application-master.yml @@ -13,7 +13,7 @@ spring: writeTimeout: 35000 nacos: discovery: - server-addr: 192.168.110.109:8848 + server-addr: 127.0.0.1:8848 #192.168.110.109:8848 aop: auto: true #开启spring的aop配置 proxy-target-class: true @@ -80,7 +80,7 @@ spring: redis: sentinel: master: mymaster - nodes: 10.60.161.59:26379, 10.60.161.59:26380, 10.60.161.59:26381 + nodes: 192.168.110.231:26379 #10.60.161.59:26379, 10.60.161.59:26380, 10.60.161.59:26381 password: pass database: sharding: 1 @@ -89,7 +89,7 @@ spring: userinfo: 4 unifast: - #文件服务 + #文件服务 storage: exeFormat: exe,sys,com #可执行文件格式不许上传 allowFilesFormat: doc,docx,xls,xlsx,ppt,pptx,jpg,png,gif,bmp,jpeg,txt,mp3,mp4,mkv #允许上传文件的格式 @@ -100,10 +100,25 @@ unifast: type: local # local, sftp, fastdfs, ceph -> 存储类型 #ceph配置 access-key: nlC38UGdiuRVeUUfWExq #访问凭证 - secret-key: GFVPUGSCCqUIPBMS4ohKf7odfyLSTl8hUvhJlMsO #访问凭证 + secret-key: 5RCE4DYEygav2oVaroeslgh4oWe5BrTqo49EYzDy #访问凭证 endpoint: http://192.168.110.231:9000 #对象网关地址 bucket-name: cosco #Bucket名称 +# 自定义 kafka 配置 +kafka: + topic: + # 发送topic + request-topic: jl_demo + # 反馈topic + request-reply-topic: jl_echo + # 最大重试次数 + retry-limit: 2 + # 回声超时时长 (单位: 秒) + reply-timeout: 5 + # 回执订阅组 + group-id: request-replay-group + + mybatis-plus: configuration: # 是否开启自动驼峰命名规则映射:从数据库列名到Java属性驼峰命名的类似映射 @@ -215,6 +230,16 @@ mall: serviceId: core-service-document-center core_service_expert: serviceId: mall-expe + # 文件块存储路径 + upload-path: /storage/uploads/ + # 文件合并设置 + merge: + target: /storage/files/ + # chunks文件块持久化介质 + chunk: + storage: + medium: disk + # 任务调度生成token配置 job: