From 05dbe8f0d4126c0057a04c504d813b3e68a27948 Mon Sep 17 00:00:00 2001 From: wanggeng <450292408@qq.com> Date: Mon, 18 Oct 2021 00:11:38 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BA=86minio=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E5=AD=98=E5=82=A8=E6=9C=8D=E5=8A=A1=E5=99=A8=E5=AF=B9?= =?UTF-8?q?=E6=8E=A5=E5=8A=9F=E8=83=BD=EF=BC=8C=E6=96=B0=E5=A2=9E=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E9=94=99=E8=AF=AF=E9=A1=B5=E9=9D=A2=EF=BC=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../wgink/properties/FileMinIoProperties.java | 48 ++ .../ink/wgink/properties/FileProperties.java | 22 + .../wgink/common/advice/ResponseAdvice.java | 16 + .../common/base/DefaultBaseController.java | 23 +- .../main/resources/templates/error/error.html | 31 ++ .../resources/templates/login/form/save.html | 2 +- .../templates/login/form/update.html | 2 +- module-file/pom.xml | 7 + .../controller/route/FileRouteController.java | 22 +- .../ink/wgink/module/file/dao/IFileDao.java | 8 + .../file/service/IDefaultFileService.java | 84 +++ .../module/file/service/IFileService.java | 16 + .../file/service/IMinIoFileService.java | 49 ++ .../service/impl/DefaultFileServiceImpl.java | 508 ++++++++++++++++++ .../file/service/impl/FileServiceImpl.java | 495 ++--------------- .../service/impl/MinIoFileServiceImpl.java | 271 ++++++++++ .../resources/mybatis/mapper/file-mapper.xml | 29 + module-file/src/test/java/MinIOTest.java | 47 ++ pom.xml | 9 + 19 files changed, 1188 insertions(+), 501 deletions(-) create mode 100644 basic-properties/src/main/java/ink/wgink/properties/FileMinIoProperties.java create mode 100644 common/src/main/resources/templates/error/error.html create mode 100644 module-file/src/main/java/ink/wgink/module/file/service/IDefaultFileService.java create mode 100644 module-file/src/main/java/ink/wgink/module/file/service/IMinIoFileService.java create mode 100644 module-file/src/main/java/ink/wgink/module/file/service/impl/DefaultFileServiceImpl.java create mode 100644 module-file/src/main/java/ink/wgink/module/file/service/impl/MinIoFileServiceImpl.java create mode 100644 module-file/src/test/java/MinIOTest.java diff --git a/basic-properties/src/main/java/ink/wgink/properties/FileMinIoProperties.java b/basic-properties/src/main/java/ink/wgink/properties/FileMinIoProperties.java new file mode 100644 index 00000000..e3e4e901 --- /dev/null +++ b/basic-properties/src/main/java/ink/wgink/properties/FileMinIoProperties.java @@ -0,0 +1,48 @@ +package ink.wgink.properties; + +/** + * @ClassName: FileMinIOProperties + * @Description: minio文件属性 + * @Author: wanggeng + * @Date: 2021/10/17 5:04 下午 + * @Version: 1.0 + */ +public class FileMinIoProperties { + + private String endpoint; + private String accessKey; + private String secretKey; + private Boolean secure; + + public String getEndpoint() { + return endpoint == null ? "" : endpoint.trim(); + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public String getAccessKey() { + return accessKey == null ? "" : accessKey.trim(); + } + + public void setAccessKey(String accessKey) { + this.accessKey = accessKey; + } + + public String getSecretKey() { + return secretKey == null ? "" : secretKey.trim(); + } + + public void setSecretKey(String secretKey) { + this.secretKey = secretKey; + } + + public Boolean getSecure() { + return secure == null ? false : secure; + } + + public void setSecure(Boolean secure) { + this.secure = secure; + } +} diff --git a/basic-properties/src/main/java/ink/wgink/properties/FileProperties.java b/basic-properties/src/main/java/ink/wgink/properties/FileProperties.java index b93d61cd..2a7b0dc4 100644 --- a/basic-properties/src/main/java/ink/wgink/properties/FileProperties.java +++ b/basic-properties/src/main/java/ink/wgink/properties/FileProperties.java @@ -22,6 +22,8 @@ public class FileProperties { private Integer maxFileCount; private Double imageOutputQuality; private FileMediaMaxDurationProperties mediaMaxDuration; + private Boolean useMinIo; + private FileMinIoProperties minIo; public String getUploadPath() { return uploadPath; @@ -87,6 +89,22 @@ public class FileProperties { this.mediaMaxDuration = mediaMaxDuration; } + public Boolean getUseMinIo() { + return useMinIo == null ? false : useMinIo; + } + + public void setUseMinIo(Boolean useMinIo) { + this.useMinIo = useMinIo; + } + + public FileMinIoProperties getMinIo() { + return minIo; + } + + public void setMinIo(FileMinIoProperties minIo) { + this.minIo = minIo; + } + @Override public String toString() { final StringBuilder sb = new StringBuilder("{"); @@ -106,6 +124,10 @@ public class FileProperties { .append(imageOutputQuality); sb.append(",\"mediaMaxDuration\":") .append(mediaMaxDuration); + sb.append(",\"useMinIo\":") + .append(useMinIo); + sb.append(",\"minIo\":") + .append(minIo); sb.append('}'); return sb.toString(); } diff --git a/common/src/main/java/ink/wgink/common/advice/ResponseAdvice.java b/common/src/main/java/ink/wgink/common/advice/ResponseAdvice.java index b605012b..057b269e 100644 --- a/common/src/main/java/ink/wgink/common/advice/ResponseAdvice.java +++ b/common/src/main/java/ink/wgink/common/advice/ResponseAdvice.java @@ -24,6 +24,8 @@ import javax.servlet.http.HttpServletResponse; import java.io.IOException; import java.io.UnsupportedEncodingException; import java.sql.SQLSyntaxErrorException; +import java.util.ArrayList; +import java.util.List; /** @@ -109,6 +111,20 @@ public class ResponseAdvice { response.getWriter().write(JSON.toJSONString(result)); } else { request.getSession().setAttribute("errorMessage", JSON.toJSONString(result)); + if (!response.isCommitted()) { + StackTraceElement[] stackTraces = e.getStackTrace(); + List errorStackTraces = new ArrayList(); + for(StackTraceElement stackTraceElement: stackTraces) { + errorStackTraces.add(stackTraceElement.toString()); + } + try { + request.getSession().setAttribute("errorStackTraces", errorStackTraces); + response.sendRedirect(String.format("%s/system-error", request.getContextPath())); + } catch (IOException ioException) { + ioException.printStackTrace(); + } + return; + } } } diff --git a/common/src/main/java/ink/wgink/common/base/DefaultBaseController.java b/common/src/main/java/ink/wgink/common/base/DefaultBaseController.java index 9f02df7c..57786823 100644 --- a/common/src/main/java/ink/wgink/common/base/DefaultBaseController.java +++ b/common/src/main/java/ink/wgink/common/base/DefaultBaseController.java @@ -1,38 +1,20 @@ package ink.wgink.common.base; -import ink.wgink.exceptions.ParamsException; -import ink.wgink.exceptions.SearchException; -import ink.wgink.interfaces.menu.IMenuBaseService; -import ink.wgink.interfaces.user.IUserCheckService; -import ink.wgink.pojo.dtos.menu.MenuDTO; -import ink.wgink.pojo.dtos.permission.SystemApiDTO; -import ink.wgink.util.RegexUtil; import ink.wgink.util.map.HashMapUtil; import ink.wgink.util.request.RequestUtil; -import io.swagger.models.Operation; -import io.swagger.models.Path; -import io.swagger.models.Swagger; -import io.swagger.models.Tag; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.ResponseBody; import org.springframework.web.servlet.ModelAndView; -import springfox.documentation.service.Documentation; -import springfox.documentation.spring.web.DocumentationCache; -import springfox.documentation.swagger2.mappers.ServiceModelToSwagger2Mapper; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; import java.net.URLDecoder; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; import java.util.Map; /** @@ -53,6 +35,11 @@ public class DefaultBaseController { @Autowired protected HttpServletResponse httpServletResponse; + @GetMapping("system-error") + public ModelAndView error() { + return new ModelAndView("error/error"); + } + /** * 获取请求参数 * diff --git a/common/src/main/resources/templates/error/error.html b/common/src/main/resources/templates/error/error.html new file mode 100644 index 00000000..39256284 --- /dev/null +++ b/common/src/main/resources/templates/error/error.html @@ -0,0 +1,31 @@ + + + + + + + + + + + + +
+
+ +
+ + +
+ +
+ + + diff --git a/login-base/src/main/resources/templates/login/form/save.html b/login-base/src/main/resources/templates/login/form/save.html index 6f93753f..d0999d1a 100644 --- a/login-base/src/main/resources/templates/login/form/save.html +++ b/login-base/src/main/resources/templates/login/form/save.html @@ -60,7 +60,7 @@ {{# for(var i = 0, item = files[i]; item = files[i++];) { }}
- + diff --git a/login-base/src/main/resources/templates/login/form/update.html b/login-base/src/main/resources/templates/login/form/update.html index 4aec0938..a7064b83 100644 --- a/login-base/src/main/resources/templates/login/form/update.html +++ b/login-base/src/main/resources/templates/login/form/update.html @@ -60,7 +60,7 @@ {{# for(var i = 0, item = files[i]; item = files[i++];) { }}
- + diff --git a/module-file/pom.xml b/module-file/pom.xml index 3cca457a..80c48a20 100644 --- a/module-file/pom.xml +++ b/module-file/pom.xml @@ -32,6 +32,13 @@ thumbnailator + + + + io.minio + minio + + \ No newline at end of file diff --git a/module-file/src/main/java/ink/wgink/module/file/controller/route/FileRouteController.java b/module-file/src/main/java/ink/wgink/module/file/controller/route/FileRouteController.java index 4a23637c..1f1bd8a2 100644 --- a/module-file/src/main/java/ink/wgink/module/file/controller/route/FileRouteController.java +++ b/module-file/src/main/java/ink/wgink/module/file/controller/route/FileRouteController.java @@ -3,9 +3,9 @@ package ink.wgink.module.file.controller.route; import ink.wgink.common.base.DefaultBaseController; import ink.wgink.exceptions.ParamsException; import ink.wgink.interfaces.consts.ISystemConstant; -import ink.wgink.properties.FileProperties; import ink.wgink.module.file.service.IFileService; import ink.wgink.pojo.result.ErrorResult; +import ink.wgink.properties.FileProperties; import io.swagger.annotations.*; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -15,11 +15,9 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.servlet.ModelAndView; -import javax.servlet.AsyncContext; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.util.Map; -import java.util.concurrent.CompletableFuture; /** * @ClassName: FileController @@ -121,23 +119,6 @@ public class FileRouteController extends DefaultBaseController { @ApiResponses({@ApiResponse(code = 400, message = "请求失败", response = ErrorResult.class)}) @GetMapping("download/{isOpen}/{fileId}") public void downLoad(HttpServletRequest request, HttpServletResponse response, @PathVariable("isOpen") Boolean isOpen, @PathVariable("fileId") String fileId) { - // 开启异步处理 - AsyncContext asyncContext = request.startAsync(request, response); - CompletableFuture.runAsync(() -> { - asyncDownload(asyncContext, (HttpServletRequest) asyncContext.getRequest(), (HttpServletResponse) asyncContext.getResponse(), isOpen, fileId); - }); - } - - /** - * 异步下载文件 - * - * @param asyncContext - * @param request - * @param response - * @param isOpen - * @param fileId - */ - private void asyncDownload(AsyncContext asyncContext, HttpServletRequest request, HttpServletResponse response, Boolean isOpen, String fileId) { if (isOpen == null) { throw new ParamsException("参数错误"); } @@ -148,7 +129,6 @@ public class FileRouteController extends DefaultBaseController { params.put("fileId", fileId); params.put("isOpen", isOpen); fileService.downLoadFile(request, response, params); - asyncContext.complete(); } /** diff --git a/module-file/src/main/java/ink/wgink/module/file/dao/IFileDao.java b/module-file/src/main/java/ink/wgink/module/file/dao/IFileDao.java index 0ae14d9e..d4676bde 100644 --- a/module-file/src/main/java/ink/wgink/module/file/dao/IFileDao.java +++ b/module-file/src/main/java/ink/wgink/module/file/dao/IFileDao.java @@ -91,4 +91,12 @@ public interface IFileDao extends IInitBaseTable { */ List listByMd5(String fileMd5) throws SearchException; + /** + * 文件列表 + * + * @param params + * @return + * @throws SearchException + */ + List listPO(Map params) throws SearchException; } diff --git a/module-file/src/main/java/ink/wgink/module/file/service/IDefaultFileService.java b/module-file/src/main/java/ink/wgink/module/file/service/IDefaultFileService.java new file mode 100644 index 00000000..d0b34b7d --- /dev/null +++ b/module-file/src/main/java/ink/wgink/module/file/service/IDefaultFileService.java @@ -0,0 +1,84 @@ +package ink.wgink.module.file.service; + +import ink.wgink.exceptions.FileException; +import ink.wgink.exceptions.base.SystemException; +import ink.wgink.module.file.enums.UploadTypeEnum; +import ink.wgink.pojo.pos.FilePO; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Map; + +/** + * @ClassName: IDefaultFileService + * @Description: 默认文件业务 + * @Author: wanggeng + * @Date: 2021/10/17 10:18 上午 + * @Version: 1.0 + */ +public interface IDefaultFileService { + + /** + * 上传文件 + * + * @param token app的Token + * @param uploadFile 上传文件 + * @param uploadTypeEnum 上传文件类型 + * @param params 参数 + * @throws SystemException + */ + void uploadFile(String token, MultipartFile uploadFile, UploadTypeEnum uploadTypeEnum, Map params) throws SystemException; + + /** + * 物理删除文件 + * + * @param filePOs 要删除的文件列表 + */ + void deleteFile(List filePOs); + + /** + * 保存文件信息 + * + * @param token + * @param fileName + * @param uploadPath + * @param uploadFileName + * @param fileType + * @param fileSize + */ + void saveUploadFileInfo(String token, String fileName, String uploadPath, String uploadFileName, String fileType, long fileSize, Map params); + + /** + * 获取上传文件路径 + * + * @param baseUploadPath + * @param uploadTypeEnum + * @param fileType + * @return + * @throws FileException + */ + String getUploadPath(String baseUploadPath, UploadTypeEnum uploadTypeEnum, String fileType) throws FileException; + + /** + * 下载文件 + * + * @param request + * @param response + * @param params + * @param canRange + */ + void downLoadFile(HttpServletRequest request, HttpServletResponse response, Map params, boolean canRange); + + /** + * 下载文件 + * + * @param request + * @param response + * @param filePO + * @param isOpen + * @param canRange + */ + void downLoadFile(HttpServletRequest request, HttpServletResponse response, FilePO filePO, boolean isOpen, boolean canRange); +} diff --git a/module-file/src/main/java/ink/wgink/module/file/service/IFileService.java b/module-file/src/main/java/ink/wgink/module/file/service/IFileService.java index e2238aa9..9315c4a5 100644 --- a/module-file/src/main/java/ink/wgink/module/file/service/IFileService.java +++ b/module-file/src/main/java/ink/wgink/module/file/service/IFileService.java @@ -74,6 +74,14 @@ public interface IFileService { * uEditor 文件列表 */ String UEDITOR_LIST_FILE = "listfile"; + /** + * 文件MD5值开头 + */ + public static final String FILE_MD5_PREFIX = "MD5:"; + /** + * 文件引用值开头 + */ + public static final String FILE_REF_PREFIX = "REF:"; /** * 保存文件 @@ -236,6 +244,14 @@ public interface IFileService { */ List list(List idList); + /** + * 通过ID列表获取文件列表 + * + * @param idList + * @return + */ + List listPO(List idList); + /** * 校验视频长度是否符合 * diff --git a/module-file/src/main/java/ink/wgink/module/file/service/IMinIoFileService.java b/module-file/src/main/java/ink/wgink/module/file/service/IMinIoFileService.java new file mode 100644 index 00000000..77fab261 --- /dev/null +++ b/module-file/src/main/java/ink/wgink/module/file/service/IMinIoFileService.java @@ -0,0 +1,49 @@ +package ink.wgink.module.file.service; + +import ink.wgink.module.file.enums.UploadTypeEnum; +import ink.wgink.pojo.pos.FilePO; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.List; +import java.util.Map; + +/** + * @ClassName: IMinIOService + * @Description: minIO + * @Author: wanggeng + * @Date: 2021/10/17 10:15 上午 + * @Version: 1.0 + */ +public interface IMinIoFileService { + + /** + * 物理删除文件 + * + * @param filePOs 要删除的文件列表 + */ + void deleteFile(List filePOs); + + /** + * 上传文件 + * + * @param token + * @param uploadFile + * @param uploadTypeEnum + * @param params + */ + void uploadFile(String token, MultipartFile uploadFile, UploadTypeEnum uploadTypeEnum, Map params); + + /** + * 下载文件 + * + * @param request + * @param response + * @param filePO + * @param isOpen + * @param canRange + */ + void downLoadFile(HttpServletRequest request, HttpServletResponse response, FilePO filePO, boolean isOpen, boolean canRange); + +} diff --git a/module-file/src/main/java/ink/wgink/module/file/service/impl/DefaultFileServiceImpl.java b/module-file/src/main/java/ink/wgink/module/file/service/impl/DefaultFileServiceImpl.java new file mode 100644 index 00000000..d93d5cdf --- /dev/null +++ b/module-file/src/main/java/ink/wgink/module/file/service/impl/DefaultFileServiceImpl.java @@ -0,0 +1,508 @@ +package ink.wgink.module.file.service.impl; + +import ink.wgink.common.base.DefaultBaseService; +import ink.wgink.exceptions.FileException; +import ink.wgink.exceptions.SaveException; +import ink.wgink.exceptions.base.SystemException; +import ink.wgink.module.file.dao.IFileDao; +import ink.wgink.module.file.enums.UploadTypeEnum; +import ink.wgink.module.file.pojo.dtos.FileInfoDTO; +import ink.wgink.module.file.service.IDefaultFileService; +import ink.wgink.module.file.service.IFileService; +import ink.wgink.pojo.pos.FilePO; +import ink.wgink.properties.FileProperties; +import ink.wgink.util.UUIDUtil; +import ink.wgink.util.date.DateUtil; +import ink.wgink.util.request.StaticResourceRequestUtil; +import net.coobird.thumbnailator.Thumbnails; +import org.apache.catalina.connector.ClientAbortException; +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.*; +import java.net.URLEncoder; +import java.nio.channels.Channels; +import java.nio.channels.FileChannel; +import java.nio.channels.WritableByteChannel; +import java.security.MessageDigest; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * @ClassName: DefaultFileServiceImpl + * @Description: 默认文件处理 + * @Author: wanggeng + * @Date: 2021/10/17 10:22 上午 + * @Version: 1.0 + */ +@Service +public class DefaultFileServiceImpl extends DefaultBaseService implements IDefaultFileService { + + private static final char[] HEX_CODE = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; + + @Autowired + private IFileDao fileDao; + @Autowired + private FileProperties fileProperties; + private String[] imageTypes; + private String[] videoTypes; + private String[] audioTypes; + private String[] fileTypes; + + /** + * 上传文件 + * + * @param token + * @param uploadFile + * @param uploadTypeEnum + * @param params + * @throws SystemException + */ + @Override + public void uploadFile(String token, MultipartFile uploadFile, UploadTypeEnum uploadTypeEnum, Map params) throws SystemException { + String baseUploadPath = fileProperties.getUploadPath(); + if (StringUtils.isBlank(baseUploadPath)) { + throw new SystemException("上传路径未配置"); + } + String fileName = uploadFile.getOriginalFilename(); + // 文件大小 + long fileSize = uploadFile.getSize(); + // 文件类型 + String fileType = getFileType(fileName); + // 文件保存路径 + String uploadPath = getUploadPath(baseUploadPath, uploadTypeEnum, fileType); + // 文件保存名称 + String uploadFileName = getUploadFileName(fileType); + String fileMd5 = uploadFile(uploadFile, uploadPath, uploadFileName); + if (fileMd5 == null) { + throw new SaveException("文件上传失败"); + } + // 获取MD5相同的文件 + List fileInfoDTOs = fileDao.listByMd5(IFileService.FILE_MD5_PREFIX + fileMd5); + if (fileInfoDTOs.size() > 0) { + // 删除新增的文件 + File uploadedFile = new File(uploadPath + File.separator + uploadFileName); + if (uploadedFile.exists()) { + uploadedFile.delete(); + } + // 保存记录，但文件信息都是原有的文件 + params.clear(); + FileInfoDTO fileInfoDTO = fileInfoDTOs.get(0); + params.put("fileSummary", "REF:" + fileInfoDTO.getFileId()); + saveFile(token, params, fileInfoDTO.getFileName(), fileInfoDTO.getFilePath(), fileInfoDTO.getFileUrl(), fileInfoDTO.getFileType(), fileInfoDTO.getFileSize()); + return; + } + params.put("fileSummary", "MD5:" + fileMd5); + saveUploadFileInfo(token, fileName, uploadPath, uploadFileName, fileType, fileSize, params); + } + + @Override + public void deleteFile(List filePOs) { + Map fileParams = getHashMap(4); + // 删除文件 + for (FilePO filePO : filePOs) { + // 如果文件描述为空，可以直接删除源文件 + if (StringUtils.isBlank(filePO.getFileSummary())) { + deleteSourceFile(filePO.getFilePath()); + continue; + } + // 文件描述不为空时，需要判断是否删除的是源文件，源文件在一个系统中只保留一份 + // 如果是引用文件的数据，不删除源文件 + if (filePO.getFileSummary().startsWith(IFileService.FILE_REF_PREFIX)) { + continue; + } + // 如果不是MD5源文件，略过 + if (!filePO.getFileSummary().startsWith(IFileService.FILE_MD5_PREFIX)) { + continue; + } + // 如果删除的是源文件，需要查询系统中是否还存在引用的数据 + List fileInfoDTOs = fileDao.listByMd5(IFileService.FILE_REF_PREFIX + filePO.getFileId()); + // 如果不存在对源文件引用的数据，则直接删除源文件 + if (fileInfoDTOs.size() == 0) { + deleteSourceFile(filePO.getFilePath()); + continue; + } + fileParams.clear(); + // 如果存在引用数据，取出第一个修改为源文件，并将其他的引用更新为新的源文件ID + FileInfoDTO firstFileInfoDTO = fileInfoDTOs.get(0); + fileParams.put("fileSummary", firstFileInfoDTO.getFileSummary()); + fileParams.put("fileId", firstFileInfoDTO.getFileId()); + fileDao.updateSummary(fileParams); + // 获取其他的ID列表，更新文件引用关系 + List otherFileIds = new ArrayList<>(); + for (int i = 1; i < fileInfoDTOs.size(); i++) { + otherFileIds.add(fileInfoDTOs.get(i).getFileId()); + } + // 如果不存在其它的引用，略过 + if (otherFileIds.isEmpty()) { + continue; + } + fileParams.remove("fileId"); + fileParams.put("fileSummary", IFileService.FILE_REF_PREFIX + firstFileInfoDTO.getFileId()); + fileParams.put("fileIds", otherFileIds); + fileDao.updateSummary(fileParams); + } + } + + /** + * 删除源文件 + * + * @param sourceFilePath 源文件路径 + */ + private void deleteSourceFile(String sourceFilePath) { + File file = new File(sourceFilePath); + if (file.exists()) { + boolean isDelete = file.delete(); + if (isDelete) { + LOG.debug("文件删除成功"); + } else { + LOG.debug("文件删除失败"); + } + } + } + + /** + * 保存文件 + * + * @param token token + * @param params 参数 + * @param fileName 文件名 + * @param fileFullPath 文件全路径 + * @param fileUrl 文件相对地址 + * @param fileType 文件类型 + * @param fileSize 文件大小 + */ + private void saveFile(String token, Map params, String fileName, String fileFullPath, String fileUrl, String fileType, long fileSize) { + params.put("fileId", UUIDUtil.getUUID()); + params.put("fileName", fileName); + params.put("filePath", fileFullPath); + params.put("fileUrl", fileUrl); + params.put("fileType", fileType); + params.put("fileSize", fileSize); + params.put("isBack", 0); + if (StringUtils.isBlank(token)) { + setSaveInfo(params); + } else { + setAppSaveInfo(token, params); + } + fileDao.save(params); + } + + @Override + public void saveUploadFileInfo(String token, String fileName, String uploadPath, String uploadFileName, String fileType, long fileSize, Map params) { + String fixPath = uploadPath.replace(fileProperties.getUploadPath(), ""); + if ("\\".equals(File.separator)) { + fixPath = fixPath.replace("\\", "/"); + } + if (fixPath.startsWith("/")) { + fixPath = fixPath.substring(1, fixPath.length() - 1); + } + String fileFullPath = String.format("%s%s%s", uploadPath, File.separator, uploadFileName); + // 压缩图片 + if (isImageFile(fileType)) { + compressImage(fileFullPath); + File photo = new File(fileFullPath); + fileSize = photo.length(); + } + saveFile(token, params, fileName, fileFullPath, String.format("files/%s/%s", fixPath, uploadFileName), fileType, fileSize); + } + + @Override + public void downLoadFile(HttpServletRequest request, HttpServletResponse response, Map params, boolean canRange) { + FilePO filePO = fileDao.getPO(params); + boolean isOpen = Boolean.valueOf(params.get("isOpen").toString()); + downLoadFile(request, response, filePO, isOpen, canRange); + } + + @Override + public void downLoadFile(HttpServletRequest request, HttpServletResponse response, FilePO filePO, boolean isOpen, boolean canRange) { + try ( + RandomAccessFile randomAccessFile = new RandomAccessFile(filePO.getFilePath(), "r"); + FileChannel fileChannel = randomAccessFile.getChannel(); + OutputStream outputStream = response.getOutputStream(); + WritableByteChannel writableByteChannel = Channels.newChannel(outputStream); + ) { + response.setHeader("Content-Length", filePO.getFileSize()); + response.setContentType(StaticResourceRequestUtil.getContentType(filePO.getFileType())); + if (!isOpen) { + // 下载 + response.setHeader("Content-Disposition", "attachment;fileName=" + URLEncoder.encode(filePO.getFileName(), "UTF-8")); + } else { + // 直接打开 + response.setHeader("Content-Disposition", "inline;fileName=" + URLEncoder.encode(filePO.getFileName(), "UTF-8")); + // 如果是图片资源，开启缓存 + if (isImageFile(filePO.getFileType())) { + // 如果存在校验修改时间且未做修改，返回304，使用本地资源 + String ifModifiedSince = request.getHeader("If-Modified-Since"); + if (StringUtils.isNotBlank(ifModifiedSince) && StringUtils.equalsIgnoreCase(ifModifiedSince, filePO.getGmtModified())) { + response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + return; + } + // 缓存有效时间为7天 + response.setHeader("Expires", DateTime.now().plusDays(7).toDateTime(DateTimeZone.forID("GMT")).toString()); + // 一小时之内不发送新请求 + response.setHeader("max-age", "3600"); + response.setHeader("Last-Modified", filePO.getGmtModified()); + } + } + String rangeString = null; + if (canRange && request != null) { + rangeString = request.getHeader("Range"); + LOG.debug("range: {}", rangeString); + } + long contentLength = Long.valueOf(filePO.getFileSize()); + long startRange = 0; + long endRange = contentLength - 1; + if (!StringUtils.isBlank(rangeString)) { + if (!isOpen) { + response.setContentType("multipart/byteranges"); + } + String[] rangeArray = rangeString.substring(rangeString.indexOf("=") + 1).split("-"); + startRange = Long.valueOf(rangeArray[0]); + if (rangeArray.length > 1) { + endRange = Long.valueOf(rangeArray[1]); + } + setRangeHeader(startRange, endRange, response, filePO.getFileId(), contentLength); + randomAccessFile.seek(startRange); + } + LOG.debug("startRange: {}, endRange: {}", startRange, endRange); + long totalOutputLength = endRange - startRange + 1; + fileChannel.transferTo(startRange, totalOutputLength, writableByteChannel); + outputStream.flush(); + } catch (Exception e) { + if (e instanceof ClientAbortException) { + LOG.debug("客户端断开连接"); + } else { + + throw new FileException("文件输出异常", e); + } + } + } + + /** + * 处理视频流问题 + * + * @param startRange + * @param endRange + * @param response + * @param fileId + * @param contentLength + */ + private void setRangeHeader(long startRange, long endRange, HttpServletResponse response, String fileId, long contentLength) { + // 这里不设置，会出现第一次加载很慢的情况 + response.setHeader("Content-Length", String.valueOf(endRange - startRange + 1)); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", startRange, endRange, contentLength)); + response.setHeader("Accept-Ranges", "bytes"); + response.setHeader("Etag", fileId); + response.setStatus(206); + } + + /** + * 是否是图片文件, GIF图片不压缩 + * + * @param fileType + * @return + */ + private boolean isImageFile(String fileType) { + if (StringUtils.equalsIgnoreCase("gif", fileType)) { + return false; + } + String imageTypes = fileProperties.getImageTypes(); + for (String imageType : imageTypes.split(",")) { + if (StringUtils.equalsIgnoreCase(fileType, imageType)) { + return true; + } + } + return false; + } + + /** + * 压缩图片 + * + * @param fileFullPath + */ + private void compressImage(String fileFullPath) { + if (fileFullPath.endsWith(".blob")) { + return; + } + try { + Thumbnails.of(fileFullPath).scale(1.0f).outputQuality(fileProperties.getImageOutputQuality()).toFile(fileFullPath); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + + /** + * 获取文件类型 + * + * @param fileName + * @return + */ + private String getFileType(String fileName) { + String[] names = fileName.split("\\."); + if (names != null) { + return names[names.length - 1].toLowerCase(); + } + return ""; + } + + public String[] getImageTypes() { + return imageTypes == null ? fileProperties.getImageTypes().split(",") : imageTypes; + } + + public void setImageTypes(String[] imageTypes) { + this.imageTypes = imageTypes; + } + + public String[] getVideoTypes() { + return videoTypes == null ? fileProperties.getVideoTypes().split(",") : videoTypes; + } + + public void setVideoTypes(String[] videoTypes) { + this.videoTypes = videoTypes; + } + + public String[] getAudioTypes() { + return audioTypes == null ? fileProperties.getAudioTypes().split(",") : audioTypes; + } + + public void setAudioTypes(String[] audioTypes) { + this.audioTypes = audioTypes; + } + + public String[] getFileTypes() { + return fileTypes == null ? fileProperties.getFileTypes().split(",") : fileTypes; + } + + public void setFileTypes(String[] fileTypes) { + this.fileTypes = fileTypes; + } + + + /** + * 校验类型 + * + * @param types + * @param fileType + * @return + */ + private boolean isTypeCorrect(String[] types, String fileType) { + for (String type : types) { + if (StringUtils.equalsIgnoreCase(fileType, type)) { + return true; + } + } + return false; + } + + @Override + public String getUploadPath(String baseUploadPath, UploadTypeEnum uploadTypeEnum, String fileType) throws FileException { + StringBuilder filePath = new StringBuilder(); + if (!baseUploadPath.endsWith(File.separator)) { + filePath.append(baseUploadPath).append(File.separator); + } else { + filePath.append(baseUploadPath); + } + boolean hasFileType = !StringUtils.isBlank(fileType); + if (uploadTypeEnum.getValue() == UploadTypeEnum.IMAGE.getValue()) { + if (hasFileType && !isTypeCorrect(getImageTypes(), fileType)) { + throw new FileException("图片格式不支持上传"); + } + filePath.append("images"); + } else if (uploadTypeEnum.getValue() == UploadTypeEnum.VIDEO.getValue()) { + if (hasFileType && !isTypeCorrect(getVideoTypes(), fileType)) { + throw new FileException("视频格式不支持上传"); + } + filePath.append("videos"); + } else if (uploadTypeEnum.getValue() == UploadTypeEnum.AUDIO.getValue()) { + if (hasFileType && !isTypeCorrect(getAudioTypes(), fileType)) { + throw new FileException("音频格式不支持上传"); + } + filePath.append("audios"); + } else if (uploadTypeEnum.getValue() == UploadTypeEnum.ERROR_EXCEL.getValue()) { + filePath.append("errorexcels"); + } else { + if (hasFileType && !isTypeCorrect(getFileTypes(), fileType)) { + throw new FileException("文件格式不支持上传"); + } + filePath.append("files"); + } + filePath.append(File.separator).append(DateUtil.getDays()); + return filePath.toString(); + } + + /** + * 获取上传文件名称 + * + * @param fileType + * @return + */ + private String getUploadFileName(String fileType) { + String uploadFileName = UUIDUtil.get32UUID(); + if (!StringUtils.isEmpty(fileType)) { + uploadFileName += "." + fileType; + } + return uploadFileName; + } + + /** + * 保存文件到本地 + * + * @param uploadFile + * @param filePath + * @param uploadFileName + * @return + */ + private String uploadFile(MultipartFile uploadFile, String filePath, String uploadFileName) throws FileException { + String fileMd5; + File uploadFolder = new File(filePath); + if (!uploadFolder.exists()) { + uploadFolder.mkdirs(); + } + InputStream uploadFileInputStream = null; + FileOutputStream uploadFileOutputStream = null; + try { + uploadFileInputStream = uploadFile.getInputStream(); + uploadFileOutputStream = new FileOutputStream(new File(uploadFolder + "/" + uploadFileName)); + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + for (byte[] buf = new byte[IFileService.INPUT_STREAM_SIZE]; uploadFileInputStream.read(buf) > -1; ) { + uploadFileOutputStream.write(buf, 0, buf.length); + messageDigest.update(buf, 0, buf.length); + } + uploadFileOutputStream.flush(); + // 计算文件的MD5 + byte[] data = messageDigest.digest(); + StringBuilder fileMd5SB = new StringBuilder(data.length * 2); + for (byte b : data) { + fileMd5SB.append(HEX_CODE[(b >> 4) & 0xF]); + fileMd5SB.append(HEX_CODE[(b & 0xF)]); + } + fileMd5 = fileMd5SB.toString(); + } catch (Exception e) { + LOG.error(e.getMessage(), e); + throw new FileException("文件上传失败"); + } finally { + try { + if (null != uploadFileOutputStream) { + uploadFileOutputStream.close(); + } + if (null != uploadFileInputStream) { + uploadFileInputStream.close(); + } + } catch (Exception e1) { + LOG.error(e1.getMessage()); + throw new FileException("文件上传失败"); + } + } + return fileMd5; + } + + +} diff --git a/module-file/src/main/java/ink/wgink/module/file/service/impl/FileServiceImpl.java b/module-file/src/main/java/ink/wgink/module/file/service/impl/FileServiceImpl.java index ab46bf4f..f86ae793 100644 --- a/module-file/src/main/java/ink/wgink/module/file/service/impl/FileServiceImpl.java +++ b/module-file/src/main/java/ink/wgink/module/file/service/impl/FileServiceImpl.java @@ -7,7 +7,6 @@ import com.github.pagehelper.PageInfo; import ink.wgink.common.base.DefaultBaseService; import ink.wgink.exceptions.FileException; import ink.wgink.exceptions.ParamsException; -import ink.wgink.exceptions.SaveException; import ink.wgink.exceptions.SearchException; import ink.wgink.exceptions.base.SystemException; import ink.wgink.module.file.dao.IFileDao; @@ -15,7 +14,9 @@ import ink.wgink.module.file.enums.UploadTypeEnum; import ink.wgink.module.file.pojo.dtos.FileDTO; import ink.wgink.module.file.pojo.dtos.FileInfoDTO; import ink.wgink.module.file.pojo.vos.FileVO; +import ink.wgink.module.file.service.IDefaultFileService; import ink.wgink.module.file.service.IFileService; +import ink.wgink.module.file.service.IMinIoFileService; import ink.wgink.pojo.ListPage; import ink.wgink.pojo.pos.FilePO; import ink.wgink.pojo.result.ErrorResult; @@ -23,18 +24,12 @@ import ink.wgink.pojo.result.SuccessResultList; import ink.wgink.properties.FileProperties; import ink.wgink.util.ResourceUtil; import ink.wgink.util.UUIDUtil; -import ink.wgink.util.date.DateUtil; import ink.wgink.util.map.HashMapUtil; -import ink.wgink.util.request.StaticResourceRequestUtil; import it.sauronsoftware.jave.Encoder; import it.sauronsoftware.jave.EncoderException; import it.sauronsoftware.jave.MultimediaInfo; -import net.coobird.thumbnailator.Thumbnails; -import org.apache.catalina.connector.ClientAbortException; import org.apache.commons.fileupload.disk.DiskFileItem; import org.apache.commons.lang3.StringUtils; -import org.joda.time.DateTime; -import org.joda.time.DateTimeZone; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -44,12 +39,8 @@ import org.springframework.web.multipart.commons.CommonsMultipartFile; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.io.*; -import java.net.URLEncoder; -import java.nio.channels.Channels; -import java.nio.channels.FileChannel; -import java.nio.channels.WritableByteChannel; -import java.security.MessageDigest; +import java.io.File; +import java.io.IOException; import java.util.*; /** @@ -63,25 +54,16 @@ import java.util.*; public class FileServiceImpl extends DefaultBaseService implements IFileService { private static final Logger LOG = LoggerFactory.getLogger(FileServiceImpl.class); - private static final char[] HEX_CODE = {'0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; - /** - * 文件MD5值开头 - */ - public static final String FILE_MD5_PREFIX = "MD5:"; - /** - * 文件引用值开头 - */ - public static final String FILE_REF_PREFIX = "REF:"; + @Autowired private FileProperties fileProperties; @Autowired private IFileDao fileDao; - - private String[] imageTypes; - private String[] videoTypes; - private String[] audioTypes; - private String[] fileTypes; + @Autowired + private IDefaultFileService defaultFileService; + @Autowired + private IMinIoFileService minIoFileService; public List listFileInfo(Map params) { return fileDao.listInfo(params); @@ -117,73 +99,17 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService public void delete(List ids) { Map params = getHashMap(2); params.put("fileIds", ids); - Map fileParams = getHashMap(4); - List fileInfoWithPathDTOs = fileDao.listInfo(params); - // 删除文件 - for (FileInfoDTO fileInfoDTO : fileInfoWithPathDTOs) { - // 如果文件描述为空，可以直接删除源文件 - if (StringUtils.isBlank(fileInfoDTO.getFileSummary())) { - deleteSourceFile(fileInfoDTO.getFilePath()); - continue; - } - // 文件描述不为空时，需要判断是否删除的是源文件，源文件在一个系统中只保留一份 - // 如果是引用文件的数据，不删除源文件 - if (fileInfoDTO.getFileSummary().startsWith(FILE_REF_PREFIX)) { - continue; - } - // 如果不是MD5源文件，略过 - if (!fileInfoDTO.getFileSummary().startsWith(FILE_MD5_PREFIX)) { - continue; - } - // 如果删除的是源文件，需要查询系统中是否还存在引用的数据 - List fileInfoDTOs = fileDao.listByMd5(FILE_REF_PREFIX + fileInfoDTO.getFileId()); - // 如果不存在对源文件引用的数据，则直接删除源文件 - if (fileInfoDTOs.size() == 0) { - deleteSourceFile(fileInfoDTO.getFilePath()); - continue; - } - fileParams.clear(); - // 如果存在引用数据，取出第一个修改为源文件，并将其他的引用更新为新的源文件ID - FileInfoDTO firstFileInfoDTO = fileInfoDTOs.get(0); - fileParams.put("fileSummary", firstFileInfoDTO.getFileSummary()); - fileParams.put("fileId", firstFileInfoDTO.getFileId()); - fileDao.updateSummary(fileParams); - // 获取其他的ID列表，更新文件引用关系 - List otherFileIds = new ArrayList<>(); - for (int i = 1; i < fileInfoDTOs.size(); i++) { - otherFileIds.add(fileInfoDTOs.get(i).getFileId()); - } - // 如果不存在其它的引用，略过 - if (otherFileIds.isEmpty()) { - continue; - } - fileParams.remove("fileId"); - fileParams.put("fileSummary", FILE_REF_PREFIX + firstFileInfoDTO.getFileId()); - fileParams.put("fileIds", otherFileIds); - fileDao.updateSummary(fileParams); + List filePOs = listPO(ids); + if (fileProperties.getUseMinIo()) { + minIoFileService.deleteFile(filePOs); + } else { + defaultFileService.deleteFile(filePOs); } // 删除记录 fileDao.delete(params); } - /** - * 删除源文件 - * - * @param sourceFilePath 源文件路径 - */ - private void deleteSourceFile(String sourceFilePath) { - File file = new File(sourceFilePath); - if (file.exists()) { - boolean isDelete = file.delete(); - if (isDelete) { - LOG.debug("文件删除成功"); - } else { - LOG.debug("文件删除失败"); - } - } - } - @Override public String uploadSingle(MultipartFile uploadFile, UploadTypeEnum uploadTypeEnum, Map params) throws SystemException { uploadFile(null, uploadFile, uploadTypeEnum, params); @@ -217,45 +143,16 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService * @throws SystemException */ private void uploadFile(String token, MultipartFile uploadFile, UploadTypeEnum uploadTypeEnum, Map params) throws SystemException { - String baseUploadPath = fileProperties.getUploadPath(); - if (StringUtils.isBlank(baseUploadPath)) { - throw new SystemException("上传路径未配置"); + if (fileProperties.getUseMinIo()) { + minIoFileService.uploadFile(token, uploadFile, uploadTypeEnum, params); + } else { + defaultFileService.uploadFile(token, uploadFile, uploadTypeEnum, params); } - String fileName = uploadFile.getOriginalFilename(); - // 文件大小 - long fileSize = uploadFile.getSize(); - // 文件类型 - String fileType = getFileType(fileName); - // 文件保存路径 - String uploadPath = getUploadPath(baseUploadPath, uploadTypeEnum, fileType); - // 文件保存名称 - String uploadFileName = getUploadFileName(fileType); - String fileMd5 = uploadFile(uploadFile, uploadPath, uploadFileName); - if (fileMd5 == null) { - throw new SaveException("文件上传失败"); - } - // 获取MD5相同的文件 - List fileInfoDTOs = fileDao.listByMd5(FILE_MD5_PREFIX + fileMd5); - if (fileInfoDTOs.size() > 0) { - // 删除新增的文件 - File uploadedFile = new File(uploadPath + File.separator + uploadFileName); - if (uploadedFile.exists()) { - uploadedFile.delete(); - } - // 保存记录，但文件信息都是原有的文件 - params.clear(); - FileInfoDTO fileInfoDTO = fileInfoDTOs.get(0); - params.put("fileSummary", "REF:" + fileInfoDTO.getFileId()); - saveFile(token, params, fileInfoDTO.getFileName(), fileInfoDTO.getFilePath(), fileInfoDTO.getFileUrl(), fileInfoDTO.getFileType(), fileInfoDTO.getFileSize()); - return; - } - params.put("fileSummary", "MD5:" + fileMd5); - saveUploadFileInfo(token, fileName, uploadPath, uploadFileName, fileType, fileSize, params); } @Override public void uploadErrorExcelFileInfo(String fileName, String uploadPath, long fileSize, Map params) { - saveUploadFileInfo(null, fileName, uploadPath, fileName, "xls", fileSize, params); + defaultFileService.saveUploadFileInfo(null, fileName, uploadPath, fileName, "xls", fileSize, params); } @Override @@ -264,7 +161,7 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService if (StringUtils.isBlank(baseUploadPath)) { throw new SystemException("上传路径未配置"); } - return getUploadPath(baseUploadPath, UploadTypeEnum.ERROR_EXCEL, null); + return defaultFileService.getUploadPath(baseUploadPath, UploadTypeEnum.ERROR_EXCEL, null); } @Override @@ -274,32 +171,11 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService return fileDao.list(params); } - /** - * 保存文件信息 - * - * @param token - * @param fileName - * @param uploadPath - * @param uploadFileName - * @param fileType - * @param fileSize - */ - private void saveUploadFileInfo(String token, String fileName, String uploadPath, String uploadFileName, String fileType, long fileSize, Map params) { - String fixPath = uploadPath.replace(fileProperties.getUploadPath(), ""); - if ("\\".equals(File.separator)) { - fixPath = fixPath.replace("\\", "/"); - } - if (fixPath.startsWith("/")) { - fixPath = fixPath.substring(1, fixPath.length() - 1); - } - String fileFullPath = String.format("%s%s%s", uploadPath, File.separator, uploadFileName); - // 压缩图片 - if (isImageFile(fileType)) { - compressImage(fileFullPath); - File photo = new File(fileFullPath); - fileSize = photo.length(); - } - saveFile(token, params, fileName, fileFullPath, String.format("files/%s/%s", fixPath, uploadFileName), fileType, fileSize); + @Override + public List listPO(List idList) { + Map params = getHashMap(2); + params.put("fileIds", idList); + return fileDao.listPO(params); } @Override @@ -313,32 +189,6 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService return fileId; } - /** - * 保存文件 - * - * @param token token - * @param params 参数 - * @param fileName 文件名 - * @param fileFullPath 文件全路径 - * @param fileUrl 文件相对地址 - * @param fileType 文件类型 - * @param fileSize 文件大小 - */ - private void saveFile(String token, Map params, String fileName, String fileFullPath, String fileUrl, String fileType, long fileSize) { - params.put("fileId", UUIDUtil.getUUID()); - params.put("fileName", fileName); - params.put("filePath", fileFullPath); - params.put("fileUrl", fileUrl); - params.put("fileType", fileType); - params.put("fileSize", fileSize); - params.put("isBack", 0); - if (StringUtils.isBlank(token)) { - setSaveInfo(params); - } else { - setAppSaveInfo(token, params); - } - fileDao.save(params); - } @Override public void downLoadFile(HttpServletResponse response, Map params) { @@ -354,85 +204,18 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService public void downLoadFile(HttpServletRequest request, HttpServletResponse response, Map params, boolean canRange) { FilePO filePO = fileDao.getPO(params); if (null == filePO) { - throw new SearchException("文件获取失败"); + throw new FileException("文件不存在"); } - try ( - RandomAccessFile randomAccessFile = new RandomAccessFile(filePO.getFilePath(), "r"); - FileChannel fileChannel = randomAccessFile.getChannel(); - OutputStream outputStream = response.getOutputStream(); - WritableByteChannel writableByteChannel = Channels.newChannel(outputStream); - ) { - boolean isOpen = Boolean.valueOf(params.get("isOpen").toString()); - response.setHeader("Content-Length", filePO.getFileSize()); - response.setContentType(StaticResourceRequestUtil.getContentType(filePO.getFileType())); - if (!isOpen) { - response.setHeader("Content-Disposition", "attachment;fileName=" + URLEncoder.encode(filePO.getFileName(), "UTF-8")); - } else { - response.setHeader("Content-Disposition", "inline;fileName=" + URLEncoder.encode(filePO.getFileName(), "UTF-8")); - // 如果是图片资源，开启缓存 - if (isImageFile(filePO.getFileType())) { - // 如果存在校验修改时间且未做修改，返回304，使用本地资源 - String ifModifiedSince = request.getHeader("If-Modified-Since"); - if (StringUtils.isNotBlank(ifModifiedSince) && StringUtils.equalsIgnoreCase(ifModifiedSince, filePO.getGmtModified())) { - response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); - return; - } - // 缓存有效时间为7天 - response.setHeader("Expires", DateTime.now().plusDays(7).toDateTime(DateTimeZone.forID("GMT")).toString()); - // 一小时之内不发送新请求 - response.setHeader("max-age", "3600"); - response.setHeader("Last-Modified", filePO.getGmtModified()); - } - } - String rangeString = null; - if (canRange && request != null) { - rangeString = request.getHeader("Range"); - LOG.debug("range: {}", rangeString); - } - long contentLength = Long.valueOf(filePO.getFileSize()); - long startRange = 0; - long endRange = contentLength - 1; - if (!StringUtils.isBlank(rangeString)) { - if (!isOpen) { - response.setContentType("multipart/byteranges"); - } - String[] rangeArray = rangeString.substring(rangeString.indexOf("=") + 1).split("-"); - startRange = Long.valueOf(rangeArray[0]); - if (rangeArray.length > 1) { - endRange = Long.valueOf(rangeArray[1]); - } - setRangeHeader(startRange, endRange, response, filePO.getFileId(), contentLength); - randomAccessFile.seek(startRange); - } - LOG.debug("startRange: {}, endRange: {}", startRange, endRange); - long totalOutputLength = endRange - startRange + 1; - fileChannel.transferTo(startRange, totalOutputLength, writableByteChannel); - outputStream.flush(); - } catch (Exception e) { - if (e instanceof ClientAbortException) { - LOG.debug("客户端断开连接"); - } else { - throw new FileException("文件输出异常", e); - } + boolean isOpen = Boolean.valueOf(params.get("isOpen").toString()); + // 没有备份的数据从本地下载 + if (filePO.getIsBack() == 0) { + defaultFileService.downLoadFile(request, response, filePO, isOpen, canRange); + return; } - } - - /** - * 处理视频流问题 - * - * @param startRange - * @param endRange - * @param response - * @param fileId - * @param contentLength - */ - private void setRangeHeader(long startRange, long endRange, HttpServletResponse response, String fileId, long contentLength) { - // 这里不设置，会出现第一次加载很慢的情况 - response.setHeader("Content-Length", String.valueOf(endRange - startRange + 1)); - response.setHeader("Content-Range", String.format("bytes %d-%d/%d", startRange, endRange, contentLength)); - response.setHeader("Accept-Ranges", "bytes"); - response.setHeader("Etag", fileId); - response.setStatus(206); + if (fileProperties.getUseMinIo()) { + minIoFileService.downLoadFile(request, response, filePO, isOpen, canRange); + } + throw new FileException("文件下载异常"); } @Override @@ -598,212 +381,4 @@ public class FileServiceImpl extends DefaultBaseService implements IFileService return JSONObject.parseObject(uEditorConfig); } - - public String[] getImageTypes() { - return imageTypes == null ? fileProperties.getImageTypes().split(",") : imageTypes; - } - - public void setImageTypes(String[] imageTypes) { - this.imageTypes = imageTypes; - } - - public String[] getVideoTypes() { - return videoTypes == null ? fileProperties.getVideoTypes().split(",") : videoTypes; - } - - public void setVideoTypes(String[] videoTypes) { - this.videoTypes = videoTypes; - } - - public String[] getAudioTypes() { - return audioTypes == null ? fileProperties.getAudioTypes().split(",") : audioTypes; - } - - public void setAudioTypes(String[] audioTypes) { - this.audioTypes = audioTypes; - } - - public String[] getFileTypes() { - return fileTypes == null ? fileProperties.getFileTypes().split(",") : fileTypes; - } - - public void setFileTypes(String[] fileTypes) { - this.fileTypes = fileTypes; - } - - /** - * 校验类型 - * - * @param types - * @param fileType - * @return - */ - private boolean isTypeCorrect(String[] types, String fileType) { - for (String type : types) { - if (StringUtils.equalsIgnoreCase(fileType, type)) { - return true; - } - } - return false; - } - - /** - * 获取文件类型 - * - * @param fileName - * @return - */ - private String getFileType(String fileName) { - String[] names = fileName.split("\\."); - if (names != null) { - return names[names.length - 1].toLowerCase(); - } - return ""; - } - - /** - * 获取上传文件路径 - * - * @param baseUploadPath - * @param uploadTypeEnum - * @param fileType - * @return - * @throws FileException - */ - private String getUploadPath(String baseUploadPath, UploadTypeEnum uploadTypeEnum, String fileType) throws FileException { - StringBuilder filePath = new StringBuilder(); - if (!baseUploadPath.endsWith(File.separator)) { - filePath.append(baseUploadPath).append(File.separator); - } else { - filePath.append(baseUploadPath); - } - boolean hasFileType = !StringUtils.isBlank(fileType); - if (uploadTypeEnum.getValue() == UploadTypeEnum.IMAGE.getValue()) { - if (hasFileType && !isTypeCorrect(getImageTypes(), fileType)) { - throw new FileException("图片格式不支持上传"); - } - filePath.append("images"); - } else if (uploadTypeEnum.getValue() == UploadTypeEnum.VIDEO.getValue()) { - if (hasFileType && !isTypeCorrect(getVideoTypes(), fileType)) { - throw new FileException("视频格式不支持上传"); - } - filePath.append("videos"); - } else if (uploadTypeEnum.getValue() == UploadTypeEnum.AUDIO.getValue()) { - if (hasFileType && !isTypeCorrect(getAudioTypes(), fileType)) { - throw new FileException("音频格式不支持上传"); - } - filePath.append("audios"); - } else if (uploadTypeEnum.getValue() == UploadTypeEnum.ERROR_EXCEL.getValue()) { - filePath.append("errorexcels"); - } else { - if (hasFileType && !isTypeCorrect(getFileTypes(), fileType)) { - throw new FileException("文件格式不支持上传"); - } - filePath.append("files"); - } - filePath.append(File.separator).append(DateUtil.getDays()); - return filePath.toString(); - } - - /** - * 获取上传文件名称 - * - * @param fileType - * @return - */ - private String getUploadFileName(String fileType) { - String uploadFileName = UUIDUtil.get32UUID(); - if (!StringUtils.isEmpty(fileType)) { - uploadFileName += "." + fileType; - } - return uploadFileName; - } - - /** - * 保存文件到本地 - * - * @param uploadFile - * @param filePath - * @param uploadFileName - * @return - */ - private String uploadFile(MultipartFile uploadFile, String filePath, String uploadFileName) throws FileException { - String fileMd5; - File uploadFolder = new File(filePath); - if (!uploadFolder.exists()) { - uploadFolder.mkdirs(); - } - InputStream uploadFileInputStream = null; - FileOutputStream uploadFileOutputStream = null; - try { - uploadFileInputStream = uploadFile.getInputStream(); - uploadFileOutputStream = new FileOutputStream(new File(uploadFolder + "/" + uploadFileName)); - MessageDigest messageDigest = MessageDigest.getInstance("MD5"); - for (byte[] buf = new byte[INPUT_STREAM_SIZE]; uploadFileInputStream.read(buf) > -1; ) { - uploadFileOutputStream.write(buf, 0, buf.length); - messageDigest.update(buf, 0, buf.length); - } - uploadFileOutputStream.flush(); - // 计算文件的MD5 - byte[] data = messageDigest.digest(); - StringBuilder fileMd5SB = new StringBuilder(data.length * 2); - for (byte b : data) { - fileMd5SB.append(HEX_CODE[(b >> 4) & 0xF]); - fileMd5SB.append(HEX_CODE[(b & 0xF)]); - } - fileMd5 = fileMd5SB.toString(); - } catch (Exception e) { - LOG.error(e.getMessage(), e); - throw new FileException("文件上传失败"); - } finally { - try { - if (null != uploadFileOutputStream) { - uploadFileOutputStream.close(); - } - if (null != uploadFileInputStream) { - uploadFileInputStream.close(); - } - } catch (Exception e1) { - LOG.error(e1.getMessage()); - throw new FileException("文件上传失败"); - } - } - return fileMd5; - } - - /** - * 是否是图片文件, GIF图片不压缩 - * - * @param fileType - * @return - */ - private boolean isImageFile(String fileType) { - if (StringUtils.equalsIgnoreCase("gif", fileType)) { - return false; - } - String imageTypes = fileProperties.getImageTypes(); - for (String imageType : imageTypes.split(",")) { - if (StringUtils.equalsIgnoreCase(fileType, imageType)) { - return true; - } - } - return false; - } - - /** - * 压缩图片 - * - * @param fileFullPath - */ - private void compressImage(String fileFullPath) { - if (fileFullPath.endsWith(".blob")) { - return; - } - try { - Thumbnails.of(fileFullPath).scale(1.0f).outputQuality(fileProperties.getImageOutputQuality()).toFile(fileFullPath); - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } - } - } diff --git a/module-file/src/main/java/ink/wgink/module/file/service/impl/MinIoFileServiceImpl.java b/module-file/src/main/java/ink/wgink/module/file/service/impl/MinIoFileServiceImpl.java new file mode 100644 index 00000000..49607915 --- /dev/null +++ b/module-file/src/main/java/ink/wgink/module/file/service/impl/MinIoFileServiceImpl.java @@ -0,0 +1,271 @@ +package ink.wgink.module.file.service.impl; + +import ink.wgink.common.base.DefaultBaseService; +import ink.wgink.exceptions.FileException; +import ink.wgink.exceptions.base.SystemException; +import ink.wgink.module.file.dao.IFileDao; +import ink.wgink.module.file.enums.UploadTypeEnum; +import ink.wgink.module.file.service.IFileService; +import ink.wgink.module.file.service.IMinIoFileService; +import ink.wgink.pojo.pos.FilePO; +import ink.wgink.properties.FileMinIoProperties; +import ink.wgink.properties.FileProperties; +import ink.wgink.util.UUIDUtil; +import ink.wgink.util.request.StaticResourceRequestUtil; +import io.minio.MinioClient; +import io.minio.PutObjectOptions; +import io.minio.errors.*; +import org.apache.commons.lang3.StringUtils; +import org.joda.time.DateTime; +import org.joda.time.DateTimeZone; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.web.multipart.MultipartFile; + +import javax.annotation.PostConstruct; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URLEncoder; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.util.List; +import java.util.Map; + +/** + * @ClassName: MinIOServiceImpl + * @Description: minIO + * @Author: wanggeng + * @Date: 2021/10/17 10:16 上午 + * @Version: 1.0 + */ +@Service +public class MinIoFileServiceImpl extends DefaultBaseService implements IMinIoFileService { + + @Autowired + private FileProperties fileProperties; + @Autowired + private IFileDao fileDao; + private FileMinIoProperties minIoProperties; + private MinioClient minioClient; + + + @PostConstruct + public void init() throws InvalidPortException, InvalidEndpointException { + minIoProperties = fileProperties.getMinIo(); + minioClient = new MinioClient(minIoProperties.getEndpoint(), minIoProperties.getAccessKey(), minIoProperties.getSecretKey(), minIoProperties.getSecure()); + } + + @Override + public void deleteFile(List filePOs) { + + } + + @Override + public void uploadFile(String token, MultipartFile uploadFile, UploadTypeEnum uploadTypeEnum, Map params) { + String fileId = UUIDUtil.getUUID(); + params.put("fileId", fileId); + + String fileName = uploadFile.getOriginalFilename(); + // 文件大小 + long fileSize = uploadFile.getSize(); + // 文件类型 + String fileType = getFileType(fileName); + String uploadFileName = fileId + "." + fileType; + String bucketName = getBucketName(uploadTypeEnum); + // 上传文件 + upload(uploadFileName, fileSize, bucketName, uploadFile); + saveFile(token, fileId, fileName, fileType, fileSize, bucketName); + } + + @Override + public void downLoadFile(HttpServletRequest request, HttpServletResponse response, FilePO filePO, boolean isOpen, boolean canRange) { + // minIo中文件名 + String objectName = filePO.getFileId() + "." + filePO.getFileType(); + InputStream inputStream = null; + try (OutputStream outputStream = response.getOutputStream();) { + response.setHeader("Content-Length", filePO.getFileSize()); + response.setContentType(StaticResourceRequestUtil.getContentType(filePO.getFileType())); + if (!isOpen) { + // 下载 + response.setHeader("Content-Disposition", "attachment;fileName=" + URLEncoder.encode(filePO.getFileName(), "UTF-8")); + inputStream = minioClient.getObject(filePO.getFileUrl(), objectName); + } else { + // 直接打开 + response.setHeader("Content-Disposition", "inline;fileName=" + URLEncoder.encode(filePO.getFileName(), "UTF-8")); + // 如果是图片资源，开启缓存 + if (isImageFile(filePO.getFileType())) { + // 如果存在校验修改时间且未做修改，返回304，使用本地资源 + String ifModifiedSince = request.getHeader("If-Modified-Since"); + if (StringUtils.isNotBlank(ifModifiedSince) && StringUtils.equalsIgnoreCase(ifModifiedSince, filePO.getGmtModified())) { + response.setStatus(HttpServletResponse.SC_NOT_MODIFIED); + return; + } + // 缓存有效时间为7天 + response.setHeader("Expires", DateTime.now().plusDays(7).toDateTime(DateTimeZone.forID("GMT")).toString()); + // 一小时之内不发送新请求 + response.setHeader("max-age", "3600"); + response.setHeader("Last-Modified", filePO.getGmtModified()); + } + String rangeString = null; + if (canRange && request != null) { + rangeString = request.getHeader("Range"); + LOG.debug("range: {}", rangeString); + } + long contentLength = Long.valueOf(filePO.getFileSize()); + long startRange = 0; + long endRange = contentLength - 1; + if (!StringUtils.isBlank(rangeString)) { + if (!isOpen) { + response.setContentType("multipart/byteranges"); + } + String[] rangeArray = rangeString.substring(rangeString.indexOf("=") + 1).split("-"); + startRange = Long.valueOf(rangeArray[0]); + if (rangeArray.length > 1) { + endRange = Long.valueOf(rangeArray[1]); + } + setRangeHeader(startRange, endRange, response, filePO.getFileId(), contentLength); + inputStream = minioClient.getObject(filePO.getFileUrl(), objectName, startRange, endRange); + } else { + inputStream = minioClient.getObject(filePO.getFileUrl(), objectName); + } + } + byte[] readBuf = new byte[IFileService.INPUT_STREAM_SIZE]; + for (int length = 0; (length = inputStream.read(readBuf)) > 0; ) { + outputStream.write(readBuf, 0, length); + } + outputStream.flush(); + inputStream.close(); + } catch (IOException | ErrorResponseException | InsufficientDataException | InternalException | InvalidBucketNameException | InvalidKeyException | InvalidResponseException | NoSuchAlgorithmException | XmlParserException e) { + throw new FileException("文件输出异常", e); + } finally { + if (inputStream != null) { + try { + inputStream.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + + + /** + * 上传文件到minIo + * + * @param uploadFileName + * @param fileSize + * @param bucketName + * @param uploadFile + */ + private void upload(String uploadFileName, long fileSize, String bucketName, MultipartFile uploadFile) { + try (InputStream inputStream = uploadFile.getInputStream()) { + // 创建桶 + if (!minioClient.bucketExists(bucketName)) { + minioClient.makeBucket(bucketName); + } + PutObjectOptions putObjectOptions = new PutObjectOptions(fileSize, PutObjectOptions.MAX_PART_SIZE); + minioClient.putObject(bucketName, uploadFileName, inputStream, putObjectOptions); + } catch (MinioException | InvalidKeyException | IOException | NoSuchAlgorithmException e) { + throw new SystemException(e); + } + } + + /** + * 保存文件 + * + * @param token + * @param fileId + * @param fileName + * @param fileType + * @param fileSize + * @param bucketName 桶名称 + */ + private void saveFile(String token, String fileId, String fileName, String fileType, long fileSize, String bucketName) { + Map params = getHashMap(2); + params.put("fileId", fileId); + params.put("fileName", fileName); + params.put("filePath", null); + params.put("fileUrl", bucketName); + params.put("fileType", fileType); + params.put("fileSize", fileSize); + params.put("fileSummary", "minio"); + params.put("isBack", 1); + if (StringUtils.isBlank(token)) { + setSaveInfo(params); + } else { + setAppSaveInfo(token, params); + } + fileDao.save(params); + } + + /** + * 获取文件类型 + * + * @param fileName + * @return + */ + private String getFileType(String fileName) { + String[] names = fileName.split("\\."); + if (names != null) { + return names[names.length - 1].toLowerCase(); + } + return ""; + } + + /** + * 获取桶文件名 + * + * @param uploadTypeEnum 上传类型 + * @return + */ + private String getBucketName(UploadTypeEnum uploadTypeEnum) { + if (uploadTypeEnum.getValue() == UploadTypeEnum.IMAGE.getValue()) { + return "images"; + } else if (uploadTypeEnum.getValue() == UploadTypeEnum.VIDEO.getValue()) { + return "videos"; + } else if (uploadTypeEnum.getValue() == UploadTypeEnum.AUDIO.getValue()) { + return "audios"; + } + return "files"; + } + + /** + * 是否是图片 + * + * @param fileType + * @return + */ + private boolean isImageFile(String fileType) { + if (StringUtils.equalsIgnoreCase("gif", fileType)) { + return false; + } + String imageTypes = fileProperties.getImageTypes(); + for (String imageType : imageTypes.split(",")) { + if (StringUtils.equalsIgnoreCase(fileType, imageType)) { + return true; + } + } + return false; + } + + /** + * 处理视频流问题 + * + * @param startRange + * @param endRange + * @param response + * @param fileId + * @param contentLength + */ + private void setRangeHeader(long startRange, long endRange, HttpServletResponse response, String fileId, long contentLength) { + // 这里不设置，会出现第一次加载很慢的情况 + response.setHeader("Content-Length", String.valueOf(endRange - startRange + 1)); + response.setHeader("Content-Range", String.format("bytes %d-%d/%d", startRange, endRange, contentLength)); + response.setHeader("Accept-Ranges", "bytes"); + response.setHeader("Etag", fileId); + response.setStatus(206); + } +} diff --git a/module-file/src/main/resources/mybatis/mapper/file-mapper.xml b/module-file/src/main/resources/mybatis/mapper/file-mapper.xml index 7a38490c..f5e7d404 100644 --- a/module-file/src/main/resources/mybatis/mapper/file-mapper.xml +++ b/module-file/src/main/resources/mybatis/mapper/file-mapper.xml @@ -277,4 +277,33 @@ file_summary = #{_parameter} + + + \ No newline at end of file diff --git a/module-file/src/test/java/MinIOTest.java b/module-file/src/test/java/MinIOTest.java new file mode 100644 index 00000000..c00feb77 --- /dev/null +++ b/module-file/src/test/java/MinIOTest.java @@ -0,0 +1,47 @@ +import ink.wgink.util.request.StaticResourceRequestUtil; +import io.minio.MinioClient; +import io.minio.PutObjectOptions; +import io.minio.errors.MinioException; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; + +/** + * @ClassName: MinIOTest + * @Description: + * @Author: wanggeng + * @Date: 2021/10/17 12:05 上午 + * @Version: 1.0 + */ +public class MinIOTest { + + public static void main(String[] args) throws Exception { + try { + // 使用MinIO服务的URL，端口，Access key和Secret key创建一个MinioClient对象 + MinioClient minioClient = new MinioClient("http://192.168.0.188:19001", "demotest", "demotest", false); + // 检查存储桶是否已经存在 + boolean isExist = minioClient.bucketExists("images"); + if (isExist) { + System.out.println("Bucket already exists."); + } else { + // 创建一个名为asiatrip的存储桶，用于存储照片的zip文件。 + minioClient.makeBucket("images"); + } + // 使用putObject上传一个文件到存储桶中。 + File file = new File("/Users/wanggeng/Desktop/UploadFiles/images/20210820/1.jpeg"); + InputStream inputStream = new FileInputStream(file); + PutObjectOptions putObjectOptions = new PutObjectOptions(file.length(), PutObjectOptions.MAX_PART_SIZE); + // 下载 + // putObjectOptions.setContentType("application/octet-stream"); + // 直接打开 + String contentType = StaticResourceRequestUtil.getContentType("jpeg"); + putObjectOptions.setContentType(contentType); + minioClient.putObject("images", "demo.jpeg", inputStream, putObjectOptions); + System.out.println("successfully uploaded"); + } catch (MinioException e) { + System.out.println("Error occurred: " + e); + } + } + +} diff --git a/pom.xml b/pom.xml index 2084aff8..0cb6c177 100644 --- a/pom.xml +++ b/pom.xml @@ -89,6 +89,7 @@ 1.7.30 6.0.0 1.10 + 7.0.2 @@ -505,6 +506,14 @@ ${xmlgraphics.version} + + + + io.minio + minio + ${minio.version} + +