文件上传增加S3协议的OSS支持

hsweb-system/hsweb-system-file/pom.xml

@@ -54,6 +54,14 @@
             <artifactId>spring-test</artifactId>
             <scope>test</scope>
         </dependency>
+
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+            <version>${aws.sdk.version}</version>
+            <optional>true</optional>
+        </dependency>
+
     </dependencies>
 
 </project>

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/FileServiceConfiguration.java

@@ -5,13 +5,15 @@
 import org.hswebframework.web.file.web.ReactiveFileController;
 import org.springframework.boot.autoconfigure.AutoConfiguration;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnWebApplication;
 import org.springframework.boot.context.properties.EnableConfigurationProperties;
 import org.springframework.context.annotation.Bean;
-import org.springframework.context.annotation.Configuration;
+
 
 @AutoConfiguration
 @EnableConfigurationProperties(FileUploadProperties.class)
+@ConditionalOnProperty(name = "hsweb.file.storage", havingValue = "local", matchIfMissing = true)
 public class FileServiceConfiguration {
 
 

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/FileUploadProperties.java

@@ -1,5 +1,6 @@
 package org.hswebframework.web.file;
 
+import lombok.Data;
 import lombok.Getter;
 import lombok.Setter;
 import org.apache.commons.collections4.CollectionUtils;
@@ -29,6 +30,8 @@ public class FileUploadProperties {
 
     private String staticLocation = "/static";
 
+    private S3 s3;
+
     //是否使用原始文件名进行存储
     private boolean useOriginalFileName = false;
 
@@ -126,4 +129,14 @@ public static class StaticFileInfo {
         private String relativeLocation;
         private String location;
     }
+
+    @Data
+    public static class S3 {
+        private String endpoint;
+        private String accessKey;
+        private String secretKey;
+        private String region;
+        private String bucket;
+        private String baseUrl;
+    }
 }

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/S3FileStorageConfiguration.java

@@ -0,0 +1,49 @@
+package org.hswebframework.web.file;
+
+import org.hswebframework.web.file.service.FileStorageService;
+import org.hswebframework.web.file.service.S3FileStorageService;
+import org.hswebframework.web.file.web.ReactiveFileController;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+
+import java.net.URI;
+
+@Configuration
+@ConditionalOnClass(S3Client.class)
+@ConditionalOnProperty(name = "hsweb.file.storage", havingValue = "s3", matchIfMissing = false)
+@EnableConfigurationProperties(FileUploadProperties.class)
+public class S3FileStorageConfiguration {
+
+
+    @Bean
+    @ConditionalOnMissingBean
+    public S3Client s3Client(FileUploadProperties properties) {
+        return S3Client.builder()
+                .endpointOverride(URI.create(properties.getS3().getEndpoint()))
+                .credentialsProvider(StaticCredentialsProvider.create(
+                        AwsBasicCredentials.create(properties.getS3().getAccessKey(), properties.getS3().getSecretKey())))
+                .region(Region.of(properties.getS3().getRegion()))
+                .build();
+    }
+
+    @Bean
+    @ConditionalOnMissingBean(FileStorageService.class)
+    public FileStorageService s3FileStorageService(FileUploadProperties properties, S3Client s3Client) {
+        return new S3FileStorageService(properties, s3Client);
+    }
+
+    @Bean
+    @ConditionalOnMissingBean(name = "reactiveFileController")
+    public ReactiveFileController reactiveFileController(FileUploadProperties properties,
+                                                         FileStorageService storageService) {
+        return new ReactiveFileController(properties, storageService);
+    }
+}

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/service/FileStorageService.java

@@ -3,6 +3,7 @@
 import org.springframework.http.codec.multipart.FilePart;
 import reactor.core.publisher.Mono;
 
+import java.io.IOException;
 import java.io.InputStream;
 
 /**
@@ -19,7 +20,7 @@ public interface FileStorageService {
      * @param filePart FilePart
      * @return 文件访问地址
      */
-    Mono<String> saveFile(FilePart filePart);
+    Mono<String> saveFile(FilePart filePart) throws IOException;
 
     /**
      * 使用文件流保存文件,并返回文件地址

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/service/S3FileStorageService.java

@@ -0,0 +1,80 @@
+package org.hswebframework.web.file.service;
+
+import lombok.AllArgsConstructor;
+import lombok.SneakyThrows;
+import org.hswebframework.web.file.FileUploadProperties;
+import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.http.codec.multipart.FilePart;
+import reactor.core.publisher.Mono;
+import reactor.core.scheduler.Schedulers;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.Locale;
+import java.util.UUID;
+
+@AllArgsConstructor
+public class S3FileStorageService implements FileStorageService {
+
+    private final FileUploadProperties properties;
+
+    private final S3Client s3Client;
+
+    @Override
+    public Mono<String> saveFile(FilePart filePart) {
+        String filename = buildFileName(filePart.filename());
+        return DataBufferUtils.join(filePart.content())
+                .publishOn(Schedulers.boundedElastic())
+                .map(dataBuffer -> {
+                    byte[] bytes = new byte[dataBuffer.readableByteCount()];
+                    dataBuffer.read(bytes);
+                    DataBufferUtils.release(dataBuffer);
+                    return new ByteArrayInputStream(bytes);
+                })
+                .map(inputStream -> {
+                    PutObjectRequest request = PutObjectRequest.builder()
+                            .bucket(properties.getS3().getBucket())
+                            .key(filename)
+                            .build();
+
+                    s3Client.putObject(request, RequestBody.fromInputStream(inputStream, inputStream.available()));
+                    return buildFileUrl(filename);
+                });
+    }
+
+
+    @Override
+    @SneakyThrows
+    public Mono<String> saveFile(InputStream inputStream, String fileType) {
+        return Mono.fromCallable(() -> {
+                    String key = UUID.randomUUID().toString() + (fileType.startsWith(".") ? fileType : "." + fileType);
+
+                    PutObjectRequest request = PutObjectRequest.builder()
+                            .bucket(properties.getS3().getBucket())
+                            .key(key)
+                            .build();
+
+                    s3Client.putObject(request, RequestBody.fromInputStream(inputStream, inputStream.available()));
+                    return buildFileUrl(key);
+                })
+                .subscribeOn(Schedulers.boundedElastic());
+    }
+
+    private String buildFileName(String originalName) {
+        String suffix = "";
+        if (originalName != null && originalName.contains(".")) {
+            suffix = originalName.substring(originalName.lastIndexOf("."));
+        }
+        return UUID.randomUUID().toString().replace("-", "") + suffix.toLowerCase(Locale.ROOT);
+    }
+
+    private String buildFileUrl(String key) {
+        if (properties.getS3().getBaseUrl() != null && !properties.getS3().getBaseUrl().isEmpty()) {
+            return properties.getS3().getBaseUrl() + "/" + key;
+        }
+        return "https://" + properties.getS3().getBucket() + "." + properties.getS3().getEndpoint().replace("https://", "").replace("http://", "") + "/" + key;
+    }
+}

hsweb-system/hsweb-system-file/src/main/java/org/hswebframework/web/file/web/ReactiveFileController.java

@@ -11,16 +11,16 @@
 import org.hswebframework.web.authorization.exception.AccessDenyException;
 import org.hswebframework.web.file.FileUploadProperties;
 import org.hswebframework.web.file.service.FileStorageService;
-import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.core.io.buffer.DataBufferUtils;
+import org.springframework.http.MediaType;
 import org.springframework.http.codec.multipart.FilePart;
 import org.springframework.http.codec.multipart.Part;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RequestPart;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.http.server.reactive.ServerHttpRequest;
+import org.springframework.web.bind.annotation.*;
 import reactor.core.publisher.Mono;
 
-import java.io.File;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
 
 @RestController
 @Resource(id = "file", name = "文件上传")
@@ -33,11 +33,13 @@ public class ReactiveFileController {
 
     private final FileStorageService fileStorageService;
 
+
     public ReactiveFileController(FileUploadProperties properties, FileStorageService fileStorageService) {
         this.properties = properties;
         this.fileStorageService = fileStorageService;
     }
 
+
     @PostMapping("/static")
     @SneakyThrows
     @ResourceAction(id = "upload-static", name = "静态文件")
@@ -51,12 +53,60 @@ public Mono<String> uploadStatic(@RequestPart("file")
                         if (properties.denied(filePart.filename(), filePart.headers().getContentType())) {
                            return Mono.error( new AccessDenyException());
                         }
-                        return fileStorageService.saveFile(filePart);
+                        try {
+                            return fileStorageService.saveFile(filePart);
+                        } catch (IOException e) {
+                            throw new RuntimeException(e);
+                        }
                     } else {
                         return Mono.error(() -> new IllegalArgumentException("[file] part is not a file"));
                     }
                 });
 
     }
 
+    @PostMapping("/oss/static")
+    @SneakyThrows
+    @ResourceAction(id = "upload-static", name = "静态文件")
+    @Operation(summary = "上传静态文件")
+    public Mono<String> uploadOssStatic(@RequestPart("file")
+                                     @Parameter(name = "file", description = "文件", style = ParameterStyle.FORM) Mono<Part> partMono) {
+        return partMono
+                .flatMap(part -> {
+                    if (part instanceof FilePart) {
+                        FilePart filePart = ((FilePart) part);
+                        if (properties.denied(filePart.filename(), filePart.headers().getContentType())) {
+                            return Mono.error( new AccessDenyException());
+                        }
+                        try {
+                            return fileStorageService.saveFile(filePart);
+                        } catch (IOException e) {
+                            throw new RuntimeException(e);
+                        }
+                    } else {
+                        return Mono.error(() -> new IllegalArgumentException("[file] part is not a file"));
+                    }
+                });
+
+    }
+
+    @PostMapping(value = "/oss/stream", consumes = MediaType.APPLICATION_OCTET_STREAM_VALUE)
+    @Operation(summary = "上传文件流")
+    public Mono<String> uploadOssStream(ServerHttpRequest request,
+                                     @RequestParam("fileType") String fileType) {
+
+        if (properties.denied("upload." + fileType, MediaType.APPLICATION_OCTET_STREAM)) {
+            return Mono.error(new AccessDenyException());
+        }
+
+        return DataBufferUtils.join(request.getBody())
+                .map(dataBuffer -> {
+                    byte[] bytes = new byte[dataBuffer.readableByteCount()];
+                    dataBuffer.read(bytes);
+                    DataBufferUtils.release(dataBuffer);
+                    return new ByteArrayInputStream(bytes);
+                })
+                .flatMap(inputStream -> fileStorageService.saveFile(inputStream, fileType));
+    }
+
 }

hsweb-system/hsweb-system-file/src/test/java/org/hswebframework/web/file/web/OssUploadTest.java

@@ -0,0 +1,61 @@
+package org.hswebframework.web.file.web;
+
+import org.hswebframework.web.file.S3FileStorageConfiguration;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
+import org.springframework.boot.test.autoconfigure.web.reactive.WebFluxTest;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.http.HttpEntity;
+import org.springframework.http.MediaType;
+import org.springframework.test.context.junit4.SpringRunner;
+import org.springframework.test.web.reactive.server.WebTestClient;
+import org.springframework.util.StreamUtils;
+import org.springframework.web.reactive.function.BodyInserters;
+
+@WebFluxTest(ReactiveFileController.class)
+@RunWith(SpringRunner.class)
+@ImportAutoConfiguration(S3FileStorageConfiguration.class)
+public class OssUploadTest {
+
+    static {
+        System.setProperty("hsweb.file.upload.s3.endpoint", "https://oss-cn-beijing.aliyuncs.com");
+        System.setProperty("hsweb.file.upload.s3.region", "us-east-1");
+        System.setProperty("hsweb.file.upload.s3.accessKey", "");
+        System.setProperty("hsweb.file.upload.s3.secretKey", "");
+        System.setProperty("hsweb.file.upload.s3.bucket", "maydaysansan");
+        System.setProperty("hsweb.file.storage", "s3");
+    }
+
+    @Autowired
+    WebTestClient client;
+
+    @Test
+    public void testStatic(){
+        client.post()
+                .uri("/file/oss/static")
+                .contentType(MediaType.MULTIPART_FORM_DATA)
+                .body(BodyInserters.fromMultipartData("file",new HttpEntity<>(new ClassPathResource("test.json"))))
+                .exchange()
+                .expectStatus()
+                .isOk();
+
+    }
+
+    @Test
+    public void testStream() throws Exception {
+        byte[] fileBytes = StreamUtils.copyToByteArray(new ClassPathResource("test.json").getInputStream());
+
+        client.post()
+                .uri(uriBuilder ->
+                        uriBuilder.path("/file/oss/stream")
+                                .queryParam("fileType", "json")
+                                .build())
+                .contentType(MediaType.APPLICATION_OCTET_STREAM)
+                .bodyValue(fileBytes)
+                .exchange()
+                .expectStatus().isOk();
+    }
+
+}

hsweb-system/hsweb-system-file/src/test/java/org/hswebframework/web/file/web/ReactiveFileControllerTest.java

@@ -29,6 +29,7 @@ public class ReactiveFileControllerTest {
 
     static {
         System.setProperty("hsweb.file.upload.static-file-path","./target/upload");
+        System.setProperty("hsweb.file.storage","local");
 //        System.setProperty("hsweb.file.upload.use-original-file-name","true");
     }
 

pom.xml

@@ -95,6 +95,8 @@
         <swagger.version>2.7.0</swagger.version>
         <netty.version>4.1.111.Final</netty.version>
         <r2dbc.version>Borca-SR2</r2dbc.version>
+        <r2dbc.version>Borca-SR2</r2dbc.version>
+        <aws.sdk.version>2.25.5</aws.sdk.version>
     </properties>
 
     <profiles>