diff --git a/cmd/wire_gen.go b/cmd/wire_gen.go index f98deb424..65976e5aa 100644 --- a/cmd/wire_gen.go +++ b/cmd/wire_gen.go @@ -1,28 +1,8 @@ -//go:build !wireinject -// +build !wireinject - -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - // Code generated by Wire. DO NOT EDIT. //go:generate go run github.com/google/wire/cmd/wire +//go:build !wireinject +// +build !wireinject package answercmd @@ -48,6 +28,7 @@ import ( "github.com/apache/answer/internal/repo/comment" "github.com/apache/answer/internal/repo/config" "github.com/apache/answer/internal/repo/export" + "github.com/apache/answer/internal/repo/file" "github.com/apache/answer/internal/repo/file_record" "github.com/apache/answer/internal/repo/limit" "github.com/apache/answer/internal/repo/meta" @@ -182,7 +163,8 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, questionCommon := questioncommon.NewQuestionCommon(questionRepo, answerRepo, voteRepo, followRepo, tagCommonService, userCommon, collectionCommon, answerCommon, metaCommonService, configService, activityQueueService, revisionRepo, siteInfoCommonService, dataData) eventQueueService := event_queue.NewEventQueueService() fileRecordRepo := file_record.NewFileRecordRepo(dataData) - fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService, userCommon) + fileRepo := file.NewFileRepo(dataData) + fileRecordService := file_record2.NewFileRecordService(fileRecordRepo, revisionRepo, serviceConf, siteInfoCommonService, userCommon, fileRepo) userService := content.NewUserService(userRepo, userActiveActivityRepo, activityRepo, emailService, authService, siteInfoCommonService, userRoleRelService, userCommon, userExternalLoginService, userNotificationConfigRepo, userNotificationConfigService, questionCommon, eventQueueService, fileRecordService) captchaRepo := captcha.NewCaptchaRepo(dataData) captchaService := action.NewCaptchaService(captchaRepo) @@ -250,7 +232,7 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, notificationController := controller.NewNotificationController(notificationService, rankService) dashboardService := dashboard.NewDashboardService(questionRepo, answerRepo, commentCommonRepo, voteRepo, userRepo, reportRepo, configService, siteInfoCommonService, serviceConf, reviewService, revisionRepo, dataData) dashboardController := controller.NewDashboardController(dashboardService) - uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService) + uploaderService := uploader.NewUploaderService(serviceConf, siteInfoCommonService, fileRecordService, fileRepo) uploadController := controller.NewUploadController(uploaderService) activityActivityRepo := activity.NewActivityRepo(dataData, configService) activityCommon := activity_common2.NewActivityCommon(activityRepo, activityQueueService) @@ -274,7 +256,8 @@ func initApplication(debug bool, serverConf *conf.Server, dbConf *data.Database, badgeService := badge2.NewBadgeService(badgeRepo, badgeGroupRepo, badgeAwardRepo, badgeEventService, siteInfoCommonService) badgeController := controller.NewBadgeController(badgeService, badgeAwardService) controller_adminBadgeController := controller_admin.NewBadgeController(badgeService) - answerAPIRouter := router.NewAnswerAPIRouter(langController, userController, commentController, reportController, voteController, tagController, followController, collectionController, questionController, answerController, searchController, revisionController, rankController, userAdminController, reasonController, themeController, siteInfoController, controllerSiteInfoController, notificationController, dashboardController, uploadController, activityController, roleController, pluginController, permissionController, userPluginController, reviewController, metaController, badgeController, controller_adminBadgeController) + fileController := controller.NewFileController(fileRepo) + answerAPIRouter := router.NewAnswerAPIRouter(langController, userController, commentController, reportController, voteController, tagController, followController, collectionController, questionController, answerController, searchController, revisionController, rankController, userAdminController, reasonController, themeController, siteInfoController, controllerSiteInfoController, notificationController, dashboardController, uploadController, activityController, roleController, pluginController, permissionController, userPluginController, reviewController, metaController, badgeController, controller_adminBadgeController, fileController) swaggerRouter := router.NewSwaggerRouter(swaggerConf) uiRouter := router.NewUIRouter(controllerSiteInfoController, siteInfoCommonService) authUserMiddleware := middleware.NewAuthUserMiddleware(authService, siteInfoCommonService) diff --git a/internal/base/conf/conf.go b/internal/base/conf/conf.go index 4b71b206f..d5c9f5253 100644 --- a/internal/base/conf/conf.go +++ b/internal/base/conf/conf.go @@ -32,6 +32,7 @@ import ( "github.com/apache/answer/internal/service/service_config" "github.com/apache/answer/pkg/writer" "github.com/segmentfault/pacman/contrib/conf/viper" + "github.com/segmentfault/pacman/log" "gopkg.in/yaml.v3" ) @@ -50,6 +51,7 @@ type envConfigOverrides struct { SwaggerHost string SwaggerAddressPort string SiteAddr string + StorageMode string } func loadEnvs() (envOverrides *envConfigOverrides) { @@ -57,6 +59,7 @@ func loadEnvs() (envOverrides *envConfigOverrides) { SwaggerHost: os.Getenv("SWAGGER_HOST"), SwaggerAddressPort: os.Getenv("SWAGGER_ADDRESS_PORT"), SiteAddr: os.Getenv("SITE_ADDR"), + StorageMode: os.Getenv("FILE_STORAGE_MODE"), } } @@ -93,6 +96,10 @@ func (c *AllConfig) SetEnvironmentOverrides() { if envs.SwaggerAddressPort != "" { c.Swaggerui.Address = envs.SwaggerAddressPort } + if envs.StorageMode == "db" { + c.ServiceConfig.UseDbFileStorage = true + log.Info("saving files as blob in db") + } } // ReadConfig read config diff --git a/internal/controller/controller.go b/internal/controller/controller.go index cbf80f7fa..5d4530bb2 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -53,4 +53,5 @@ var ProviderSetController = wire.NewSet( NewEmbedController, NewBadgeController, NewRenderController, + NewFileController, ) diff --git a/internal/controller/file_controller.go b/internal/controller/file_controller.go new file mode 100644 index 000000000..46ba8141c --- /dev/null +++ b/internal/controller/file_controller.go @@ -0,0 +1,36 @@ +package controller + +import ( + "strconv" + + "github.com/apache/answer/internal/base/handler" + "github.com/apache/answer/internal/repo/file" + "github.com/gin-gonic/gin" +) + +type FileController struct { + fileRepo file.FileRepo +} + +func NewFileController(fileRepo file.FileRepo) *FileController { + return &FileController{fileRepo: fileRepo} +} + +func (bc *FileController) GetFile(ctx *gin.Context) { + id := ctx.Param("id") + download := ctx.DefaultQuery("download", "") + + blob, err := bc.fileRepo.GetByID(ctx.Request.Context(), id) + if err != nil || blob == nil { + handler.HandleResponse(ctx, err, "file not found") + return + } + + ctx.Header("Content-Type", blob.MimeType) + ctx.Header("Content-Length", strconv.FormatInt(blob.Size, 10)) + if download != "" { + ctx.Header("Content-Disposition", "attachment; filename=\""+download+"\"") + } + + ctx.Data(200, blob.MimeType, blob.Content) +} diff --git a/internal/entity/file_entity.go b/internal/entity/file_entity.go new file mode 100644 index 000000000..a75b2f906 --- /dev/null +++ b/internal/entity/file_entity.go @@ -0,0 +1,18 @@ +package entity + +import ( + "time" +) + +type File struct { + ID string `xorm:"pk varchar(36)"` + FileName string `xorm:"varchar(255) not null"` + MimeType string `xorm:"varchar(100)"` + Size int64 `xorm:"bigint"` + Content []byte `xorm:"blob"` + CreatedAt time.Time `xorm:"created"` +} + +func (File) TableName() string { + return "file" +} diff --git a/internal/migrations/init_data.go b/internal/migrations/init_data.go index 96151625d..791feec71 100644 --- a/internal/migrations/init_data.go +++ b/internal/migrations/init_data.go @@ -76,6 +76,7 @@ var ( &entity.BadgeAward{}, &entity.FileRecord{}, &entity.PluginKVStorage{}, + &entity.File{}, } roles = []*entity.Role{ diff --git a/internal/repo/file/file_repo.go b/internal/repo/file/file_repo.go new file mode 100644 index 000000000..a3a5cde35 --- /dev/null +++ b/internal/repo/file/file_repo.go @@ -0,0 +1,53 @@ +package file + +import ( + "context" + "database/sql" + + "github.com/apache/answer/internal/base/data" + "github.com/apache/answer/internal/base/reason" + "github.com/apache/answer/internal/entity" + "github.com/segmentfault/pacman/errors" +) + +type FileRepo interface { + Save(ctx context.Context, file *entity.File) error + GetByID(ctx context.Context, id string) (*entity.File, error) + Delete(ctx context.Context, id string) (err error) +} + +type fileRepo struct { + data *data.Data +} + +func NewFileRepo(data *data.Data) FileRepo { + return &fileRepo{data: data} +} + +func (r *fileRepo) Save(ctx context.Context, file *entity.File) error { + _, err := r.data.DB.Context(ctx).Insert(file) + if err != nil { + return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return nil +} + +func (r *fileRepo) GetByID(ctx context.Context, id string) (*entity.File, error) { + var blob entity.File + ok, err := r.data.DB.Context(ctx).ID(id).Get(&blob) + if err != nil { + return nil, errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + if !ok { + return nil, sql.ErrNoRows + } + return &blob, nil +} + +func (r *fileRepo) Delete(ctx context.Context, id string) (err error) { + _, err = r.data.DB.Context(ctx).ID(id).Delete(&entity.File{}) + if err != nil { + err = errors.InternalServer(reason.DatabaseError).WithError(err).WithStack() + } + return +} diff --git a/internal/router/answer_api_router.go b/internal/router/answer_api_router.go index e01bd9a95..c1e2ac5d8 100644 --- a/internal/router/answer_api_router.go +++ b/internal/router/answer_api_router.go @@ -57,6 +57,7 @@ type AnswerAPIRouter struct { metaController *controller.MetaController badgeController *controller.BadgeController adminBadgeController *controller_admin.BadgeController + fileController *controller.FileController } func NewAnswerAPIRouter( @@ -90,6 +91,7 @@ func NewAnswerAPIRouter( metaController *controller.MetaController, badgeController *controller.BadgeController, adminBadgeController *controller_admin.BadgeController, + fileController *controller.FileController, ) *AnswerAPIRouter { return &AnswerAPIRouter{ langController: langController, @@ -122,6 +124,7 @@ func NewAnswerAPIRouter( metaController: metaController, badgeController: badgeController, adminBadgeController: adminBadgeController, + fileController: fileController, } } @@ -148,6 +151,9 @@ func (a *AnswerAPIRouter) RegisterMustUnAuthAnswerAPIRouter(authUserMiddleware * // plugins r.GET("/plugin/status", a.pluginController.GetAllPluginStatus) + + // file branding + r.GET("/file/branding/:id", a.fileController.GetFile) } func (a *AnswerAPIRouter) RegisterUnAuthAnswerAPIRouter(r *gin.RouterGroup) { @@ -171,6 +177,10 @@ func (a *AnswerAPIRouter) RegisterUnAuthAnswerAPIRouter(r *gin.RouterGroup) { r.GET("/personal/question/page", a.questionController.PersonalQuestionPage) r.GET("/question/link", a.questionController.GetQuestionLink) + //file + r.GET("/file/post/:id", a.fileController.GetFile) + r.GET("/file/avatar/:id", a.fileController.GetFile) + // comment r.GET("/comment/page", a.commentController.GetCommentWithPage) r.GET("/personal/comment/page", a.commentController.GetCommentPersonalWithPage) @@ -310,6 +320,7 @@ func (a *AnswerAPIRouter) RegisterAnswerAPIRouter(r *gin.RouterGroup) { // meta r.PUT("/meta/reaction", a.metaController.AddOrUpdateReaction) + } func (a *AnswerAPIRouter) RegisterAnswerAdminAPIRouter(r *gin.RouterGroup) { diff --git a/internal/service/file_record/file_record_service.go b/internal/service/file_record/file_record_service.go index 29097ba8c..bb479925f 100644 --- a/internal/service/file_record/file_record_service.go +++ b/internal/service/file_record/file_record_service.go @@ -29,6 +29,7 @@ import ( "github.com/apache/answer/internal/base/constant" "github.com/apache/answer/internal/entity" + "github.com/apache/answer/internal/repo/file" "github.com/apache/answer/internal/service/revision" "github.com/apache/answer/internal/service/service_config" "github.com/apache/answer/internal/service/siteinfo_common" @@ -56,6 +57,7 @@ type FileRecordService struct { serviceConfig *service_config.ServiceConfig siteInfoService siteinfo_common.SiteInfoCommonService userService *usercommon.UserCommon + fileRepo file.FileRepo } // NewFileRecordService new file record service @@ -65,6 +67,7 @@ func NewFileRecordService( serviceConfig *service_config.ServiceConfig, siteInfoService siteinfo_common.SiteInfoCommonService, userService *usercommon.UserCommon, + fileRepo file.FileRepo, ) *FileRecordService { return &FileRecordService{ fileRecordRepo: fileRecordRepo, @@ -72,6 +75,7 @@ func NewFileRecordService( serviceConfig: serviceConfig, siteInfoService: siteInfoService, userService: userService, + fileRepo: fileRepo, } } @@ -183,6 +187,16 @@ func (fs *FileRecordService) DeleteAndMoveFileRecord(ctx context.Context, fileRe return fmt.Errorf("delete file record error: %v", err) } + if fs.serviceConfig.UseDbFileStorage { + fileURL := fileRecord.FileURL + parts := strings.Split(fileURL, "/") + fileId := parts[len(parts)-1] + if err := fs.fileRepo.Delete(ctx, fileId); err != nil { + return fmt.Errorf("failed to delete file: %v", err) + } + return nil + } + // Move the file to the deleted directory oldFilename := filepath.Base(fileRecord.FilePath) oldFilePath := filepath.Join(fs.serviceConfig.UploadPath, fileRecord.FilePath) diff --git a/internal/service/provider.go b/internal/service/provider.go index 4b1b64276..0f2573350 100644 --- a/internal/service/provider.go +++ b/internal/service/provider.go @@ -20,6 +20,7 @@ package service import ( + "github.com/apache/answer/internal/repo/file" "github.com/apache/answer/internal/service/action" "github.com/apache/answer/internal/service/activity" "github.com/apache/answer/internal/service/activity_common" @@ -40,7 +41,7 @@ import ( "github.com/apache/answer/internal/service/follow" "github.com/apache/answer/internal/service/importer" "github.com/apache/answer/internal/service/meta" - "github.com/apache/answer/internal/service/meta_common" + metacommon "github.com/apache/answer/internal/service/meta_common" "github.com/apache/answer/internal/service/notice_queue" "github.com/apache/answer/internal/service/notification" notficationcommon "github.com/apache/answer/internal/service/notification_common" @@ -128,4 +129,5 @@ var ProviderSetService = wire.NewSet( badge.NewBadgeGroupService, importer.NewImporterService, file_record.NewFileRecordService, + file.NewFileRepo, ) diff --git a/internal/service/service_config/service_config.go b/internal/service/service_config/service_config.go index 90e399e17..ca8e31177 100644 --- a/internal/service/service_config/service_config.go +++ b/internal/service/service_config/service_config.go @@ -24,4 +24,5 @@ type ServiceConfig struct { CleanUpUploads bool `json:"clean_up_uploads" mapstructure:"clean_up_uploads" yaml:"clean_up_uploads"` CleanOrphanUploadsPeriodHours int `json:"clean_orphan_uploads_period_hours" mapstructure:"clean_orphan_uploads_period_hours" yaml:"clean_orphan_uploads_period_hours"` PurgeDeletedFilesPeriodDays int `json:"purge_deleted_files_period_days" mapstructure:"purge_deleted_files_period_days" yaml:"purge_deleted_files_period_days"` + UseDbFileStorage bool `json:"use_db_file_storage" mapstructure:"use_db_file_storage" yaml:"use_db_file_storage"` } diff --git a/internal/service/uploader/upload.go b/internal/service/uploader/upload.go index 2ae5369df..4b3f9c4ef 100644 --- a/internal/service/uploader/upload.go +++ b/internal/service/uploader/upload.go @@ -30,8 +30,12 @@ import ( "path" "path/filepath" "strings" + "time" + "github.com/apache/answer/internal/entity" + "github.com/apache/answer/internal/repo/file" "github.com/apache/answer/internal/service/file_record" + "github.com/google/uuid" "github.com/apache/answer/internal/base/constant" "github.com/apache/answer/internal/base/reason" @@ -78,6 +82,7 @@ type uploaderService struct { serviceConfig *service_config.ServiceConfig siteInfoService siteinfo_common.SiteInfoCommonService fileRecordService *file_record.FileRecordService + fileRepo file.FileRepo } // NewUploaderService new upload service @@ -85,6 +90,7 @@ func NewUploaderService( serviceConfig *service_config.ServiceConfig, siteInfoService siteinfo_common.SiteInfoCommonService, fileRecordService *file_record.FileRecordService, + fileRepo file.FileRepo, ) UploaderService { for _, subPath := range subPathList { err := dir.CreateDirIfNotExist(filepath.Join(serviceConfig.UploadPath, subPath)) @@ -96,6 +102,7 @@ func NewUploaderService( serviceConfig: serviceConfig, siteInfoService: siteInfoService, fileRecordService: fileRecordService, + fileRepo: fileRepo, } } @@ -125,12 +132,12 @@ func (us *uploaderService) UploadAvatarFile(ctx *gin.Context, userID string) (ur return "", errors.BadRequest(reason.RequestFormatError).WithError(err) } - newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) - avatarFilePath := path.Join(constant.AvatarSubPath, newFilename) - url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath) + fileHeader.Filename = fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) + url, err = us.uploadImageFile(ctx, fileHeader, constant.AvatarSubPath) if err != nil { return "", err } + avatarFilePath := path.Join(constant.AvatarSubPath, fileHeader.Filename) us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserAvatar)) return url, nil @@ -214,13 +221,14 @@ func (us *uploaderService) UploadPostFile(ctx *gin.Context, userID string) ( } fileExt := strings.ToLower(path.Ext(fileHeader.Filename)) - newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) - avatarFilePath := path.Join(constant.PostSubPath, newFilename) - url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath) + fileHeader.Filename = fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) + url, err = us.uploadImageFile(ctx, fileHeader, constant.PostSubPath) if err != nil { return "", err } - us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.UserPost)) + + postFilePath := path.Join(constant.PostSubPath, fileHeader.Filename) + us.fileRecordService.AddFileRecord(ctx, userID, postFilePath, url, string(plugin.UserPost)) return url, nil } @@ -285,18 +293,18 @@ func (us *uploaderService) UploadBrandingFile(ctx *gin.Context, userID string) ( if _, ok := plugin.DefaultFileTypeCheckMapping[plugin.AdminBranding][fileExt]; !ok { return "", errors.BadRequest(reason.RequestFormatError).WithError(err) } - - newFilename := fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) - avatarFilePath := path.Join(constant.BrandingSubPath, newFilename) - url, err = us.uploadImageFile(ctx, fileHeader, avatarFilePath) + fileHeader.Filename = fmt.Sprintf("%s%s", uid.IDStr12(), fileExt) + url, err = us.uploadImageFile(ctx, fileHeader, constant.BrandingSubPath) if err != nil { return "", err } + avatarFilePath := path.Join(constant.BrandingSubPath, fileHeader.Filename) us.fileRecordService.AddFileRecord(ctx, userID, avatarFilePath, url, string(plugin.AdminBranding)) return url, nil } +// TODO add new file name func (us *uploaderService) uploadImageFile(ctx *gin.Context, file *multipart.FileHeader, fileSubPath string) ( url string, err error) { siteGeneral, err := us.siteInfoService.GetSiteGeneral(ctx) @@ -307,7 +315,37 @@ func (us *uploaderService) uploadImageFile(ctx *gin.Context, file *multipart.Fil if err != nil { return "", err } - filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath) + if us.serviceConfig.UseDbFileStorage { + src, err := file.Open() + if err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + defer src.Close() + + buffer := new(bytes.Buffer) + if _, err = io.Copy(buffer, src); err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + + file := &entity.File{ + ID: uuid.New().String(), + FileName: file.Filename, + MimeType: file.Header.Get("Content-Type"), + Size: int64(len(buffer.Bytes())), + Content: buffer.Bytes(), + CreatedAt: time.Now(), + } + + err = us.fileRepo.Save(ctx, file) + if err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + + return fmt.Sprintf("%s/answer/api/v1/file/%s/%s", siteGeneral.SiteUrl, fileSubPath, file.ID), nil + //TODO checks: DecodeAndCheckImageFile removeExif + } + filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath, file.Filename) + if err := ctx.SaveUploadedFile(file, filePath); err != nil { return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() } @@ -336,6 +374,35 @@ func (us *uploaderService) uploadAttachmentFile(ctx *gin.Context, file *multipar if err != nil { return "", err } + if us.serviceConfig.UseDbFileStorage { + src, err := file.Open() + if err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + defer src.Close() + + buf := new(bytes.Buffer) + if _, err = io.Copy(buf, src); err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + + blob := &entity.File{ + ID: uuid.New().String(), + FileName: originalFilename, + MimeType: file.Header.Get("Content-Type"), + Size: int64(len(buf.Bytes())), + Content: buf.Bytes(), + CreatedAt: time.Now(), + } + + err = us.fileRepo.Save(ctx, blob) + if err != nil { + return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack() + } + + downloadUrl = fmt.Sprintf("%s/answer/api/v1/file/%s?download=%s", siteGeneral.SiteUrl, blob.ID, url.QueryEscape(originalFilename)) + return downloadUrl, nil + } filePath := path.Join(us.serviceConfig.UploadPath, fileSubPath) if err := ctx.SaveUploadedFile(file, filePath); err != nil { return "", errors.InternalServer(reason.UnknownError).WithError(err).WithStack()