Implement File storage in file system

backend/app/app.go

@@ -162,6 +162,17 @@ func NewApp(ctx context.Context, config internal.Configuration) (*App, error) {
 		return nil, fmt.Errorf("failed to create system identity: %w", err)
 	}
 
+	// Initialize file storage service
+	fileStorage, err := internal.NewFileStorageService(config.FileStoragePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to init file storage service: %w", err)
+	}
+
+	// Migrate file data from database to file storage
+	if err := models.MigrateFileDataToStorage(db, fileStorage); err != nil {
+		logger.GetLogger().Warn().Err(err).Msg("Failed to migrate file data to storage (continuing anyway)")
+	}
+
 	sshPublicKeyBytes, err := os.ReadFile(config.SSH.PublicKeyPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read SSH public key from %s: %w", config.SSH.PublicKeyPath, err)
@@ -209,7 +220,7 @@ func NewApp(ctx context.Context, config internal.Configuration) (*App, error) {
 	handler := NewHandler(tokenHandler, db, config, mailService, gridProxy,
 		substrateClient, graphqlClient, firesquidClient,
 		sseManager, ewfEngine, config.SystemAccount.Network, sshPublicKey,
-		systemIdentity, kycClient, sponsorKeyPair, sponsorAddress, metrics, notificationService, gridClient, appCtx)
+		systemIdentity, kycClient, sponsorKeyPair, sponsorAddress, metrics, notificationService, gridClient, fileStorage, appCtx)
 
 	app := &App{
 		router:              router,
@@ -236,6 +247,7 @@ func NewApp(ctx context.Context, config internal.Configuration) (*App, error) {
 		app.metrics,
 		app.notificationService,
 		gridProxy,
+		fileStorage,
 	)
 
 	app.registerHandlers()

backend/app/deployment_handler.go

@@ -210,8 +210,8 @@ func (h *Handler) HandleGetKubeconfig(c *gin.Context) {
 		return
 	}
 
-	if cluster.Kubeconfig != "" {
-		c.JSON(http.StatusOK, gin.H{"kubeconfig": cluster.Kubeconfig})
+	if data, err := h.fileStorage.ReadKubeconfigFile(userID, cluster.ID, projectName); err == nil {
+		c.JSON(http.StatusOK, gin.H{"kubeconfig": string(data)})
 		return
 	}
 
@@ -236,9 +236,8 @@ func (h *Handler) HandleGetKubeconfig(c *gin.Context) {
 		return
 	}
 
-	cluster.Kubeconfig = kubeconfig
-	if err := h.db.UpdateCluster(&cluster); err != nil {
-		reqLog.Error().Err(err).Int("cluster_id", cluster.ID).Msg("Failed to save kubeconfig to database")
+	if _, err := h.fileStorage.WriteKubeconfigFile(userID, cluster.ID, projectName, []byte(kubeconfig)); err != nil {
+		reqLog.Error().Err(err).Int("cluster_id", cluster.ID).Msg("Failed to persist kubeconfig to file storage")
 	}
 
 	c.JSON(http.StatusOK, gin.H{"kubeconfig": kubeconfig})

backend/app/invoice_handler.go

@@ -7,6 +7,7 @@ import (
 	"kubecloud/internal"
 	"kubecloud/models"
 	"net/http"
+	"os"
 	"strconv"
 
 	"time"
@@ -157,38 +158,41 @@ func (h *Handler) DownloadInvoiceHandler(c *gin.Context) {
 		return
 	}
 
-	// Creating pdf for invoice if it doesn't have it
-	if len(invoice.FileData) == 0 {
-		user, err := h.db.GetUserByID(userID)
-		if err != nil {
-			reqLog.Error().Err(err).Msg("failed to retrieve user")
-			InternalServerError(c)
-			return
-		}
-
-		pdfContent, err := internal.CreateInvoicePDF(invoice, user, h.config.Invoice)
-		if err != nil {
-			reqLog.Error().Err(err).Msg("failed to create invoice PDF")
-			InternalServerError(c)
-			return
-		}
+	if userID != invoice.UserID {
+		Error(c, http.StatusForbidden, "User is not authorized to download this invoice", "")
+		return
+	}
 
-		invoice.FileData = pdfContent
-		if err := h.db.UpdateInvoicePDF(id, invoice.FileData); err != nil {
-			reqLog.Error().Err(err).Msg("failed to update invoice PDF")
+	data, err := h.fileStorage.ReadInvoiceFile(userID, invoice.ID)
+	if err != nil {
+		if os.IsNotExist(err) {
+			// Generate on-demand and persist, then read again
+			user, uErr := h.db.GetUserByID(userID)
+			if uErr != nil {
+				logger.GetLogger().Error().Err(uErr).Send()
+				InternalServerError(c)
+				return
+			}
+			pdfContent, gErr := internal.CreateInvoicePDF(invoice, user, h.config.Invoice)
+			if gErr != nil {
+				logger.GetLogger().Error().Err(gErr).Send()
+				InternalServerError(c)
+				return
+			}
+			if _, wErr := h.fileStorage.WriteInvoiceFile(userID, invoice.ID, pdfContent); wErr != nil {
+				logger.GetLogger().Error().Err(wErr).Msg("failed to write invoice pdf to storage")
+			}
+			data = pdfContent
+		} else {
+			logger.GetLogger().Error().Err(err).Msg("failed to read invoice pdf from storage")
 			InternalServerError(c)
 			return
 		}
 	}
-
-	if userID != invoice.UserID {
-		Error(c, http.StatusNotFound, "User is not authorized to download this invoice", "")
-		return
-	}
-
-	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fmt.Sprintf("invoice-%d-%d.pdf", invoice.UserID, invoice.ID)))
+	fileName := h.fileStorage.InvoiceFileName(userID, invoice.ID)
+	c.Writer.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%s", fileName))
 	c.Writer.Header().Set("Content-Type", "application/pdf")
-	c.Data(http.StatusOK, "application/pdf", invoice.FileData)
+	c.Data(http.StatusOK, "application/pdf", data)
 
 }
 
@@ -262,16 +266,23 @@ func (h *Handler) createUserInvoice(user models.User) error {
 	if err != nil {
 		return err
 	}
-
-	invoice.FileData = file
 	if err = h.db.CreateInvoice(&invoice); err != nil {
 		return err
 	}
 
+	if _, err := h.fileStorage.WriteInvoiceFile(user.ID, invoice.ID, file); err != nil {
+		return err
+	}
+
 	subject, body := mailservice.InvoiceMailContent(totalInvoiceCostUSD, h.config.Currency, invoice.ID)
+	// read back for attachment
+	data, err := h.fileStorage.ReadInvoiceFile(user.ID, invoice.ID)
+	if err != nil {
+		return err
+	}
 	err = h.mailService.SendMail(h.config.MailSender.Email, user.Email, subject, body, mailservice.Attachment{
-		FileName: fmt.Sprintf("invoice-%d-%d.pdf", invoice.UserID, invoice.ID),
-		Data:     invoice.FileData,
+		FileName: h.fileStorage.InvoiceFileName(user.ID, invoice.ID),
+		Data:     data,
 	})
 	if err != nil {
 		return err

backend/app/invoice_handler_test.go

@@ -3,24 +3,28 @@ package app
 import (
 	"encoding/json"
 	"fmt"
+	"kubecloud/internal"
+	"kubecloud/models"
 	"net/http"
 	"net/http/httptest"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-
-	"kubecloud/models"
 )
 
 func TestListAllInvoicesHandler(t *testing.T) {
 	app, err := SetUp(t)
 	require.NoError(t, err)
 	router := app.router
 
-	adminUser := CreateTestUser(t, app, "[email protected]", "Admin User", []byte("securepassword"), true, true, false, 0, time.Now())
-	nonAdminUser := CreateTestUser(t, app, "[email protected]", "Normal User", []byte("securepassword"), true, false, false, 0, time.Now())
+	require.NotNil(t, app.handlers.fileStorage)
+
+	adminUser := CreateTestUser(t, app, "[email protected]", "Admin User", []byte("securepassword"), true, true, true, 0, time.Now())
+	nonAdminUser := CreateTestUser(t, app, "[email protected]", "Normal User", []byte("securepassword"), true, false, true, 0, time.Now())
 
 	t.Run("Test List all invoices with empty list", func(t *testing.T) {
 		token := GetAuthToken(t, app, adminUser.ID, adminUser.Email, adminUser.Username, true)
@@ -62,6 +66,16 @@ func TestListAllInvoicesHandler(t *testing.T) {
 	err = app.handlers.db.CreateInvoice(invoice2)
 	require.NoError(t, err)
 
+	pdf1, err := internal.CreateInvoicePDF(*invoice1, *adminUser, app.config.Invoice)
+	require.NoError(t, err)
+	pdf2, err := internal.CreateInvoicePDF(*invoice2, *nonAdminUser, app.config.Invoice)
+	require.NoError(t, err)
+
+	_, err = app.handlers.fileStorage.WriteInvoiceFile(adminUser.ID, invoice1.ID, pdf1)
+	require.NoError(t, err)
+	_, err = app.handlers.fileStorage.WriteInvoiceFile(nonAdminUser.ID, invoice2.ID, pdf2)
+	require.NoError(t, err)
+
 	t.Run("Test List all invoices successfully", func(t *testing.T) {
 		token := GetAuthToken(t, app, adminUser.ID, adminUser.Email, adminUser.Username, true)
 		req, _ := http.NewRequest("GET", "/api/v1/invoices", nil)
@@ -95,6 +109,18 @@ func TestListAllInvoicesHandler(t *testing.T) {
 		}
 		assert.True(t, found1, "Admin's invoice should be in the list")
 		assert.True(t, found2, "Normal user's invoice should be in the list")
+
+		storedPDF1, err := app.handlers.fileStorage.ReadInvoiceFile(adminUser.ID, invoice1.ID)
+		require.NoError(t, err)
+		assert.Equal(t, pdf1, storedPDF1)
+		assert.Greater(t, len(storedPDF1), 100)
+
+		storedPDF2, err := app.handlers.fileStorage.ReadInvoiceFile(nonAdminUser.ID, invoice2.ID)
+		require.NoError(t, err)
+		assert.Equal(t, pdf2, storedPDF2)
+		assert.Greater(t, len(storedPDF2), 100)
+
+		assert.NotEqual(t, storedPDF1, storedPDF2)
 	})
 
 	t.Run("Test List all invoices with no token", func(t *testing.T) {
@@ -119,7 +145,7 @@ func TestListUserInvoicesHandler(t *testing.T) {
 	require.NoError(t, err)
 	router := app.router
 
-	user := CreateTestUser(t, app, "[email protected]", "Test User", []byte("securepassword"), true, false, false, 0, time.Now())
+	user := CreateTestUser(t, app, "[email protected]", "Test User", []byte("securepassword"), true, false, true, 0, time.Now())
 
 	t.Run("Test List user invoices with empty list", func(t *testing.T) {
 		token := GetAuthToken(t, app, user.ID, user.Email, user.Username, false)
@@ -153,6 +179,13 @@ func TestListUserInvoicesHandler(t *testing.T) {
 	err = app.handlers.db.CreateInvoice(invoice1)
 	require.NoError(t, err)
 
+	pdfContent, err := internal.CreateInvoicePDF(*invoice1, *user, app.config.Invoice)
+	require.NoError(t, err)
+	require.NotEmpty(t, pdfContent)
+
+	_, err = app.handlers.fileStorage.WriteInvoiceFile(user.ID, invoice1.ID, pdfContent)
+	require.NoError(t, err)
+
 	t.Run("Test List user invoices successfully", func(t *testing.T) {
 		token := GetAuthToken(t, app, user.ID, user.Email, user.Username, false)
 		req, _ := http.NewRequest("GET", "/api/v1/user/invoice", nil)
@@ -164,6 +197,15 @@ func TestListUserInvoicesHandler(t *testing.T) {
 		err := json.Unmarshal(resp.Body.Bytes(), &result)
 		assert.NoError(t, err)
 		assert.Equal(t, "Invoices are retrieved successfully", result["message"])
+
+		data := result["data"].(map[string]interface{})
+		invoicesRaw := data["invoices"].([]interface{})
+		assert.Len(t, invoicesRaw, 1)
+
+		storedPDF, err := app.handlers.fileStorage.ReadInvoiceFile(user.ID, invoice1.ID)
+		require.NoError(t, err)
+		assert.Equal(t, pdfContent, storedPDF)
+		assert.Greater(t, len(storedPDF), 100)
 	})
 
 	t.Run("Test List user invoices with no token", func(t *testing.T) {
@@ -180,6 +222,8 @@ func TestDownloadInvoiceHandler(t *testing.T) {
 	require.NoError(t, err)
 	router := app.router
 
+	require.NotNil(t, app.handlers.fileStorage)
+
 	user1 := CreateTestUser(t, app, "[email protected]", "User One", []byte("securepassword"), true, false, false, 0, time.Now())
 
 	invoice := &models.Invoice{
@@ -193,14 +237,44 @@ func TestDownloadInvoiceHandler(t *testing.T) {
 	require.NoError(t, err)
 
 	t.Run("Download an invoice successfully", func(t *testing.T) {
+		pdfContent, err := internal.CreateInvoicePDF(*invoice, *user1, app.config.Invoice)
+		require.NoError(t, err)
+		require.NotEmpty(t, pdfContent)
+		require.Greater(t, len(pdfContent), 100)
+
+		fileName, err := app.handlers.fileStorage.WriteInvoiceFile(user1.ID, invoice.ID, pdfContent)
+		require.NoError(t, err)
+		assert.Contains(t, fileName, fmt.Sprintf("user-%d", user1.ID))
+		assert.Contains(t, fileName, fmt.Sprintf("invoice-%d", invoice.ID))
+
+		filePath := filepath.Join(app.config.FileStoragePath, "invoices", fileName)
+		assert.FileExists(t, filePath)
+
+		fileInfo, err := os.Stat(filePath)
+		require.NoError(t, err)
+		assert.Equal(t, os.FileMode(0o600), fileInfo.Mode().Perm())
+
 		token := GetAuthToken(t, app, user1.ID, user1.Email, user1.Username, false)
 		req, _ := http.NewRequest("GET", fmt.Sprintf("/api/v1/user/invoice/%d", invoice.ID), nil)
 		req.Header.Set("Authorization", "Bearer "+token)
 		resp := httptest.NewRecorder()
 		router.ServeHTTP(resp, req)
+
 		assert.Equal(t, http.StatusOK, resp.Code)
 		assert.Equal(t, "application/pdf", resp.Header().Get("Content-Type"))
-		assert.True(t, len(resp.Body.Bytes()) > 0)
+
+		responseBody := resp.Body.Bytes()
+		assert.NotEmpty(t, responseBody)
+		assert.Greater(t, len(responseBody), 100)
+		assert.Equal(t, pdfContent, responseBody)
+
+		if len(responseBody) >= 4 {
+			assert.Equal(t, "%PDF", string(responseBody[:4]))
+		}
+
+		contentDisposition := resp.Header().Get("Content-Disposition")
+		assert.Contains(t, contentDisposition, "attachment")
+		assert.Contains(t, contentDisposition, fileName)
 	})
 
 	t.Run("Download invoice with no token", func(t *testing.T) {
@@ -228,4 +302,65 @@ func TestDownloadInvoiceHandler(t *testing.T) {
 		router.ServeHTTP(resp, req)
 		assert.Equal(t, http.StatusBadRequest, resp.Code)
 	})
+
+	t.Run("Download invoice missing from file storage generates on-demand", func(t *testing.T) {
+		user2 := CreateTestUser(t, app, "[email protected]", "User Two", []byte("password"), true, false, true, 0, time.Now())
+
+		invoice2 := &models.Invoice{
+			UserID:    user2.ID,
+			Total:     99.99,
+			Tax:       9.99,
+			CreatedAt: time.Now(),
+		}
+		err := app.db.CreateInvoice(invoice2)
+		require.NoError(t, err)
+
+		token := GetAuthToken(t, app, user2.ID, user2.Email, user2.Username, false)
+		req, _ := http.NewRequest("GET", fmt.Sprintf("/api/v1/user/invoice/%d", invoice2.ID), nil)
+		req.Header.Set("Authorization", "Bearer "+token)
+		resp := httptest.NewRecorder()
+		router.ServeHTTP(resp, req)
+
+		assert.Equal(t, http.StatusOK, resp.Code)
+		assert.Equal(t, "application/pdf", resp.Header().Get("Content-Type"))
+
+		responseBody := resp.Body.Bytes()
+		assert.NotEmpty(t, responseBody)
+		assert.Greater(t, len(responseBody), 100)
+
+		if len(responseBody) >= 4 {
+			assert.Equal(t, "%PDF", string(responseBody[:4]))
+		}
+
+		storedContent, err := app.handlers.fileStorage.ReadInvoiceFile(user2.ID, invoice2.ID)
+		require.NoError(t, err)
+		assert.Equal(t, responseBody, storedContent)
+	})
+
+	t.Run("User cannot download another user's invoice", func(t *testing.T) {
+		user3 := CreateTestUser(t, app, "[email protected]", "User Three", []byte("password"), true, false, true, 0, time.Now())
+
+		invoice3 := &models.Invoice{
+			UserID:    user1.ID,
+			Total:     250.00,
+			Tax:       25.00,
+			CreatedAt: time.Now(),
+		}
+		err := app.db.CreateInvoice(invoice3)
+		require.NoError(t, err)
+
+		pdfContent, err := internal.CreateInvoicePDF(*invoice3, *user1, app.config.Invoice)
+		require.NoError(t, err)
+		_, err = app.handlers.fileStorage.WriteInvoiceFile(user1.ID, invoice3.ID, pdfContent)
+		require.NoError(t, err)
+
+		token := GetAuthToken(t, app, user3.ID, user3.Email, user3.Username, false)
+		req, _ := http.NewRequest("GET", fmt.Sprintf("/api/v1/user/invoice/%d", invoice3.ID), nil)
+		req.Header.Set("Authorization", "Bearer "+token)
+		resp := httptest.NewRecorder()
+		router.ServeHTTP(resp, req)
+
+		assert.Equal(t, http.StatusForbidden, resp.Code)
+		assert.NotEqual(t, "application/pdf", resp.Header().Get("Content-Type"))
+	})
 }