fix: copy command using the new version (#344)

Author: fenos

migrations/multitenant/0006-add-tenants-external-credentials.sql

@@ -0,0 +1,38 @@
+create extension if not exists "uuid-ossp";
+
+CREATE TABLE tenants_external_credentials (
+    "id" uuid NOT NULL DEFAULT uuid_generate_v4(),
+    name text NOT NULL unique,
+    tenant_id text NOT NULL references tenants (id),
+    provider text NOT NULL default 's3',
+    access_key text NULL,
+    secret_key text NULL,
+    role text null,
+    region text not null,
+    endpoint text NULL,
+    force_path_style boolean NOT NULL default false,
+    PRIMARY KEY (id)
+);
+
+create index external_buckets_tenant_id_idx on tenants_external_credentials (tenant_id);
+
+CREATE FUNCTION tenants_external_credentials_update_notify_trigger ()
+    RETURNS TRIGGER
+AS $$
+BEGIN
+    PERFORM
+        pg_notify('tenants_external_credentials_update', '"' || NEW.id || ':' || NEW.tenant_id || '"');
+    RETURN NULL;
+END;
+$$
+    LANGUAGE plpgsql;
+
+CREATE TRIGGER tenants_external_credentials_notify_trigger
+    AFTER DELETE ON tenants_external_credentials
+    FOR EACH ROW
+EXECUTE PROCEDURE tenants_external_credentials_update_notify_trigger();
+
+CREATE TRIGGER tenants_external_credentials_notify_trigger
+    AFTER UPDATE ON tenants_external_credentials
+    FOR EACH ROW
+EXECUTE PROCEDURE tenants_external_credentials_update_notify_trigger();

src/storage/database/knex.ts

@@ -108,7 +108,13 @@ export class StorageKnexDB implements Database {
   async createBucket(
     data: Pick<
       Bucket,
-      'id' | 'name' | 'public' | 'owner' | 'file_size_limit' | 'allowed_mime_types'
+      | 'id'
+      | 'name'
+      | 'public'
+      | 'owner'
+      | 'file_size_limit'
+      | 'allowed_mime_types'
+      | 'external_credential_id'
     >
   ) {
     const bucketData = {
@@ -118,6 +124,7 @@ export class StorageKnexDB implements Database {
       public: data.public,
       allowed_mime_types: data.allowed_mime_types,
       file_size_limit: data.file_size_limit,
+      external_credential_id: data.external_credential_id,
     }
 
     const bucket = await this.runQuery('CreateBucket', async (knex) => {
@@ -204,13 +211,17 @@ export class StorageKnexDB implements Database {
 
   async updateBucket(
     bucketId: string,
-    fields: Pick<Bucket, 'public' | 'file_size_limit' | 'allowed_mime_types'>
+    fields: Pick<
+      Bucket,
+      'public' | 'file_size_limit' | 'allowed_mime_types' | 'external_credential_id'
+    >
   ) {
     const bucket = await this.runQuery('UpdateBucket', (knex) => {
       return knex.from('buckets').where('id', bucketId).update({
         public: fields.public,
         file_size_limit: fields.file_size_limit,
         allowed_mime_types: fields.allowed_mime_types,
+        external_credential_id: fields.external_credential_id,
       })
     })
 

src/storage/object.ts

@@ -23,7 +23,7 @@ export interface UploadObjectOptions {
   version?: string
 }
 
-const { urlLengthLimit, globalS3Bucket } = getConfig()
+const { urlLengthLimit } = getConfig()
 
 /**
  * ObjectStorage
@@ -159,17 +159,15 @@ export class ObjectStorage {
           // if successfully deleted, delete from s3 too
           // todo: consider moving this to a queue
           const prefixesToDelete = data.reduce((all, { name, version }) => {
-            all.push(withOptionalVersion(`${db.tenantId}/${this.bucketId}/${name}`, version))
+            all.push(withOptionalVersion(`${this.bucketId}/${name}`, version))
 
             if (version) {
-              all.push(
-                withOptionalVersion(`${db.tenantId}/${this.bucketId}/${name}`, version) + '.info'
-              )
+              all.push(withOptionalVersion(`${this.bucketId}/${name}`, version) + '.info')
             }
             return all
           }, [] as string[])
 
-          await this.backend.deleteObjects(globalS3Bucket, prefixesToDelete)
+          await this.backend.deleteObjects(prefixesToDelete)
 
           await Promise.allSettled(
             data.map((object) =>
@@ -255,38 +253,38 @@ export class ObjectStorage {
 
     const newVersion = randomUUID()
     const bucketId = this.bucketId
-    const s3SourceKey = `${this.db.tenantId}/${bucketId}/${sourceKey}`
-    const s3DestinationKey = `${this.db.tenantId}/${bucketId}/${destinationKey}`
-
-    try {
-      // We check if the user has permission to copy the object to the destination key
-      const originObject = await this.db.findObject(
-        this.bucketId,
-        sourceKey,
-        'bucket_id,metadata,version'
-      )
+    const s3SourceKey = `${bucketId}/${sourceKey}`
+    const s3DestinationKey = `${bucketId}/${destinationKey}`
+
+    // We check if the user has permission to copy the object to the destination key
+    const originObject = await this.db.findObject(
+      this.bucketId,
+      sourceKey,
+      'bucket_id,metadata,version'
+    )
 
-      await this.uploader.canUpload({
-        bucketId: this.bucketId,
-        objectName: destinationKey,
-        isUpsert: false,
-      })
+    await this.uploader.canUpload({
+      bucketId: this.bucketId,
+      objectName: destinationKey,
+      isUpsert: false,
+    })
 
+    try {
       const copyResult = await this.backend.copyObject(
-        globalS3Bucket,
         s3SourceKey,
         originObject.version,
         s3DestinationKey,
         newVersion
       )
 
-      const metadata = await this.backend.headObject(globalS3Bucket, s3DestinationKey, newVersion)
+      const metadata = await this.backend.headObject(s3DestinationKey, newVersion)
 
       const destObject = await this.db.createObject({
         ...originObject,
         name: destinationKey,
         owner,
         metadata,
+        version: newVersion,
       })
 
       await ObjectCreatedCopyEvent.sendWebhook({
@@ -325,8 +323,8 @@ export class ObjectStorage {
     }
 
     const newVersion = randomUUID()
-    const s3SourceKey = `${this.db.tenantId}/${this.bucketId}/${sourceObjectName}`
-    const s3DestinationKey = `${this.db.tenantId}/${this.bucketId}/${destinationObjectName}`
+    const s3SourceKey = `${this.bucketId}/${sourceObjectName}`
+    const s3DestinationKey = `${this.bucketId}/${destinationObjectName}`
 
     await this.db.testPermission((db) => {
       return Promise.all([
@@ -352,15 +350,9 @@ export class ObjectStorage {
       .findObject(this.bucketId, sourceObjectName, 'id, version')
 
     try {
-      await this.backend.copyObject(
-        globalS3Bucket,
-        s3SourceKey,
-        sourceObj.version,
-        s3DestinationKey,
-        newVersion
-      )
+      await this.backend.copyObject(s3SourceKey, sourceObj.version, s3DestinationKey, newVersion)
 
-      const metadata = await this.backend.headObject(globalS3Bucket, s3DestinationKey, newVersion)
+      const metadata = await this.backend.headObject(s3DestinationKey, newVersion)
 
       await this.db.asSuperUser().withTransaction(async (db) => {
         await db.createObject({
@@ -372,14 +364,14 @@ export class ObjectStorage {
         })
 
         await db.deleteObject(this.bucketId, sourceObjectName, sourceObj.version)
+        await ObjectAdminDelete.send({
+          name: sourceObjectName,
+          bucketId: this.bucketId,
+          tenant: this.db.tenant(),
+          version: sourceObj.version,
+        })
 
-        await Promise.all([
-          ObjectAdminDelete.send({
-            name: sourceObjectName,
-            bucketId: this.bucketId,
-            tenant: this.db.tenant(),
-            version: sourceObj.version,
-          }),
+        await Promise.allSettled([
           ObjectRemovedMove.sendWebhook({
             tenant: this.db.tenant(),
             name: sourceObjectName,