From a6756fad710736726f5093b6ce4e1fc7cacaffba Mon Sep 17 00:00:00 2001 From: Tigran Avetisyan Date: Wed, 29 Apr 2020 19:21:17 +0400 Subject: [PATCH 1/2] feat: create a new method for more files upload --- schemas/add.json | 172 +++++++++++++++++++++++++++++++++ src/actions/add.js | 231 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 403 insertions(+) create mode 100644 schemas/add.json create mode 100644 src/actions/add.js diff --git a/schemas/add.json b/schemas/add.json new file mode 100644 index 00000000..126fab3f --- /dev/null +++ b/schemas/add.json @@ -0,0 +1,172 @@ +{ + "$id": "add", + "type": "object", + "required": [ + "files", + "meta", + "username", + "uploadId" + ], + "properties": { + "files": { + "type": "array", + "minItems": 1, + "items": { + "anyOf": [ + { "$ref": "common#/definitions/cp-binary" }, + { "$ref": "common#/definitions/cp-image" }, + { "$ref": "common#/definitions/image" }, + { "$ref": "common#/definitions/background-image" }, + { "$ref": "common#/definitions/arbitrary" }, + { "$ref": "common#/definitions/video" }, + { "$ref": "common#/definitions/cp-pack" }, + { "$ref": "common#/definitions/cp-masks" }, + { "$ref": "common#/definitions/cp-gltf" }, + { "$ref": "common#/definitions/cp-usdz" }, + { "$ref": "common#/definitions/c-ar-color" }, + { "$ref": "common#/definitions/c-ar-masks" } + ] + } + }, + "meta": { + "allOf": [ + { "$ref": "common#/definitions/meta" }, + { + "type": "object", + "required": ["name"], + "not": { + "required": ["alias"] + } + } + ] + }, + "postAction": { + "type": "object", + "additionalProperties": false, + "minProperties": 1, + "properties": { + "update": { + "allOf": [ + { "$ref": "common#/definitions/meta" }, + { "minProperties": 1 } + ] + } + } + }, + "uploadId": { + "$ref": "common#/definitions/uploadId" + }, + "username": { + "$ref": "common#/definitions/owner" + }, + "access": { + "type": "object", + "required": [ + "setPublic" + ], + "properties": { + "setPublic": { + "type": "boolean", + "default": false + } + } + }, + "origin": { + "type": "string", + "format": "uri" + }, + "resumable": { + "type": "boolean", + "default": true + }, + "expires": { + "type": "integer", + "default": 900 + }, + "temp": { + "type": "boolean", + "default": false + }, + "unlisted": { + "type": "boolean", + "default": false + }, + "uploadType": { + "type": "string", + "enum": [ + "simple", + "image", + "background", + "model", + "text", + "pdf" + ] + }, + "directOnly": { + "type": "boolean", + "default": false + } + }, + "switch": [{ + "if": { + "required": ["resumable"], + "properties": { "resumable": { "const": false } } + }, + "then": { + "not": { + "properties": { + "unlisted": { "const": true }, + "temp": { "const": true } + } + } + }, + "continue": true + }, { + "if": { + "required": ["uploadType"], + "properties": { "uploadType": { "const": "simple" } } + }, + "then": { + "properties": { + "files": { + "contains": { + "allOf": [{ + "$ref": "common#/definitions/cp-image" + }, { + "type": "object", + "properties": { + "type": { + "const": "c-preview" + } + } + }] + } + } + } + }, + "continue": true + }, { + "if": { + "required": ["uploadType"], + "properties": { "uploadType": { "const": "text" } } + }, + "then": { + "properties": { + "files": { + "items": { + "$ref": "common#/definitions/arbitrary" + } + } + } + }, + "continue": true + }, { + "if": { + "anyOf": [ + { "properties": { "unlisted": { "const": true } } }, + { "properties": { "temp": { "const": true } } } + ] + }, + "then": { "not": { "required": ["postAction"] } } + }] +} diff --git a/src/actions/add.js b/src/actions/add.js new file mode 100644 index 00000000..249be280 --- /dev/null +++ b/src/actions/add.js @@ -0,0 +1,231 @@ +const { ActionTransport } = require('@microfleet/core'); +const Promise = require('bluebird'); +const { v4: uuidv4 } = require('uuid'); +const { HttpStatusError } = require('common-errors'); +const md5 = require('md5'); +const sumBy = require('lodash/sumBy'); +const get = require('lodash/get'); +const handlePipeline = require('../utils/pipeline-error'); +// const getLock = require('../utils/acquire-lock'); +const fetchData = require('../utils/fetch-data'); +// const isProcessed = require('../utils/is-processed'); +// const isUnlisted = require('../utils/is-unlisted'); +const hasAccess = require('../utils/has-access'); +// const isAliasTaken = require('../utils/is-alias-taken'); +const stringify = require('../utils/stringify'); +const extension = require('../utils/extension'); +const isValidBackgroundOrigin = require('../utils/is-valid-background-origin'); +// const { bustCache } = require('../utils/bust-cache'); +const { + STATUS_PENDING, + STATUS_PROCESSING, + UPLOAD_DATA, + FILES_PUBLIC_FIELD, + FILES_TEMP_FIELD, + FILES_BUCKET_FIELD, + FILES_OWNER_FIELD, + FILES_UNLISTED_FIELD, + FILES_STATUS_FIELD, + FIELDS_TO_STRINGIFY, + FILES_INDEX_TEMP, + FILES_POST_ACTION, + FILES_DIRECT_ONLY_FIELD, + FILES_CONTENT_LENGTH_FIELD, + FILES_DATA_INDEX_KEY, +} = require('../constant'); + +/** + * Initiates upload + * @param {Object} opts + * @param {Object} opts.params + * @param {String} [opts.params.origin] + * @param {Array} opts.params.files + * @param {Object} opts.params.meta + * @param {Boolean} [opts.params.directOnly=false] + * @param {Boolean} [opts.params.unlisted=false] + * @param {Boolean} [opts.params.temp=false] + * @return {Promise} + */ +async function addMoreFiles({ params }) { + const { + files, + meta, + username, + uploadId, + temp, + unlisted, + origin, + resumable, + expires, + uploadType, + postAction, + directOnly, + } = params; + + const { redis, config: { uploadTTL } } = this; + + this.log.info({ params }, 'preparing upload'); + + const provider = this.provider('upload', params); + const prefix = md5(username); + const isPublic = get(params, 'access.setPublic', false); + const bucketName = provider.bucket.name; + const uploadKey = FILES_DATA_INDEX_KEY(uploadId); + + await Promise + .bind(this, ['files:upload:pre', params]) + .spread(this.hook) + // do some extra meta validation + .return(meta) + .tap(isValidBackgroundOrigin) + .return(uploadKey) + .then(fetchData) + .then(hasAccess(username)) + .tap(async ({ status }) => { + if (status === STATUS_PENDING || status === STATUS_PROCESSING) { + throw new HttpStatusError(409, 'file is being processed or upload has not been finished yet'); + } + + this.log.info({ params }, 'preprocessed upload'); + + const parts = await Promise.map(files, async ({ md5Hash, type, ...rest }) => { + // generate filename + const filename = [ + // name + [prefix, uploadId, uuidv4()].join('/'), + // extension + extension(type, rest.contentType).slice(1), + ].join('.'); + + const metadata = { + ...rest, + md5Hash: Buffer.from(md5Hash, 'hex').toString('base64'), + [FILES_BUCKET_FIELD]: bucketName, + }; + + // this is an override, because safari has a bug: + // it doesn't decode gzip encoding when contentType is not one of + // it's supported ones + if (type === 'c-bin') { + metadata.contentType = 'text/plain'; + } + + // basic extension headers + const extensionHeaders = Object.create(null); + + if (isPublic) { + extensionHeaders['x-goog-acl'] = 'public-read'; + } + + let location; + if (resumable) { + location = await provider.initResumableUpload({ + filename, + origin, + public: isPublic, + metadata: { + ...metadata, + }, + }); + } else { + // simple upload + location = await provider.createSignedURL({ + action: 'write', + md5: metadata.md5Hash, + type: metadata.contentType, + resource: filename, + extensionHeaders, + expires: Date.now() + (expires * 1000), + }); + } + + return { + ...metadata, + type, + filename, + location, + }; + }); + + const serialized = Object.create(null); + for (const field of FIELDS_TO_STRINGIFY) { + stringify(meta, field, serialized); + } + + const fileData = { + ...meta, + ...serialized, + uploadId, + startedAt: Date.now(), + files: JSON.stringify(parts), + parts: files.length, + [FILES_CONTENT_LENGTH_FIELD]: sumBy(parts, 'contentLength'), + [FILES_STATUS_FIELD]: STATUS_PENDING, + [FILES_OWNER_FIELD]: username, + [FILES_BUCKET_FIELD]: bucketName, + }; + + if (uploadType) { + fileData.uploadType = uploadType; + } + + if (isPublic) { + fileData[FILES_PUBLIC_FIELD] = 1; + } + + if (temp) { + fileData[FILES_TEMP_FIELD] = 1; + } + + if (unlisted) { + fileData[FILES_UNLISTED_FIELD] = 1; + } + + if (directOnly) { + fileData[FILES_DIRECT_ONLY_FIELD] = 1; + } + + const pipeline = redis.pipeline(); + + pipeline + .sadd(FILES_INDEX_TEMP, uploadId) + .hmset(uploadKey, fileData) + .expire(uploadKey, uploadTTL); + + parts.forEach((part) => { + const partKey = `${UPLOAD_DATA}:${part.filename}`; + pipeline + .hmset(partKey, { + [FILES_BUCKET_FIELD]: bucketName, + [FILES_STATUS_FIELD]: STATUS_PENDING, + uploadId, + }) + .expire(partKey, uploadTTL); + }); + + // in case we have post action provided - save it for when we complete "finish" action + if (postAction) { + const postActionKey = `${FILES_POST_ACTION}:${uploadId}`; + pipeline.set(postActionKey, JSON.stringify(postAction), 'EX', uploadTTL); + } + + this.log.info({ params }, 'created signed urls and preparing to save them to database'); + + handlePipeline(await pipeline.exec()); + + const data = { + ...fileData, + ...meta, + files: parts, + }; + + await Promise + .bind(this, ['files:upload:post', data]) + .spread(this.hook); + + return data; + }); +} + +addMoreFiles.transports = [ActionTransport.amqp]; +module.exports = addMoreFiles; From 2a45dba9dcba3c395b724ce1ad98df0a7fae163b Mon Sep 17 00:00:00 2001 From: Tigran Avetisyan Date: Wed, 29 Apr 2020 19:24:13 +0400 Subject: [PATCH 2/2] feat: add uploadId param description --- src/actions/add.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/actions/add.js b/src/actions/add.js index 249be280..2db6e274 100644 --- a/src/actions/add.js +++ b/src/actions/add.js @@ -41,6 +41,7 @@ const { * @param {String} [opts.params.origin] * @param {Array} opts.params.files * @param {Object} opts.params.meta + * @param {String} opts.params.uploadId * @param {Boolean} [opts.params.directOnly=false] * @param {Boolean} [opts.params.unlisted=false] * @param {Boolean} [opts.params.temp=false]