From 4960876fff279830a15b3ad0c562abd9768ed9a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Thu, 11 Jan 2024 08:59:39 +0000 Subject: [PATCH 1/4] FT: Provide AWS KMS connector for bucket ciphering Add configuration mechanism for the new AWS KMS connector. Depends on changes in Arsenal to have support of this new connector. --- lib/Config.js | 28 ++++++++++++++++++++++++++++ lib/kms/wrapper.js | 5 +++++ 2 files changed, 33 insertions(+) diff --git a/lib/Config.js b/lib/Config.js index 6e1e853889..361673f227 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -1092,6 +1092,34 @@ class Config extends EventEmitter { } } + // Use env variables as default values. + // We use the same env variables as the AWS CLI does. + // Please note that if no config is specified here, the AWS Client + // seems to fallback on the local AWS configuration files + // (those contained in ~/.aws directory) + this.kms_aws = { + region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION, + endpoint: process.env.AWS_ENDPOINT_URL_KMS || process.env.AWS_ENDPOINT_URL, + ak: process.env.AWS_ACCESS_KEY_ID, + sk: process.env.AWS_SECRET_ACCESS_KEY + }; + if (config.kms_aws) { + const {region, endpoint, ak, sk} = config.kms_aws; + if (region) { + this.kms_aws.region = region; + } + if (endpoint) { + this.kms_aws.endpoint = endpoint; + } + /* Configure credentials. + Currently only support AK+SK authentication, both must be supplied. + */ + if (ak && sk) { + this.kms_aws.ak = ak; + this.kms_aws.sk = sk; + } + } + this.healthChecks = defaultHealthChecks; if (config.healthChecks && config.healthChecks.allowFrom) { assert(config.healthChecks.allowFrom instanceof Array, diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index 4a927f9d94..c2163ca9ae 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -7,6 +7,7 @@ const logger = require('../utilities/logger'); const inMemory = require('./in_memory/backend').backend; const file = require('./file/backend'); const KMIPClient = require('arsenal').network.kmipClient; +const AWSClient = require('arsenal').network.awsClient; const Common = require('./common'); let scalityKMS; let scalityKMSImpl; @@ -42,6 +43,10 @@ if (config.backends.kms === 'mem') { } client = new KMIPClient(kmipConfig); implName = 'kmip'; +} else if (config.backends.kms === 'aws') { + const awsConfig = { kms_aws: config.kms_aws }; + client = new AWSClient(awsConfig); + implName = 'aws'; } else { throw new Error('KMS backend is not configured'); } From 394e5f25453865192e165f1285d394e63564c3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Thu, 18 Jan 2024 13:33:44 +0000 Subject: [PATCH 2/4] FT: ciphering, use generateDataKey operation of the KMS backend when available Up to now, the datakey was always generated using a locally generated random number. This commit allow to use the "generateDataKey" operation of a KMS when it is implemented. It fallback to random number generation if not available. The benefit of generating the datakey in the KMS is a better entropy source resulting in a "better" datakey. --- lib/kms/wrapper.js | 74 +++++++++++++++++++++++++++------------------- 1 file changed, 43 insertions(+), 31 deletions(-) diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index c2163ca9ae..4778dbeca6 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -136,19 +136,6 @@ class KMS { }); } - /** - * - * @param {object} log - logger object - * @returns {buffer} newKey - a data key - */ - static createDataKey(log) { - log.debug('creating a new data key'); - const newKey = Common.createDataKey(); - log.trace('data key created by the kms'); - return newKey; - } - - /** * createCipherBundle * @param {object} serverSideEncryptionInfo - info for encryption @@ -167,8 +154,6 @@ class KMS { */ static createCipherBundle(serverSideEncryptionInfo, log, cb) { - const dataKey = this.createDataKey(log); - const { algorithm, configuredMasterKeyId, masterKeyId: bucketMasterKeyId } = serverSideEncryptionInfo; let masterKeyId = bucketMasterKeyId; @@ -186,27 +171,54 @@ class KMS { }; async.waterfall([ - function cipherDataKey(next) { - log.debug('ciphering a data key'); - return client.cipherDataKey(cipherBundle.cryptoScheme, - cipherBundle.masterKeyId, - dataKey, log, (err, cipheredDataKey) => { - if (err) { - log.debug('error from kms', - { implName, error: err }); - return next(err); - } - log.trace('data key ciphered by the kms'); - return next(null, cipheredDataKey); - }); + function generateDataKey(next) { + /* There are 2 ways of generating a datakey : + - using the generateDataKey of the KMS backend if it exists + (currently only implemented for the AWS KMS backend). This is + the prefered solution since a dedicated KMS should offer a better + entropy for generating random content. + - using local random number generation, and then use the KMS to + encrypt the datakey. This method is used when the KMS backend doesn't + provide the generateDataKey method. + */ + if (client.generateDataKey) { + log.debug('creating a data key using the KMS'); + return client.generateDataKey(cipherBundle.cryptoScheme, + cipherBundle.masterKeyId, + log, (err, plainTextDataKey, cipheredDataKey) => { + if (err) { + log.debug('error from kms', + { implName, error: err }); + return next(err); + } + log.trace('data key generated by the kms'); + return next(null, plainTextDataKey, cipheredDataKey); + }) + } else { + log.debug('creating a new data key'); + const dataKey = Common.createDataKey(); + + log.debug('ciphering the data key'); + return client.cipherDataKey(cipherBundle.cryptoScheme, + cipherBundle.masterKeyId, + dataKey, log, (err, cipheredDataKey) => { + if (err) { + log.debug('error from kms', + { implName, error: err }); + return next(err); + } + log.trace('data key ciphered by the kms'); + return next(null, dataKey, cipheredDataKey); + }); + } }, - function createCipher(cipheredDataKey, next) { + function createCipher(plainTextDataKey, cipheredDataKey, next) { log.debug('creating a cipher'); cipherBundle.cipheredDataKey = cipheredDataKey.toString('base64'); return Common.createCipher(cipherBundle.cryptoScheme, - dataKey, 0, log, (err, cipher) => { - dataKey.fill(0); + plainTextDataKey, 0, log, (err, cipher) => { + plainTextDataKey.fill(0); if (err) { log.debug('error from kms', { implName, error: err }); From e9c80f58a39471de3461b1fd650271da51906e93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Mon, 24 Jun 2024 14:54:44 +0000 Subject: [PATCH 3/4] FT: AWS KMS, fix lint errors + env var renaming after 1st code review --- lib/Config.js | 25 +++++++++++++------------ lib/kms/wrapper.js | 4 ++-- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/Config.js b/lib/Config.js index 361673f227..92de14749f 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -1093,30 +1093,31 @@ class Config extends EventEmitter { } // Use env variables as default values. - // We use the same env variables as the AWS CLI does. + // We use the same env variables as the AWS CLI does but prefixed with "KMS_", + // allowing distinct endpoints betweens AWS compatibility components. // Please note that if no config is specified here, the AWS Client // seems to fallback on the local AWS configuration files // (those contained in ~/.aws directory) - this.kms_aws = { - region: process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION, - endpoint: process.env.AWS_ENDPOINT_URL_KMS || process.env.AWS_ENDPOINT_URL, - ak: process.env.AWS_ACCESS_KEY_ID, - sk: process.env.AWS_SECRET_ACCESS_KEY + this.kmsAWS = { + region: process.env.KMS_AWS_REGION || process.env.KMS_AWS_DEFAULT_REGION, + endpoint: process.env.KMS_AWS_ENDPOINT_URL_KMS || process.env.KMS_AWS_ENDPOINT_URL, + ak: process.env.KMS_AWS_ACCESS_KEY_ID, + sk: process.env.KMS_AWS_SECRET_ACCESS_KEY, }; - if (config.kms_aws) { - const {region, endpoint, ak, sk} = config.kms_aws; + if (config.kmsAWS) { + const { region, endpoint, ak, sk } = config.kmsAWS; if (region) { - this.kms_aws.region = region; + this.kmsAWS.region = region; } if (endpoint) { - this.kms_aws.endpoint = endpoint; + this.kmsAWS.endpoint = endpoint; } /* Configure credentials. Currently only support AK+SK authentication, both must be supplied. */ if (ak && sk) { - this.kms_aws.ak = ak; - this.kms_aws.sk = sk; + this.kmsAWS.ak = ak; + this.kmsAWS.sk = sk; } } diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index 4778dbeca6..c814b02876 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -44,7 +44,7 @@ if (config.backends.kms === 'mem') { client = new KMIPClient(kmipConfig); implName = 'kmip'; } else if (config.backends.kms === 'aws') { - const awsConfig = { kms_aws: config.kms_aws }; + const awsConfig = { kmsAWS: config.kmsAWS }; client = new AWSClient(awsConfig); implName = 'aws'; } else { @@ -193,7 +193,7 @@ class KMS { } log.trace('data key generated by the kms'); return next(null, plainTextDataKey, cipheredDataKey); - }) + }); } else { log.debug('creating a new data key'); const dataKey = Common.createDataKey(); From 6b9a95db80ee5f83dfd079561e7445c59d8c1a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Branca?= Date: Tue, 30 Jul 2024 09:53:58 +0000 Subject: [PATCH 4/4] AWS KMS: TLS configuration --- lib/Config.js | 46 ++++++++++++++++++++++++++++++++++++++++++++-- lib/kms/wrapper.js | 8 +++++--- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/lib/Config.js b/lib/Config.js index 92de14749f..6660a6a50d 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -491,7 +491,7 @@ class Config extends EventEmitter { } const tlsFilePath = (tlsFileName[0] === '/') ? tlsFileName - : path.join(this._basepath, tlsFileName); + : path.join(this._basePath, tlsFileName); let tlsFileContent; try { tlsFileContent = fs.readFileSync(tlsFilePath); @@ -502,6 +502,19 @@ class Config extends EventEmitter { return tlsFileContent; } + // Load TLS file or array of files + // if tlsFilename is a string, result will be a Buffer containing the file content + // if tlsFilename is an array of string, result will be an array of Buffer + _loadTlsFileArray(tlsFileName) { + let res; + if (Array.isArray(tlsFileName)) { + res = tlsFileName.map(this._loadTlsFile); + } else { + res = this._loadTlsFile(tlsFileName); + } + return res; + } + /** * Parse list of endpoints. * @param {string[] | undefined} listenOn - List of string of the form "ip:port" @@ -1105,7 +1118,7 @@ class Config extends EventEmitter { sk: process.env.KMS_AWS_SECRET_ACCESS_KEY, }; if (config.kmsAWS) { - const { region, endpoint, ak, sk } = config.kmsAWS; + const { region, endpoint, ak, sk, tls } = config.kmsAWS; if (region) { this.kmsAWS.region = region; } @@ -1119,6 +1132,35 @@ class Config extends EventEmitter { this.kmsAWS.ak = ak; this.kmsAWS.sk = sk; } + + if (tls) { + this.kmsAWS.tls = {}; + if (tls.rejectUnauthorized !== undefined) { + assert(typeof tls.rejectUnauthorized === 'boolean'); + this.kmsAWS.tls.rejectUnauthorized = tls.rejectUnauthorized; + } + // min & max TLS: One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' + // (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) + if (tls.minVersion !== undefined) { + assert(typeof tls.minVersion === 'string', + 'bad config: KMS AWS TLS minVersion must be a string'); + this.kmsAWS.tls.minVersion = tls.minVersion; + } + if (tls.maxVersion !== undefined) { + assert(typeof tls.maxVersion === 'string', + 'bad config: KMS AWS TLS maxVersion must be a string'); + this.kmsAWS.tls.maxVersion = tls.maxVersion; + } + if (tls.ca !== undefined) { + this.kmsAWS.tls.ca = this._loadTlsFileArray(tls.ca); + } + if (tls.cert !== undefined) { + this.kmsAWS.tls.cert = this._loadTlsFileArray(tls.cert); + } + if (tls.key !== undefined) { + this.kmsAWS.tls.key = this._loadTlsFileArray(tls.key); + } + } } this.healthChecks = defaultHealthChecks; diff --git a/lib/kms/wrapper.js b/lib/kms/wrapper.js index c814b02876..b081715798 100644 --- a/lib/kms/wrapper.js +++ b/lib/kms/wrapper.js @@ -181,9 +181,10 @@ class KMS { encrypt the datakey. This method is used when the KMS backend doesn't provide the generateDataKey method. */ + let res; if (client.generateDataKey) { log.debug('creating a data key using the KMS'); - return client.generateDataKey(cipherBundle.cryptoScheme, + res = client.generateDataKey(cipherBundle.cryptoScheme, cipherBundle.masterKeyId, log, (err, plainTextDataKey, cipheredDataKey) => { if (err) { @@ -199,7 +200,7 @@ class KMS { const dataKey = Common.createDataKey(); log.debug('ciphering the data key'); - return client.cipherDataKey(cipherBundle.cryptoScheme, + res = client.cipherDataKey(cipherBundle.cryptoScheme, cipherBundle.masterKeyId, dataKey, log, (err, cipheredDataKey) => { if (err) { @@ -210,7 +211,8 @@ class KMS { log.trace('data key ciphered by the kms'); return next(null, dataKey, cipheredDataKey); }); - } + } + return res; }, function createCipher(plainTextDataKey, cipheredDataKey, next) { log.debug('creating a cipher');