feat: update user and organization metadata

rfcs/inactive_users/user_and_organization_meta_update.md

@@ -0,0 +1,24 @@
+# User/Organization metadata update
+## Overview and Motivation
+When user or organization metadata is updated, the Service should track audiences with assigned metadata.
+For each assigned meta hash always exists a single `audience`, but there is no list of `audiences` assigned to the user or organization.
+
+To achieve this ability, I advise these updates: 
+
+## Audience lists
+Audiences stored in sets with names created from `USERS_AUDIENCE` or `ORGANISATION_AUDIENCE` constants and `Id`
+(e.g.: `{ms-users}10110110111!audiences`). Both keys contain `audience` names that are currently have assigned values.
+
+The `audience` list will be updated on each update of the metadata.
+
+## Metadata Handling classes
+Service logic is updated to use 2 specific classes that will perform all CRUD operations on User or Organization metadata.
+
+* Classes located in: `utils/metadata/{user|organization}.js`.
+* Both classes use same [Redis backend](#redis-metadata-backend-class).
+
+## Redis Metadata Backend class
+The class performs all work on metadata using Redis DB as a backend.
+
+## Notice
+* All User or Organization metadata operations should be performed using Provided classes otherwise, audiences won't be tracked.

src/actions/alias.js

@@ -6,9 +6,10 @@ const isActive = require('../utils/is-active');
 const isBanned = require('../utils/is-banned');
 const key = require('../utils/key');
 const handlePipeline = require('../utils/pipeline-error');
+const UserMetadata = require('../utils/metadata/user');
+
 const {
   USERS_DATA,
-  USERS_METADATA,
   USERS_ALIAS_TO_ID,
   USERS_ID_FIELD,
   USERS_ALIAS_FIELD,
@@ -69,10 +70,12 @@ async function assignAlias({ params }) {
       return Promise.reject(err);
     }
 
-    const pipeline = redis.pipeline([
-      ['hset', key(userId, USERS_DATA), USERS_ALIAS_FIELD, alias],
-      ['hset', key(userId, USERS_METADATA, defaultAudience), USERS_ALIAS_FIELD, JSON.stringify(alias)],
-    ]);
+    const pipeline = redis.pipeline();
+
+    pipeline.hset(key(userId, USERS_DATA), USERS_ALIAS_FIELD, alias);
+    UserMetadata
+      .using(userId, defaultAudience, pipeline)
+      .update(USERS_ALIAS_FIELD, JSON.stringify(alias));
 
     if (activeUser) {
       pipeline.sadd(USERS_PUBLIC_INDEX, username);

src/actions/ban.js

@@ -4,9 +4,10 @@ const mapValues = require('lodash/mapValues');
 const redisKey = require('../utils/key.js');
 const { getInternalData } = require('../utils/userData');
 const handlePipeline = require('../utils/pipeline-error');
+const UserMetadata = require('../utils/metadata/user');
+
 const {
-  USERS_DATA, USERS_METADATA,
-  USERS_BANNED_FLAG, USERS_TOKENS, USERS_BANNED_DATA,
+  USERS_DATA, USERS_BANNED_FLAG, USERS_TOKENS, USERS_BANNED_DATA,
 } = require('../constants.js');
 
 // helper
@@ -25,26 +26,32 @@ function lockUser({
       remoteip: remoteip || '',
     },
   };
+  const pipeline = redis.pipeline();
+
+  pipeline.hset(redisKey(id, USERS_DATA), USERS_BANNED_FLAG, 'true');
+  // set .banned on metadata for filtering & sorting users by that field
+  UserMetadata
+    .using(id, defaultAudience, pipeline)
+    .updateMulti(mapValues(data, stringify));
+  pipeline.del(redisKey(id, USERS_TOKENS));
 
-  return redis
-    .pipeline()
-    .hset(redisKey(id, USERS_DATA), USERS_BANNED_FLAG, 'true')
-    // set .banned on metadata for filtering & sorting users by that field
-    .hmset(redisKey(id, USERS_METADATA, defaultAudience), mapValues(data, stringify))
-    .del(redisKey(id, USERS_TOKENS))
-    .exec();
+  return pipeline.exec();
 }
 
 function unlockUser({ id }) {
   const { redis, config } = this;
   const { jwt: { defaultAudience } } = config;
+  const pipeline = redis.pipeline();
 
-  return redis
-    .pipeline()
-    .hdel(redisKey(id, USERS_DATA), USERS_BANNED_FLAG)
-    // remove .banned on metadata for filtering & sorting users by that field
-    .hdel(redisKey(id, USERS_METADATA, defaultAudience), 'banned', USERS_BANNED_DATA)
-    .exec();
+  pipeline.hdel(redisKey(id, USERS_DATA), USERS_BANNED_FLAG);
+  // remove .banned on metadata for filtering & sorting users by that field
+  UserMetadata
+    .using(id, defaultAudience, pipeline)
+    .delete([
+      'banned',
+      USERS_BANNED_DATA,
+    ]);
+  return pipeline.exec();
 }
 
 /**

src/actions/organization/delete.js

@@ -3,6 +3,7 @@ const snakeCase = require('lodash/snakeCase');
 const redisKey = require('../../utils/key');
 const handlePipeline = require('../../utils/pipeline-error');
 const { checkOrganizationExists, getInternalData } = require('../../utils/organization');
+const OrganizationMetadata = require('../../utils/metadata/organization');
 const {
   ORGANIZATIONS_DATA,
   ORGANIZATIONS_METADATA,
@@ -32,11 +33,15 @@ async function deleteOrganization({ params }) {
   const organizationMembersListKey = redisKey(organizationId, ORGANIZATIONS_MEMBERS);
   const organizationMembersIds = await redis.zrange(organizationMembersListKey, 0, -1);
   const organization = await getInternalData.call(this, organizationId);
+  const organizationMetadata = new OrganizationMetadata(redis);
 
   const pipeline = redis.pipeline();
 
   pipeline.del(redisKey(organizationId, ORGANIZATIONS_DATA));
   pipeline.del(redisKey(organizationId, ORGANIZATIONS_METADATA, audience));
+  // delete organization audiences index
+  pipeline.del(organizationMetadata.audience.getAudienceKey(organizationId));
+
   pipeline.srem(ORGANIZATIONS_INDEX, organizationId);
   if (organizationMembersIds) {
     organizationMembersIds.forEach((memberId) => {

src/actions/organization/members/permission.js

@@ -5,6 +5,7 @@ const { checkOrganizationExists } = require('../../../utils/organization');
 const redisKey = require('../../../utils/key');
 const handlePipeline = require('../../../utils/pipeline-error');
 const getUserId = require('../../../utils/userData/get-user-id');
+const UserMetadata = require('../../../utils/metadata/user');
 const { ErrorUserNotMember, USERS_METADATA, ORGANIZATIONS_MEMBERS } = require('../../../constants');
 
 /**
@@ -41,7 +42,10 @@ async function setOrganizationMemberPermission({ params }) {
   permissions = JSON.stringify(permissions);
 
   const pipeline = redis.pipeline();
-  pipeline.hset(memberMetadataKey, organizationId, permissions);
+
+  UserMetadata
+    .using(userId, audience, pipeline)
+    .update(organizationId, permissions);
   pipeline.hset(redisKey(organizationId, ORGANIZATIONS_MEMBERS, userId), 'permissions', permissions);
 
   return pipeline.exec().then(handlePipeline);

src/actions/organization/members/remove.js

@@ -3,6 +3,7 @@ const redisKey = require('../../../utils/key');
 const getUserId = require('../../../utils/userData/get-user-id');
 const handlePipeline = require('../../../utils/pipeline-error');
 const { checkOrganizationExists } = require('../../../utils/organization');
+const UserMetadata = require('../../../utils/metadata/user');
 const {
   ORGANIZATIONS_MEMBERS,
   USERS_METADATA,
@@ -36,7 +37,9 @@ async function removeMember({ params }) {
   const pipeline = redis.pipeline();
   pipeline.del(memberKey);
   pipeline.zrem(redisKey(organizationId, ORGANIZATIONS_MEMBERS), memberKey);
-  pipeline.hdel(memberMetadataKey, organizationId);
+  UserMetadata
+    .using(userId, audience, pipeline)
+    .delete(organizationId);
 
   return pipeline.exec().then(handlePipeline);
 }

src/actions/register.js

@@ -9,7 +9,7 @@ const reduce = require('lodash/reduce');
 const last = require('lodash/last');
 
 // internal deps
-const setMetadata = require('../utils/update-metadata');
+const UserMetadata = require('../utils/metadata/user');
 const redisKey = require('../utils/key');
 const jwt = require('../utils/jwt');
 const isDisposable = require('../utils/is-disposable');
@@ -212,18 +212,17 @@ async function performRegistration({ service, params }) {
   }
 
   await pipeline.exec().then(handlePipeline);
-
-  await setMetadata.call(service, {
-    userId,
-    audience,
-    metadata: audience.map((metaAudience) => ({
-      $set: Object.assign(metadata[metaAudience] || {}, metaAudience === defaultAudience && {
-        [USERS_ID_FIELD]: userId,
-        [USERS_USERNAME_FIELD]: username,
-        [USERS_CREATED_FIELD]: created,
-      }),
-    })),
-  });
+  await UserMetadata
+    .using(userId, audience, service.redis)
+    .batchUpdate({
+      metadata: audience.map((metaAudience) => ({
+        $set: Object.assign(metadata[metaAudience] || {}, metaAudience === defaultAudience && {
+          [USERS_ID_FIELD]: userId,
+          [USERS_USERNAME_FIELD]: username,
+          [USERS_CREATED_FIELD]: created,
+        }),
+      })),
+    });
 
   // assign alias
   if (alias) {

src/actions/remove.js

@@ -7,6 +7,7 @@ const key = require('../utils/key');
 const { getInternalData } = require('../utils/userData');
 const getMetadata = require('../utils/get-metadata');
 const handlePipeline = require('../utils/pipeline-error');
+const UserMetadata = require('../utils/metadata/user');
 const {
   USERS_INDEX,
   USERS_PUBLIC_INDEX,
@@ -15,7 +16,6 @@ const {
   USERS_USERNAME_TO_ID,
   USERS_USERNAME_FIELD,
   USERS_DATA,
-  USERS_METADATA,
   USERS_TOKENS,
   USERS_ID_FIELD,
   USERS_ALIAS_FIELD,
@@ -72,6 +72,8 @@ async function removeUser({ params }) {
   const alias = internal[USERS_ALIAS_FIELD];
   const userId = internal[USERS_ID_FIELD];
   const resolvedUsername = internal[USERS_USERNAME_FIELD];
+  const metaAudiences = await UserMetadata.using(userId, null, redis).getAudience();
+  const userMetadata = UserMetadata.using(userId, null, transaction);
 
   if (alias) {
     transaction.hdel(USERS_ALIAS_TO_ID, alias.toLowerCase(), alias);
@@ -94,7 +96,9 @@ async function removeUser({ params }) {
 
   // remove metadata & internal data
   transaction.del(key(userId, USERS_DATA));
-  transaction.del(key(userId, USERS_METADATA, audience));
+  for (const metaAudience of metaAudiences) {
+    userMetadata.deleteMetadata(metaAudience);
+  }
 
   // remove auth tokens
   transaction.del(key(userId, USERS_TOKENS));

src/actions/updateMetadata.js

@@ -1,6 +1,5 @@
-const omit = require('lodash/omit');
 const Promise = require('bluebird');
-const updateMetadata = require('../utils/update-metadata');
+const UserMetadata = require('../utils/metadata/user');
 const { getUserId } = require('../utils/userData');
 
 /**
@@ -19,12 +18,15 @@ const { getUserId } = require('../utils/userData');
  * @apiParam (Payload) {Object} [script] - if present will be called with passed metadata keys & username, provides direct scripting access.
  *   Be careful with granting access to this function.
  */
-module.exports = function updateMetadataAction(request) {
-  return Promise
+module.exports = async function updateMetadataAction(request) {
+  const { username: _, audience, ...updateParams } = request.params;
+  const userId = await Promise
     .bind(this, request.params.username)
-    .then(getUserId)
-    .then((userId) => ({ ...omit(request.params, 'username'), userId }))
-    .then(updateMetadata);
+    .then(getUserId);
+
+  return UserMetadata
+    .using(userId, audience, this.redis)
+    .batchUpdate(updateParams);
 };
 
 module.exports.transports = [require('@microfleet/core').ActionTransport.amqp];

src/auth/oauth/utils/attach.js

@@ -1,13 +1,13 @@
 const get = require('lodash/get');
 const redisKey = require('../../../utils/key');
-const updateMetadata = require('../../../utils/update-metadata');
+const UserMetadata = require('../../../utils/metadata/user');
 const handlePipeline = require('../../../utils/pipeline-error');
 const {
   USERS_SSO_TO_ID,
   USERS_DATA,
 } = require('../../../constants');
 
-module.exports = function attach(account, user) {
+module.exports = async function attach(account, user) {
   const { redis, config } = this;
   const { id: userId } = user;
   const {
@@ -23,17 +23,18 @@ module.exports = function attach(account, user) {
   // link uid to user id
   pipeline.hset(USERS_SSO_TO_ID, uid, userId);
 
-  return pipeline.exec().then(handlePipeline)
-    .bind(this)
-    .return({
-      userId,
-      audience,
-      metadata: {
-        $set: {
-          [provider]: profile,
-        },
+  handlePipeline(await pipeline.exec());
+
+  const updateParams = {
+    metadata: {
+      $set: {
+        [provider]: profile,
       },
-    })
-    .then(updateMetadata)
-    .return(profile);
+    },
+  };
+  await UserMetadata
+    .using(userId, audience, redis)
+    .batchUpdate(updateParams);
+
+  return profile;
 };

src/auth/oauth/utils/detach.js

@@ -2,15 +2,15 @@ const Errors = require('common-errors');
 
 const get = require('../../../utils/get-value');
 const redisKey = require('../../../utils/key');
-const updateMetadata = require('../../../utils/update-metadata');
+const UserMetadata = require('../../../utils/metadata/user');
 const handlePipeline = require('../../../utils/pipeline-error');
 
 const {
   USERS_SSO_TO_ID,
   USERS_DATA,
 } = require('../../../constants');
 
-module.exports = function detach(provider, userData) {
+module.exports = async function detach(provider, userData) {
   const { id: userId } = userData;
   const { redis, config } = this;
   const audience = get(config, 'jwt.defaultAudience');
@@ -28,16 +28,17 @@ module.exports = function detach(provider, userData) {
   // delete account reference
   pipeline.hdel(USERS_SSO_TO_ID, uid);
 
-  return pipeline.exec().then(handlePipeline)
-    .bind(this)
-    .return({
-      userId,
-      audience,
-      metadata: {
-        $remove: [
-          provider,
-        ],
-      },
-    })
-    .then(updateMetadata);
+  handlePipeline(await pipeline.exec());
+
+  const updateParams = {
+    metadata: {
+      $remove: [
+        provider,
+      ],
+    },
+  };
+
+  return UserMetadata
+    .using(userId, audience, redis)
+    .batchUpdate(updateParams);
 };

src/constants.js

@@ -18,6 +18,7 @@ module.exports = exports = {
   // hashes
   USERS_DATA: 'data',
   USERS_METADATA: 'metadata',
+  USERS_AUDIENCE: 'users-audiences',
   USERS_TOKENS: 'tokens',
   USERS_API_TOKENS: 'api-tokens',
   USERS_API_TOKENS_ZSET: 'api-tokens-set',
@@ -26,6 +27,7 @@ module.exports = exports = {
   USERS_ORGANIZATIONS: 'user-organizations',
   ORGANIZATIONS_DATA: 'data',
   ORGANIZATIONS_METADATA: 'metadata',
+  ORGANIZATIONS_AUDIENCE: 'organization-audiences',
   ORGANIZATIONS_MEMBERS: 'members',
 
   // standard JWT with TTL