diff --git a/src/agent/block_store_speed.js b/src/agent/block_store_speed.js index 27c69aef47..1d480effe8 100644 --- a/src/agent/block_store_speed.js +++ b/src/agent/block_store_speed.js @@ -4,7 +4,7 @@ // const _ = require('lodash'); const argv = require('minimist')(process.argv); const cluster = require('cluster'); -const mongodb = require('mongodb'); +const mongo_utils = require('../util/mongo_utils'); const api = require('../api'); const config = require('../../config'); @@ -61,7 +61,7 @@ async function worker(client) { } async function write_block(client) { - const block_id = new mongodb.ObjectId(); + const block_id = new mongo_utils.ObjectId(); return client.block_store.write_block({ [RPC_BUFFERS]: { data: Buffer.allocUnsafe(argv.size) }, block_md: { diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index 0541b68f98..7c264eec31 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -2,6 +2,7 @@ export as namespace nb; import * as fs from 'fs'; import * as mongodb from 'mongodb'; +import * as mongo_utils from '../util/mongo_utils'; import { EventEmitter } from 'events'; import { Readable, Writable } from 'stream'; import { IncomingMessage, ServerResponse } from 'http'; @@ -46,7 +47,7 @@ interface Base { toString?(): string; } -type ID = mongodb.ObjectID; +type ID = mongo_utils.ObjectId; type DBBuffer = mongodb.Binary | Buffer; interface System extends Base { @@ -719,8 +720,8 @@ interface DBClient { populate(docs: object[] | object, doc_path: string, collection: DBCollection, fields: object): Promise; resolve_object_ids_recursive(idmap: object, item: object): object; resolve_object_ids_paths(idmap: object, item: object, paths: string[], allow_missing: boolean): object; - new_object_id(): mongodb.ObjectId; - parse_object_id(id_str: string): mongodb.ObjectId; + new_object_id(): mongo_utils.ObjectId; + parse_object_id(id_str: string): mongo_utils.ObjectId; fix_id_type(doc: object[] | object): object[] | object; is_object_id(id: object[] | object): boolean; is_err_duplicate_key(err: object): boolean; diff --git a/src/server/analytic_services/activity_log_store.js b/src/server/analytic_services/activity_log_store.js index c0411730b9..678588d5a2 100644 --- a/src/server/analytic_services/activity_log_store.js +++ b/src/server/analytic_services/activity_log_store.js @@ -1,7 +1,7 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); const _ = require('lodash'); const db_client = require('../../util/db_client'); @@ -25,7 +25,7 @@ class ActivityLogStore { } make_activity_log_id(id_str) { - return new mongodb.ObjectID(id_str); + return new mongo_utils.ObjectId(id_str); } diff --git a/src/server/analytic_services/history_data_store.js b/src/server/analytic_services/history_data_store.js index 3ab9068cec..09de5a55f7 100644 --- a/src/server/analytic_services/history_data_store.js +++ b/src/server/analytic_services/history_data_store.js @@ -1,7 +1,7 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); // const dbg = require('../../util/debug_module')(__filename); const config = require('../../../config.js'); @@ -30,7 +30,7 @@ class HistoryDataStore { const time_stamp = new Date(); const record_expiration_date = new Date(time_stamp.getTime() - config.STATISTICS_COLLECTOR_EXPIRATION); const record = { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), time_stamp, system_snapshot: item, history_type: 'SYSTEM' diff --git a/src/server/bg_services/mirror_writer.js b/src/server/bg_services/mirror_writer.js index d58b777aab..ea3a35e076 100644 --- a/src/server/bg_services/mirror_writer.js +++ b/src/server/bg_services/mirror_writer.js @@ -3,7 +3,7 @@ const util = require('util'); const _ = require('lodash'); -const { ObjectId } = require('mongodb'); +const { ObjectId } = require('../../util/mongo_utils'); const config = require('../../../config'); const dbg = require('../../util/debug_module')(__filename); diff --git a/src/server/node_services/nodes_store.js b/src/server/node_services/nodes_store.js index 3b54631aa1..89425324cf 100644 --- a/src/server/node_services/nodes_store.js +++ b/src/server/node_services/nodes_store.js @@ -2,7 +2,7 @@ 'use strict'; const _ = require('lodash'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); const dbg = require('../../util/debug_module')(__filename); const node_schema = require('./node_schema'); @@ -24,7 +24,7 @@ class NodesStore { } make_node_id(id_str) { - return new mongodb.ObjectId(id_str); + return new mongo_utils.ObjectId(id_str); } is_connected() { diff --git a/src/server/notifications/alerts_log_store.js b/src/server/notifications/alerts_log_store.js index 77d7aa915e..8d30474ab9 100644 --- a/src/server/notifications/alerts_log_store.js +++ b/src/server/notifications/alerts_log_store.js @@ -1,7 +1,7 @@ /* Copyright (C) 2016 NooBaa */ 'use strict'; -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); const _ = require('lodash'); const P = require('../../util/promise'); const db_client = require('../../util/db_client'); @@ -23,7 +23,7 @@ class AlertsLogStore { } make_alert_log_id(id_str) { - return new mongodb.ObjectID(id_str); + return new mongo_utils.ObjectId(id_str); } create(alert_log) { @@ -94,12 +94,12 @@ class AlertsLogStore { let _id; if (ids) { - const obj_ids = ids.map(id => new mongodb.ObjectID(id)); + const obj_ids = ids.map(id => new mongo_utils.ObjectId(id)); _id = { $in: obj_ids }; } else if (till) { - _id = { $lt: new mongodb.ObjectID(till) }; + _id = { $lt: new mongo_utils.ObjectId(till) }; } else if (since) { - _id = { $gt: new mongodb.ObjectID(since) }; + _id = { $gt: new mongo_utils.ObjectId(since) }; } return _.omitBy({ diff --git a/src/server/object_services/md_store.js b/src/server/object_services/md_store.js index 14b716cc58..e6e1260587 100644 --- a/src/server/object_services/md_store.js +++ b/src/server/object_services/md_store.js @@ -7,7 +7,7 @@ const _ = require('lodash'); const assert = require('assert'); const moment = require('moment'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); const mime = require('mime-types'); const P = require('../../util/promise'); @@ -92,18 +92,18 @@ class MDStore { if (zero_suffix) { suffix = '0'.repeat(16); } else { - suffix = String(new mongodb.ObjectId()).slice(8, 24); + suffix = String(new mongo_utils.ObjectId()).slice(8, 24); } const hex_id = padded_hex_time + suffix; assert(padded_hex_time.length === 8); assert(suffix.length === 16); assert(hex_id.length === 24); assert(parseInt(padded_hex_time, 16) === Math.floor(time / 1000)); - return new mongodb.ObjectId(hex_id); + return new mongo_utils.ObjectId(hex_id); } is_valid_md_id(id_str) { - return mongodb.ObjectId.isValid(id_str); + return mongo_utils.ObjectId.isValid(id_str); } ///////////// @@ -2113,7 +2113,7 @@ function sort_list_uploads_with_delimiter(a, b) { * @returns {nb.ID} */ function make_md_id(id_str) { - return new mongodb.ObjectId(id_str); + return new mongo_utils.ObjectId(id_str); } diff --git a/src/server/system_services/config_file_store.js b/src/server/system_services/config_file_store.js index 7a9ab7d435..2c91d17971 100644 --- a/src/server/system_services/config_file_store.js +++ b/src/server/system_services/config_file_store.js @@ -4,7 +4,7 @@ const _ = require('lodash'); const dbg = require('../../util/debug_module')(__filename); const db_client = require('../../util/db_client'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); const config_file_schema = require('./schemas/config_file_schema'); const config_file_indexes = require('./schemas/config_file_indexes'); @@ -26,7 +26,7 @@ class ConfigFileStore { async insert(item) { dbg.log0(`insert`, item); _.defaults(item, { - _id: new mongodb.ObjectId() + _id: new mongo_utils.ObjectId() }); // There shouldn't be more than one record, this is being on the safe side this._config_files.validate(item); diff --git a/src/server/system_services/replication_store.js b/src/server/system_services/replication_store.js index 66b98e7a70..b9ee00c415 100644 --- a/src/server/system_services/replication_store.js +++ b/src/server/system_services/replication_store.js @@ -2,7 +2,7 @@ 'use strict'; const _ = require('lodash'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../util/mongo_utils'); const db_client = require('../../util/db_client'); const dbg = require('../../util/debug_module')(__filename); const replication_schema = require('./schemas/replication_configuration_schema'); @@ -25,7 +25,7 @@ class ReplicationStore { item = _.omitBy(item, _.isNil); dbg.log1(`insert_replication`, item); const record = { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), ...item }; this._replicationconfigs.validate(record); diff --git a/src/test/integration_tests/api/s3/test_lifecycle.js b/src/test/integration_tests/api/s3/test_lifecycle.js index 74d1b37455..cf607b5bb6 100644 --- a/src/test/integration_tests/api/s3/test_lifecycle.js +++ b/src/test/integration_tests/api/s3/test_lifecycle.js @@ -9,7 +9,7 @@ const { NodeHttpHandler } = require("@smithy/node-http-handler"); const util = require('util'); const mocha = require('mocha'); const assert = require('assert'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../../../util/mongo_utils'); const _ = require('lodash'); const crypto = require('crypto'); const stream = require('stream'); @@ -153,7 +153,7 @@ mocha.describe('lifecycle', () => { if (tagging) update.tagging = tagging; console.log('create_mock_object bucket', bucket, 'key', key, 'update', util.inspect(update)); - const id = new mongodb.ObjectId(obj_id); + const id = new mongo_utils.ObjectId(obj_id); console.log('create_mock_object id', id, 'obj_id', obj_id); const updateResult = await MDStore.instance().update_object_by_id(id, update); @@ -385,7 +385,7 @@ mocha.describe('lifecycle', () => { source_stream: readable_buffer(mp_data, split, finish), }); console.log("upload_multipart", resp); - multiparts_ids.push(new mongodb.ObjectId(resp.multipart_id)); + multiparts_ids.push(new mongo_utils.ObjectId(resp.multipart_id)); }; // upload the real multiparts we want to complete with await Promise.all(_.times(num_parts, @@ -395,7 +395,7 @@ mocha.describe('lifecycle', () => { // go back in time const update = { // eslint-disable-next-line new-cap - upload_started: new mongodb.ObjectId(moment().subtract(age, 'days').unix()), + upload_started: new mongo_utils.ObjectId(moment().subtract(age, 'days').unix()), }; console.log('create_mock_multipart_upload bucket', bucket, 'obj_id', obj_id, 'multiparts_ids', multiparts_ids); @@ -610,7 +610,7 @@ mocha.describe('lifecycle', () => { // everything but last will be aged, // For simple Expiration rule all version should be expired if (expire_all || i < version_count - 1) { - obj_upload_ids.push(new mongodb.ObjectId(obj_id)); + obj_upload_ids.push(new mongo_utils.ObjectId(obj_id)); } } // go back in time diff --git a/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js b/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js index bf314a0adc..89d9ec2106 100644 --- a/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js +++ b/src/test/integration_tests/internal/test_agent_blocks_reclaimer.js @@ -9,7 +9,7 @@ const _ = require('lodash'); const mocha = require('mocha'); const assert = require('assert'); const crypto = require('crypto'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../../util/mongo_utils'); const P = require('../../../util/promise'); const config = require('../../../../config'); @@ -246,13 +246,13 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should mark reclaimed on deleted nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), rpc_address: 'n2n://SlothTown', online: false, deleted: new Date() }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), node: nodes[0]._id, deleted: new Date() }]; @@ -271,12 +271,12 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should not mark reclaimed on offline nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), rpc_address: 'n2n://SlothTown', online: false, }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), node: nodes[0]._id, deleted: new Date(), fail_to_delete: true @@ -296,14 +296,14 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should mark reclaimed on non existing nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), rpc_address: 'n2n://SlothTown', online: true, }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), // Non existing node on purpose - node: new mongodb.ObjectId(), + node: new mongo_utils.ObjectId(), deleted: new Date() }]; const reclaimer_mock = @@ -321,16 +321,16 @@ mocha.describe('mocked agent_blocks_reclaimer', function() { mocha.it('should not mark reclaimed on failure to delete', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), rpc_address: 'n2n://SlothTown', online: true, }]; const blocks = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), node: nodes[0]._id, deleted: new Date() }, { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), node: nodes[0]._id, deleted: new Date(), fail_to_delete: true diff --git a/src/test/integration_tests/internal/test_agent_blocks_verifier.js b/src/test/integration_tests/internal/test_agent_blocks_verifier.js index 635fdcbf45..6dceef73d1 100644 --- a/src/test/integration_tests/internal/test_agent_blocks_verifier.js +++ b/src/test/integration_tests/internal/test_agent_blocks_verifier.js @@ -13,7 +13,7 @@ const assert = require('assert'); const AgentBlocksVerifier = require('../../../server/bg_services/agent_blocks_verifier').AgentBlocksVerifier; const db_client = require('../../../util/db_client'); const schema_utils = require('../../../util/schema_utils'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../../util/mongo_utils'); const config = require('../../../../config'); const { ChunkDB, BlockDB } = require('../../../server/object_services/map_db_types'); @@ -121,15 +121,15 @@ class VerifierMock extends AgentBlocksVerifier { mocha.describe('mocked agent_blocks_verifier', function() { - const tier_id = new mongodb.ObjectId(); - const bucket_id = new mongodb.ObjectId(); - const system_id = new mongodb.ObjectId(); + const tier_id = new mongo_utils.ObjectId(); + const bucket_id = new mongo_utils.ObjectId(); + const system_id = new mongo_utils.ObjectId(); mocha.it('should verify blocks on nodes', async function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [make_node('bla2', false)]; const chunk_coder_configs = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), chunk_coder_config: { frag_digest_type: 'sloth_type' } @@ -151,7 +151,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { const self = this; // eslint-disable-line no-invalid-this const nodes = [make_node('bla1', true)]; const chunk_coder_configs = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), chunk_coder_config: { frag_digest_type: 'sloth_type' } @@ -174,7 +174,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { const self = this; // eslint-disable-line no-invalid-this // const nodes = [make_node('node1')]; const chunk_coder_configs = [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), system: system_id, chunk_coder_config: { frag_digest_type: 'sloth_type' @@ -182,7 +182,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { }]; const chunks = [make_schema_chunk(chunk_coder_configs[0]._id, [make_schema_frag()])]; const pools = [make_schema_pool('pool1')]; - const blocks = [make_schema_block(chunks[0].frags[0]._id, chunks[0]._id, new mongodb.ObjectId(), pools[0]._id)]; + const blocks = [make_schema_block(chunks[0].frags[0]._id, chunks[0]._id, new mongo_utils.ObjectId(), pools[0]._id)]; const verifier_mock = new VerifierMock(blocks, [], chunks, pools); try { @@ -202,7 +202,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_block(frag_id, chunk_id, node_id, pool_id) { return { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), node: node_id, frag: frag_id, chunk: chunk_id, @@ -219,7 +219,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_frag() { return { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), digest: Buffer.from('bla') }; } @@ -230,7 +230,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_chunk(cc_id, frags) { return { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), system: system_id, bucket: bucket_id, tier: tier_id, @@ -256,7 +256,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_node(node_name, offline) { return { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), name: node_name, pool: 'pool1', node_type: 'BLOCK_STORE_FS', @@ -286,7 +286,7 @@ mocha.describe('mocked agent_blocks_verifier', function() { */ function make_schema_pool(name) { return { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), name: name, system: undefined, resource_type: 'HOSTS', diff --git a/src/test/integration_tests/internal/test_map_reader.js b/src/test/integration_tests/internal/test_map_reader.js index cdef8ac62d..b0b0df7251 100644 --- a/src/test/integration_tests/internal/test_map_reader.js +++ b/src/test/integration_tests/internal/test_map_reader.js @@ -9,7 +9,7 @@ coretest.setup({ pools_to_create: [coretest.POOL_LIST[0]] }); // const util = require('util'); const mocha = require('mocha'); // const assert = require('assert'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../../util/mongo_utils'); // const P = require('../../util/promise'); // const MDStore = require('../../server/object_services/md_store').MDStore; @@ -40,14 +40,14 @@ coretest.describe_mapper_test_case({ // TODO test_map_reader mocha.it('read_object_mapping', function() { - const obj = { size: 100, _id: new mongodb.ObjectId() }; + const obj = { size: 100, _id: new mongo_utils.ObjectId() }; const start = 0; const end = 100; return map_reader.read_object_mapping(obj, start, end); }); mocha.it('read_object_mapping_admin', function() { - const obj = { size: 100, _id: new mongodb.ObjectId() }; + const obj = { size: 100, _id: new mongo_utils.ObjectId() }; const skip = 0; const limit = 100; return map_reader.read_object_mapping_admin(obj, skip, limit); diff --git a/src/test/unit_tests/db/test_schema_keywords.js b/src/test/unit_tests/db/test_schema_keywords.js index 2c546fad34..50e7c48b0a 100644 --- a/src/test/unit_tests/db/test_schema_keywords.js +++ b/src/test/unit_tests/db/test_schema_keywords.js @@ -5,7 +5,7 @@ const mocha = require('mocha'); const { default: Ajv } = require('ajv'); const schema_keywords = require('../../../util/schema_keywords'); const SensitiveString = require('../../../util/sensitive_string'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../../util/mongo_utils'); const assert = require('assert'); /** @@ -75,7 +75,7 @@ mocha.describe('Test Schema Keywords', function() { mocha.it('Test keyword objectid', async function() { const validator = ajv.getSchema('test_schema_keywords#/methods/params'); - const should_pass = { key3: new mongodb.ObjectId() }; + const should_pass = { key3: new mongo_utils.ObjectId() }; assert.strictEqual(validator(should_pass), true); const should_fail = { key3: 'not_an_objectid' }; assert.strictEqual(validator(should_fail), false); diff --git a/src/test/unit_tests/internal/test_mapper.js b/src/test/unit_tests/internal/test_mapper.js index 633928ee03..927b9d3dce 100644 --- a/src/test/unit_tests/internal/test_mapper.js +++ b/src/test/unit_tests/internal/test_mapper.js @@ -10,7 +10,7 @@ const _ = require('lodash'); const util = require('util'); const mocha = require('mocha'); const assert = require('assert'); -const mongodb = require('mongodb'); +const mongo_utils = require('../../../util/mongo_utils'); const config = require('../../../../config.js'); const mapper = require('../../../server/object_services/mapper'); @@ -32,45 +32,45 @@ coretest.describe_mapper_test_case({ }) => { const frags = _.concat( - _.times(data_frags, data_index => ({ _id: new mongodb.ObjectId(), data_index })), - _.times(parity_frags, parity_index => ({ _id: new mongodb.ObjectId(), parity_index })) + _.times(data_frags, data_index => ({ _id: new mongo_utils.ObjectId(), data_index })), + _.times(parity_frags, parity_index => ({ _id: new mongo_utils.ObjectId(), parity_index })) ); - const first_pools = _.times(num_pools, i => ({ _id: new mongodb.ObjectId(), name: 'first_pool' + i, })); - const second_pools = _.times(num_pools, i => ({ _id: new mongodb.ObjectId(), name: 'second_pool' + i, })); - const external_pools = _.times(num_pools, i => ({ _id: new mongodb.ObjectId(), name: 'external_pool' + i, })); + const first_pools = _.times(num_pools, i => ({ _id: new mongo_utils.ObjectId(), name: 'first_pool' + i, })); + const second_pools = _.times(num_pools, i => ({ _id: new mongo_utils.ObjectId(), name: 'second_pool' + i, })); + const external_pools = _.times(num_pools, i => ({ _id: new mongo_utils.ObjectId(), name: 'external_pool' + i, })); const pool_by_id = _.keyBy(_.concat(first_pools, second_pools, external_pools), '_id'); const first_mirrors = data_placement === 'MIRROR' ? first_pools.map(pool => ({ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), spread_pools: [pool] })) : [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), spread_pools: first_pools }]; const second_mirrors = data_placement === 'MIRROR' ? second_pools.map(pool => ({ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), spread_pools: [pool] })) : [{ - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), spread_pools: second_pools }]; const first_tier = { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), name: 'first_tier', data_placement, mirrors: first_mirrors, chunk_config: { chunk_coder_config }, }; const second_tier = { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), name: 'second_tier', data_placement, mirrors: second_mirrors, chunk_config: { chunk_coder_config }, }; const tiering = { - _id: new mongodb.ObjectId(), + _id: new mongo_utils.ObjectId(), name: 'tiering_policy', tiers: [{ order: 0, @@ -653,7 +653,7 @@ coretest.describe_mapper_test_case({ const pool = params.pool || pools_to_use[pool_i]; const pool_name = pool.name; - const _id = new mongodb.ObjectID(); + const _id = new mongo_utils.ObjectId(); const _id_str = _id.toString(); return { diff --git a/src/test/unit_tests/internal/test_mirror_writer.js b/src/test/unit_tests/internal/test_mirror_writer.js index 453c5cc164..6bf159d410 100644 --- a/src/test/unit_tests/internal/test_mirror_writer.js +++ b/src/test/unit_tests/internal/test_mirror_writer.js @@ -7,7 +7,7 @@ const mocha = require('mocha'); const assert = require('assert'); const sinon = require('sinon'); const _ = require('lodash'); -const { ObjectId } = require('mongodb'); +const { ObjectId } = require('../../../util/mongo_utils'); const { MirrorWriter } = require('../../../server/bg_services/mirror_writer'); const config = require('../../../../config'); diff --git a/src/test/unit_tests/util_functions_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq b/src/test/unit_tests/util_functions_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq index 346b2f4e35..bedc86e019 100644 --- a/src/test/unit_tests/util_functions_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq +++ b/src/test/unit_tests/util_functions_tests/signature_test_suite/awscli/awscli_iwgdisgt.sreq @@ -1,19 +1,19 @@ -PUT /files/util/mongo_utils.js HTTP/1.1 -Host: 127.0.0.1 -Accept-Encoding: identity -Content-Length: 5865 -Content-MD5: lUCXsCayypL6JVFjbf9kAg== -Expect: 100-continue -Date: Thu, 08 Dec 2016 13:02:39 GMT -User-Agent: aws-cli/1.11.26 Python/2.7.10 Darwin/16.1.0 botocore/1.4.83 -Content-Type: application/javascript -Authorization: AWS 123:Zy/+Do9VcaCZcfdno7lXzjw6qHM= - +PUT /files/util/mongo_utils.js HTTP/1.1 +Host: 127.0.0.1 +Accept-Encoding: identity +Content-Length: 5887 +Content-MD5: r1I35pZpZXxuljKcjcoZyw== +Expect: 100-continue +Date: Thu, 08 Dec 2016 13:02:39 GMT +User-Agent: aws-cli/1.11.26 Python/2.7.10 Darwin/16.1.0 botocore/1.4.83 +Content-Type: application/javascript +Authorization: AWS 123:WW4i8kUV00kNk2QRoBXPxn6tGi4= + 'use strict'; const _ = require('lodash'); const util = require('util'); -const mongodb = require('mongodb'); +const mongo_utils = require('./mongo_utils'); const mongoose = require('mongoose'); const P = require('./promise'); @@ -108,7 +108,7 @@ function populate(docs, doc_path, collection, fields) { function resolve_object_ids_recursive(idmap, item) { _.each(item, (val, key) => { - if (val instanceof mongodb.ObjectId) { + if (val instanceof mongo_utils.ObjectId) { if (key !== '_id') { const obj = idmap[val]; if (obj) { @@ -144,7 +144,7 @@ function resolve_object_ids_paths(idmap, item, paths, allow_missing) { } function make_object_id(id_str) { - return new mongodb.ObjectId(id_str); + return new mongo_utils.ObjectId(id_str); } function fix_id_type(doc) { @@ -161,7 +161,7 @@ function fix_id_type(doc) { // so we have to check both for now, // until we can get rid of mongoose completely. function is_object_id(id) { - return (id instanceof mongodb.ObjectId) || + return (id instanceof mongo_utils.ObjectId) || (id instanceof mongoose.Types.ObjectId); } diff --git a/src/tools/md_blow_lifecycle.js b/src/tools/md_blow_lifecycle.js index 5c2438b95f..1c57f5d94a 100644 --- a/src/tools/md_blow_lifecycle.js +++ b/src/tools/md_blow_lifecycle.js @@ -2,7 +2,7 @@ 'use strict'; const _ = require('lodash'); -const mongodb = require('mongodb'); +const mongo_utils = require('../util/mongo_utils'); const argvParse = require('minimist'); const P = require('../util/promise'); const api = require('../api'); @@ -127,7 +127,7 @@ async function blow_version_objects() { const { obj_id } = await client.object.create_object_upload({ bucket: argv.bucket, key: argv.version_key, content_type }); await client.object.complete_object_upload({ obj_id, bucket: argv.bucket, key: argv.version_key }); if (i < argv.version_count - 2) { - obj_upload_ids.push(new mongodb.ObjectId(obj_id)); + obj_upload_ids.push(new mongo_utils.ObjectId(obj_id)); } } @@ -199,7 +199,8 @@ async function blow_multipart_object(index) { create_time, }; console.log('create_mock_multipart_upload bucket', argv.bucket, 'obj_id', params.obj_id, 'multiparts_ids', complete_params.multipart_id); - const update_result = await MDStore.instance().update_multipart_by_id(new mongodb.ObjectId(complete_params.multipart_id), update); + const update_result = await MDStore.instance().update_multipart_by_id( + new mongo_utils.ObjectId(complete_params.multipart_id), update); console.log('update_multiparts_by_ids', update_result); } diff --git a/src/util/db_client.js b/src/util/db_client.js index 78bcee9d5e..ada33d124a 100644 --- a/src/util/db_client.js +++ b/src/util/db_client.js @@ -2,7 +2,7 @@ /** @typedef {typeof import('../sdk/nb')} nb */ 'use strict'; -const mongodb = require('mongodb'); +const mongo_utils = require('./mongo_utils'); const { EventEmitter } = require('events'); const dbg = require('./debug_module')(__filename); @@ -36,8 +36,8 @@ class NoneDBClient extends EventEmitter { async populate(docs, doc_path, collection, fields) { return this.noop(); } resolve_object_ids_recursive(idmap, item) { return this.noop(); } resolve_object_ids_paths(idmap, item, paths, allow_missing) { return this.noop(); } - new_object_id() { return new mongodb.ObjectId(); } - parse_object_id(id_str) { return new mongodb.ObjectId(String(id_str || undefined)); } + new_object_id() { return new mongo_utils.ObjectId(); } + parse_object_id(id_str) { return new mongo_utils.ObjectId(String(id_str || undefined)); } fix_id_type(doc) { return doc; } is_object_id(id) { return false; } is_err_duplicate_key(err) { return false; } diff --git a/src/util/mongo_client.js b/src/util/mongo_client.js index 260d1506ad..d0acc5859b 100644 --- a/src/util/mongo_client.js +++ b/src/util/mongo_client.js @@ -6,6 +6,7 @@ const fs = require('fs'); const { default: Ajv } = require('ajv'); const util = require('util'); const mongodb = require('mongodb'); +const mongo_utils = require('./mongo_utils'); const EventEmitter = require('events').EventEmitter; const P = require('./promise'); @@ -259,7 +260,7 @@ class MongoClient extends EventEmitter { resolve_object_ids_recursive(idmap, item) { _.each(item, (val, key) => { - if (val instanceof mongodb.ObjectId) { + if (val instanceof mongo_utils.ObjectId) { if (key !== '_id') { const obj = idmap[val.toHexString()]; if (obj) { @@ -298,7 +299,7 @@ class MongoClient extends EventEmitter { * @returns {nb.ID} */ new_object_id() { - return new mongodb.ObjectId(); + return new mongo_utils.ObjectId(); } /** @@ -307,20 +308,20 @@ class MongoClient extends EventEmitter { */ parse_object_id(id_str) { if (!id_str) throw new TypeError('parse_object_id: arg required ' + id_str); - return new mongodb.ObjectId(String(id_str)); + return new mongo_utils.ObjectId(id_str); } fix_id_type(doc) { if (_.isArray(doc)) { _.each(doc, d => this.fix_id_type(d)); } else if (doc && doc._id) { - doc._id = new mongodb.ObjectId(doc._id); + doc._id = new mongo_utils.ObjectId(doc._id); } return doc; } is_object_id(id) { - return (id instanceof mongodb.ObjectId); + return (id instanceof mongo_utils.ObjectId) || mongo_utils.is_object_id(id); } is_err_duplicate_key(err) { diff --git a/src/util/mongo_utils.js b/src/util/mongo_utils.js index 440abfb257..6e9af151d2 100644 --- a/src/util/mongo_utils.js +++ b/src/util/mongo_utils.js @@ -5,7 +5,7 @@ // const _ = require('lodash'); // const util = require('util'); -const mongodb = require('mongodb'); +// const mongodb = require('mongodb'); // const { RpcError } = require('../rpc'); @@ -158,10 +158,34 @@ const mongodb = require('mongodb'); // return doc; // } -function is_object_id(id) { - return (id instanceof mongodb.ObjectId); +function is_object_id(id, generate = false) { + const err_msg = 'Argument passed must be a string of 24 hex characters'; + + if (id === null || id === undefined) { + return generate ? mongoObjectId() : false; + } + + if (typeof id === 'number' && Number.isInteger(id) && id > 0) { + if (!generate) return true; + return id.toString(16).padStart(8, '0') + mongoObjectId().substring(8); + } + + let hex_string = null; + if (typeof id === 'string') { + hex_string = id; + } else if (id._id && typeof id._id === 'string') { + hex_string = id._id; + } + + if (hex_string && (/^[0-9a-f]{24}$/i).test(hex_string)) { + return generate ? hex_string.toLowerCase() : true; + } + + if (generate) throw new Error(err_msg); + return false; } + function mongoObjectId() { // eslint-disable-next-line no-bitwise const timestamp = (new Date().getTime() / 1000 | 0).toString(16); @@ -230,6 +254,41 @@ function mongoObjectId() { // return client.db().command({ dbStats: 1 }); // } +/** + * MongoDB ObjectId compatibility class + * behaves like mongodb.ObjectId with minimal validations + */ +class ObjectID { + constructor(id_str) { + this._id = is_object_id(id_str, true); + } + + toString() { + return this._id; + } + + toHexString() { + return this._id; + } + + valueOf() { + return this._id; + } + + toJSON() { + return this._id; + } + + getTimestamp() { + const timestamp = parseInt(this._id.substring(0, 8), 16); + return new Date(timestamp * 1000); + } + + static isValid(id) { + return is_object_id(id); + } +} + // // EXPORTS // exports.mongo_operators = mongo_operators; // exports.obj_ids_difference = obj_ids_difference; @@ -249,4 +308,5 @@ exports.is_object_id = is_object_id; // exports.check_update_one = check_update_one; // exports.make_object_diff = make_object_diff; // exports.get_db_stats = get_db_stats; +exports.ObjectId = ObjectID; exports.mongoObjectId = mongoObjectId; diff --git a/src/util/postgres_client.js b/src/util/postgres_client.js index 16ffa0359a..512e980295 100644 --- a/src/util/postgres_client.js +++ b/src/util/postgres_client.js @@ -19,6 +19,7 @@ const common_api = require('../api/common_api'); const schema_utils = require('./schema_utils'); const schema_keywords = require('./schema_keywords'); const mongodb = require('mongodb'); +const mongo_utils = require('./mongo_utils'); const mongo_to_pg = require('mongo-query-to-postgres-jsonb'); const fs = require('fs'); // TODO: Shouldn't be like that, we shouldn't use MongoDB functions to compare @@ -55,7 +56,13 @@ function decode_json(schema, val) { return val; } if (schema.objectid === true) { - return new mongodb.ObjectId(val); + if (typeof val === 'string') { + return new mongo_utils.ObjectId(val); + } else if (val instanceof mongo_utils.ObjectId) { + return val; + } else { + return new mongo_utils.ObjectId(String(val)); + } } if (schema.date === true) { return new Date(val); @@ -98,7 +105,7 @@ function encode_json(schema, val) { const ops = handle_ops_encoding(schema, val); if (ops) return ops; - if (schema.objectid === true && val instanceof mongodb.ObjectID) { + if (schema.objectid === true && val instanceof mongo_utils.ObjectId) { return val.toString(); } @@ -1219,7 +1226,7 @@ class PostgresTable { const new_row = {}; for (const key of Object.keys(row)) { if (key === '_id') { - new_row._id = new mongodb.ObjectID(row[key]); + new_row._id = new mongo_utils.ObjectId(row[key]); } else { new_row[key] = parseInt(row[key], 10); } @@ -1658,7 +1665,7 @@ class PostgresClient extends EventEmitter { } generate_id() { - return new mongodb.ObjectId(); + return new mongo_utils.ObjectId(); } collection(name) { @@ -1829,7 +1836,7 @@ class PostgresClient extends EventEmitter { resolve_object_ids_recursive(idmap, item) { _.each(item, (val, key) => { - if (val instanceof mongodb.ObjectId) { + if (val instanceof mongo_utils.ObjectId) { if (key !== '_id') { const obj = idmap[val.toHexString()]; if (obj) { @@ -1868,7 +1875,7 @@ class PostgresClient extends EventEmitter { * @returns {nb.ID} */ new_object_id() { - return new mongodb.ObjectId(); + return new mongo_utils.ObjectId(); } /** @@ -1876,20 +1883,20 @@ class PostgresClient extends EventEmitter { * @returns {nb.ID} */ parse_object_id(id_str) { - return new mongodb.ObjectId(String(id_str || undefined)); + return new mongo_utils.ObjectId(id_str); } fix_id_type(doc) { if (_.isArray(doc)) { _.each(doc, d => this.fix_id_type(d)); } else if (doc && doc._id) { - doc._id = new mongodb.ObjectId(doc._id); + doc._id = new mongo_utils.ObjectId(doc._id); } return doc; } is_object_id(id) { - return (id instanceof mongodb.ObjectId); + return (id instanceof mongo_utils.ObjectId) || mongo_utils.is_object_id(id); } // TODO: Figure out error codes diff --git a/src/util/schema_keywords.js b/src/util/schema_keywords.js index 1deb1de3c0..24c0f59bcc 100644 --- a/src/util/schema_keywords.js +++ b/src/util/schema_keywords.js @@ -60,7 +60,7 @@ const KEYWORDS = js_utils.deep_freeze({ } }, - // schema: { objectid: true } will match (new mongodb.ObjectId()) or (new mongodb.ObjectId()).valueOf() + // schema: { objectid: true } will match (new mongo_utils.ObjectId()) or (new mongo_utils.ObjectId()).valueOf() objectid: { keyword: 'objectid', // schemaType: 'boolean',