Skip to content

Commit 77618d4

Browse files
ar2rsawseenar2rsawseen
authored
Merge pull request #6820 from Countly/ar2rsawseen/newarchitecture
Remove unused collections
2 parents 67e38d6 + e4db46d commit 77618d4

File tree

24 files changed

+130
-1085
lines changed

24 files changed

+130
-1085
lines changed

‎api/parts/data/usage.js‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -868,19 +868,6 @@ function processMetrics(user, uniqueLevelsZero, uniqueLevelsMonth, params, done)
868868
}
869869
}
870870

871-
if (!isNewUser) {
872-
/*
873-
If metricChanges object contains a uid this means we have at least one metric that has changed
874-
in this begin_session so we'll insert it into metric_changesAPPID collection.
875-
Inserted document has below format;
876-
877-
{ "uid" : "1", "ts" : 1463778143, "d" : { "o" : "iPhone1", "n" : "iPhone2" }, "av" : { "o" : "1:0", "n" : "1:1" } }
878-
*/
879-
if (plugins.getConfig("api", params.app && params.app.plugins, true).metric_changes && metricChanges.uid && params.qstring.begin_session) {
880-
common.db.collection('metric_changes' + params.app_id).insert(metricChanges, function() {});
881-
}
882-
}
883-
884871
if (done) {
885872
done();
886873
}

‎api/parts/mgmt/app_users.js‎

Lines changed: 65 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -267,80 +267,78 @@ usersApi.delete = function(app_id, query, params, callback) {
267267
console.log("Error generating list of uids", err0, res);
268268
}
269269
if (res && res[0] && res[0].uid && res[0].uid.length) {
270-
common.db.collection("metric_changes" + app_id).remove({uid: {$in: res[0].uid}}, function() {
271-
plugins.dispatch("/i/app_users/delete", {
272-
app_id: app_id,
273-
query: query,
274-
uids: res[0].uid,
275-
params: params
276-
}, function(_, otherPluginResults) {
277-
const rejectReasons = otherPluginResults.reduce((acc, result) => {
278-
if (result.status === "rejected") {
279-
acc.push((result.reason && result.reason.message) || '');
280-
}
270+
plugins.dispatch("/i/app_users/delete", {
271+
app_id: app_id,
272+
query: query,
273+
uids: res[0].uid,
274+
params: params
275+
}, function(_, otherPluginResults) {
276+
const rejectReasons = otherPluginResults.reduce((acc, result) => {
277+
if (result.status === "rejected") {
278+
acc.push((result.reason && result.reason.message) || '');
279+
}
281280

282-
return acc;
283-
}, []);
281+
return acc;
282+
}, []);
284283

285-
if (rejectReasons.length > 0) {
286-
log.e("User deletion failed\n%j", rejectReasons.join("\n"));
287-
common.returnMessage(params, 500, { errorMessage: "User deletion failed. Failed to delete some data related to this user." });
288-
return;
289-
}
284+
if (rejectReasons.length > 0) {
285+
log.e("User deletion failed\n%j", rejectReasons.join("\n"));
286+
common.returnMessage(params, 500, { errorMessage: "User deletion failed. Failed to delete some data related to this user." });
287+
return;
288+
}
290289

291-
//remove from drill_events
292-
plugins.dispatch("/core/delete_granular_data", {
293-
db: "countly_drill",
294-
collection: "drill_events",
295-
query: { a: app_id + "", uid: { $in: res[0].uid } }
296-
});
290+
//remove from drill_events
291+
plugins.dispatch("/core/delete_granular_data", {
292+
db: "countly_drill",
293+
collection: "drill_events",
294+
query: { a: app_id + "", uid: { $in: res[0].uid } }
295+
});
297296

298-
common.db.collection("app_users" + app_id).remove({uid: {$in: res[0].uid}}, function(err) {
299-
if (res[0].exported) {
300-
//delete exports if exist
301-
for (let i = 0;i < res[0].exported.length; i++) {
302-
let id = res[0].exported[i].split("/");
303-
id = id[id.length - 1]; //last one is filename
304-
id = id.substr(id.length - 7);
305-
306-
deleteMyExport(id).then(
307-
function() {},
308-
function(err5) {
309-
console.log(err5);
310-
}
311-
);
312-
}
313-
}
314-
//deleting userimages(if they exist);
315-
if (res[0].picture) {
316-
for (let i = 0;i < res[0].picture.length; i++) {
317-
//remove /userimages/
318-
let id = res[0].picture[i].substr(12, res[0].picture[i].length - 12);
319-
var pp = path.resolve(__dirname, './../../../frontend/express/public/userimages/' + id);
320-
countlyFs.deleteFile("userimages", pp, {id: id}, function(err6) {
321-
if (err6) {
322-
console.log(err6);
323-
}
324-
});
325-
}
297+
common.db.collection("app_users" + app_id).remove({uid: {$in: res[0].uid}}, function(err) {
298+
if (res[0].exported) {
299+
//delete exports if exist
300+
for (let i = 0;i < res[0].exported.length; i++) {
301+
let id = res[0].exported[i].split("/");
302+
id = id[id.length - 1]; //last one is filename
303+
id = id.substr(id.length - 7);
304+
305+
deleteMyExport(id).then(
306+
function() {},
307+
function(err5) {
308+
console.log(err5);
309+
}
310+
);
326311
}
327-
try {
328-
fs.appendFileSync(path.resolve(__dirname, './../../../log/deletedUsers' + app_id + '.txt'), res[0].uid.join("\n") + "\n", "utf-8");
312+
}
313+
//deleting userimages(if they exist);
314+
if (res[0].picture) {
315+
for (let i = 0;i < res[0].picture.length; i++) {
316+
//remove /userimages/
317+
let id = res[0].picture[i].substr(12, res[0].picture[i].length - 12);
318+
var pp = path.resolve(__dirname, './../../../frontend/express/public/userimages/' + id);
319+
countlyFs.deleteFile("userimages", pp, {id: id}, function(err6) {
320+
if (err6) {
321+
console.log(err6);
322+
}
323+
});
329324
}
330-
catch (err2) {
331-
console.log(err2);
325+
}
326+
try {
327+
fs.appendFileSync(path.resolve(__dirname, './../../../log/deletedUsers' + app_id + '.txt'), res[0].uid.join("\n") + "\n", "utf-8");
328+
}
329+
catch (err2) {
330+
console.log(err2);
331+
}
332+
plugins.dispatch("/systemlogs", {
333+
params: params,
334+
action: "app_user_deleted",
335+
data: {
336+
app_id: app_id,
337+
query: JSON.stringify(query),
338+
uids: res[0].uid,
332339
}
333-
plugins.dispatch("/systemlogs", {
334-
params: params,
335-
action: "app_user_deleted",
336-
data: {
337-
app_id: app_id,
338-
query: JSON.stringify(query),
339-
uids: res[0].uid,
340-
}
341-
});
342-
callback(err, res[0].uid);
343340
});
341+
callback(err, res[0].uid);
344342
});
345343
});
346344
}
@@ -758,7 +756,6 @@ usersApi.merge = async function(app_id, newAppUser, new_id, old_id, new_device_i
758756
var iid = app_id + "_" + newAppUser.uid + "_" + oldAppUser.uid;
759757
try {
760758
await common.db.collection('app_user_merges').updateOne({"_id": iid, "cc": {"$ne": true}}, {'$set': {"u": true}}, {upsert: false});
761-
await common.db.collection("metric_changes" + app_id).updateMany({uid: oldAppUser.uid}, {'$set': {uid: newAppUserP.uid}});
762759
}
763760
catch (e) {
764761
log.e("Failed metric changes update in app_users merge", e);
@@ -1144,21 +1141,8 @@ usersApi.export = function(app_id, query, params, callback) {
11441141
// }
11451142

11461143
//try deleting old export
1147-
deleteMyExport(export_id).then(function(err) {
1148-
if (err) {
1149-
log.e(err);
1150-
}
1144+
deleteMyExport(export_id).then(function() {
11511145
log.d("old export deleted");
1152-
return new Promise(function(resolve) {
1153-
log.d("collection marked");
1154-
//export data from metric_changes
1155-
1156-
export_safely({projection: {"appUserExport": 0}, export_id: export_id, app_id: app_id, args: [...dbargs, "--collection", "metric_changes" + app_id, "-q", '{"uid":{"$in": ["' + res[0].uid.join('","') + '"]}}', "--out", export_folder + "/metric_changes" + app_id + ".json"]}).finally(function() {
1157-
resolve();
1158-
});
1159-
});
1160-
}).then(function() {
1161-
log.d("metric_changes exported");
11621146
//export data from app_users
11631147
return export_safely({projection: {"appUserExport": 0}, export_id: export_id, app_id: app_id, args: [...dbargs, "--collection", "app_users" + app_id, "-q", '{"uid":{"$in": ["' + res[0].uid.join('","') + '"]}}', "--out", export_folder + "/app_users" + app_id + ".json"]});
11641148

‎api/parts/mgmt/apps.js‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -289,8 +289,6 @@ appsApi.createApp = async function(params) {
289289
common.db.collection('app_users' + app.ops[0]._id).ensureIndex({"lac": -1}, { background: true }, function() {});
290290
common.db.collection('app_users' + app.ops[0]._id).ensureIndex({"tsd": 1}, { background: true }, function() {});
291291
common.db.collection('app_users' + app.ops[0]._id).ensureIndex({"did": 1}, { background: true }, function() {});
292-
common.db.collection('metric_changes' + app.ops[0]._id).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, function() {});
293-
common.db.collection('metric_changes' + app.ops[0]._id).ensureIndex({uid: 1}, { background: true }, function() {});
294292
plugins.dispatch("/i/apps/create", {
295293
params: params,
296294
appId: app.ops[0]._id,
@@ -857,10 +855,6 @@ function deleteAllAppData(appId, fromAppDelete, params, app) {
857855
}
858856
common.db.collection('app_users' + appId).drop(function() {
859857
if (!fromAppDelete) {
860-
common.db.collection('metric_changes' + appId).drop(function() {
861-
common.db.collection('metric_changes' + appId).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, function() {});
862-
common.db.collection('metric_changes' + appId).ensureIndex({uid: 1}, { background: true }, function() {});
863-
});
864858
//Removes old app_user_merges collection
865859
common.db.collection('app_user_merges' + appId).drop(function() {});
866860
if (params.qstring.args.period === "reset") {
@@ -879,7 +873,6 @@ function deleteAllAppData(appId, fromAppDelete, params, app) {
879873
}
880874
}
881875
else {
882-
common.db.collection('metric_changes' + appId).drop(function() {});
883876
common.db.collection('app_user_merges' + appId).drop(function() {});
884877
plugins.dispatch("/i/apps/delete", {
885878
params: params,
@@ -986,12 +979,6 @@ function deletePeriodAppData(appId, fromAppDelete, params, app) {
986979
*/
987980
common.db.collection('app_users' + appId).update({ls: {$lte: oldestTimestampWanted}}, {$set: {ls: 1}}, function() {});
988981

989-
/*
990-
Remove all metric changes that happened before oldestTimestampWanted since we no longer need
991-
old metric changes
992-
*/
993-
common.db.collection('metric_changes' + appId).remove({ts: {$lte: oldestTimestampWanted}}, function() {});
994-
995982
plugins.dispatch("/i/apps/clear", {
996983
params: params,
997984
appId: appId,

‎bin/scripts/add_indexes.js‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,7 @@ pluginManager.dbConnection().then((countlyDb) => {
2828
() => countlyDb.collection('app_users' + app._id).ensureIndex({"lac": -1}, { background: true }, cb),
2929
() => countlyDb.collection('app_users' + app._id).ensureIndex({"tsd": 1}, { background: true }, cb),
3030
() => countlyDb.collection('app_users' + app._id).ensureIndex({"did": 1}, { background: true }, cb),
31-
() => countlyDb.collection('app_users' + app._id).dropIndex("lac_1_ls_1", cb),
32-
() => countlyDb.collection('metric_changes' + app._id).ensureIndex({ts: 1, "cc.o": 1}, { background: true }, cb),
33-
() => countlyDb.collection('metric_changes' + app._id).ensureIndex({uid: 1}, { background: true }, cb)
31+
() => countlyDb.collection('app_users' + app._id).dropIndex("lac_1_ls_1", cb)
3432
];
3533

3634
totalParallelJobs = parallelJobs.length;

‎bin/scripts/data-cleanup/remove_old_data.js‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,9 +52,6 @@ pluginManager.dbConnection().then(async(db) => {
5252
var count = await db.collection(`consent_history${app._id}`).countDocuments({ ts: { $lt: lastUnixTimestamp } });
5353
console.log("", count, "consents to be deleted");
5454
checkThreshold(count);
55-
count = await db.collection(`metric_changes${app._id}`).countDocuments({ ts: { $lt: lastUnixTimestamp } });
56-
console.log("", count, "metric changes to be deleted");
57-
checkThreshold(count);
5855
count = await db.collection(`eventTimes${app._id}`).countDocuments({ ts: { $lt: lastUnixTimestamp * 1000 } });
5956
console.log("", count, "event times to be deleted");
6057
checkThreshold(count);
@@ -64,9 +61,6 @@ pluginManager.dbConnection().then(async(db) => {
6461
var res = await db.collection(`consent_history${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp } });
6562
console.log("", res, "consents deleted");
6663
await sleep(SLEEP);
67-
res = await db.collection(`metric_changes${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp } });
68-
console.log("", res, "metric changes deleted");
69-
await sleep(SLEEP);
7064
res = await db.collection(`eventTimes${app._id}`).deleteMany({ ts: { $lt: lastUnixTimestamp * 1000 } });
7165
console.log("", res, "event times deleted");
7266
await sleep(SLEEP);

‎bin/scripts/expire-data/countly_single_app_expireDataBatches.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ var errorCn = 0;
4848
var process = {
4949
drill_events: true,
5050
/*app_crashes:true,
51-
metric_changes:true,
5251
consent_history:true,
5352
feedback:true,
5453
symbolication_jobs:true,

‎bin/scripts/expire-data/mongo_expireData.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ var COUNTLY_DRILL = 'countly_drill',
1313
var PROCESS = [
1414
/^drill_events\.*/,
1515
/^app_crashes\.*/,
16-
/^metric_changes\.*/,
1716
/^consent_history\.*/,
1817
/^feedback[^_]*/,
1918
/^symbolication_jobs/,

‎bin/scripts/expire-data/old_scripts/countly_single_app_expireDataBatches.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ var errorCn = 0;
4949
var process = {
5050
drill_events: true,
5151
/*app_crashes:true,
52-
metric_changes:true,
5352
consent_history:true,
5453
feedback:true,
5554
symbolication_jobs:true,

‎bin/scripts/fix-data/process_merges.js‎

Lines changed: 1 addition & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -115,12 +115,6 @@ function mergeUserData(db, user, resolve) {
115115
db.collection('app_users' + app_id).update({_id: newAppUser._id}, {'$set': newAppUser}, function(err6) {
116116
//Dispatch to other plugins only after callback.
117117
if (!err6) {
118-
//update metric changes document
119-
db.collection("metric_changes" + app_id).update({uid: oldAppUser.uid}, {'$set': {uid: newAppUser.uid}}, {multi: true}, function(err7) {
120-
if (err7) {
121-
console.log("Failed metric changes update in app_users merge", err7);
122-
}
123-
});
124118
//delete old app users document
125119
db.collection('app_users' + app_id).remove({_id: oldAppUser._id}, function(errRemoving) {
126120
if (errRemoving) {
@@ -145,14 +139,7 @@ function mergeUserData(db, user, resolve) {
145139
if (err0) {
146140
console.log(err0);
147141
}
148-
db.collection("metric_changes" + app_id).update({uid: olduid}, {'$set': {uid: usersApi.merged_to}}, {multi: true}, function(err7) {
149-
if (err7) {
150-
console.log("Failed metric changes update in app_users merge", err7);
151-
}
152-
else {
153-
usersApi.mergeOtherPlugins({db: db, app_id: app_id, newAppUser: {uid: user.merged_to}, oldAppUser: {uid: olduid}, updateFields: {"cc": true, "mc": true}, mergeDoc: user}, resolve);
154-
}
155-
});
142+
usersApi.mergeOtherPlugins({db: db, app_id: app_id, newAppUser: {uid: user.merged_to}, oldAppUser: {uid: olduid}, updateFields: {"cc": true, "mc": true}, mergeDoc: user}, resolve);
156143
});
157144
}
158145
else {

‎jobServer/index.js‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,6 @@ if (require.main === module) {
208208
// total_users: true,
209209
// export_limit: 10000,
210210
prevent_duplicate_requests: true,
211-
// metric_changes: true,
212211
offline_mode: false,
213212
// reports_regenerate_interval: 3600,
214213
// send_test_email: "",

0 commit comments

Comments
 (0)