From c8d74d71d8686c552c3e90250abd53ff2281a5ea Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 11 Nov 2016 17:03:16 -0500 Subject: [PATCH 01/35] GPII-1987: Initial check in of a start on creating data loaders. --- scripts/dataLoader.js | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 scripts/dataLoader.js diff --git a/scripts/dataLoader.js b/scripts/dataLoader.js new file mode 100644 index 000000000..2fee700b7 --- /dev/null +++ b/scripts/dataLoader.js @@ -0,0 +1,40 @@ +/*! +Copyright 2016 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +// The data loader to load GPII test data into CouchDB + +"use strict"; + +fluid.defaults("gpii.dataLoader", { + gradeNames: ["fluid.component"], + databases: { // Supplied by integrators + // Accepted format: + // dbName: { + // data: ["pathToDataFile1", "pathToDataFile1"...] + // } + }, + components: { + dataSource: { + type: "gpii.dataLoader.dataSource" + } + } +}); + +fluid.defaults("gpii.dataLoader.dataSource", { + gradeNames: ["kettle.dataSource.URL", "kettle.dataSource.CouchDB"], + couchDbUrl: null, // Supplied by integrators + url: "%couchDbUrl/_bulk_docs" + termMap: { + couchDbUrl: "noencode:%couchDbUrl" + } +}); From ef671e4b75e74535c0729a28bfec476337296674 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 14 Nov 2016 17:41:43 -0500 Subject: [PATCH 02/35] GPII-1987: In progress with dataLoader.js --- scripts/dataLoader.js | 72 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 6 deletions(-) diff --git a/scripts/dataLoader.js b/scripts/dataLoader.js index 2fee700b7..6e6654ab2 100644 --- a/scripts/dataLoader.js +++ b/scripts/dataLoader.js @@ -17,24 +17,84 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt fluid.defaults("gpii.dataLoader", { gradeNames: ["fluid.component"], + // Accepted format: + // dbName1: { + // data: ["pathToDataFile1", "pathToDataFile1"...] + // }, + // dbName2: { + // data: ["pathToDataFile1", "pathToDataFile1"...] + // } databases: { // Supplied by integrators - // Accepted format: - // dbName: { - // data: ["pathToDataFile1", "pathToDataFile1"...] - // } }, + couchDbUrl: null, // Supplied by integrators components: { dataSource: { type: "gpii.dataLoader.dataSource" } + }, + invokers: { + load: { + funcName: "gpii.dataLoader.load", + args: ["{that}", "{that}.dataSource"] + } + }, + events: { + onDataLoaded: null + }, + distributeOptions: { + couchDbUrl: { + record: "{that}.options.couchDbUrl", + target: "{that > dataSource}.options.couchDbUrl" + } } }); +gpii.dataLoader.load = function (that, dataSource) { + var promises = []; + + // Process databases one by one + fluid.each(that.options.databases, function (filePaths, dbName) { + filePaths = fluid.makeArray(filePaths); + fluid.each(filePaths, function (onePath) { + var actualPath = fluid.module.resolvePath(onePath); + var data = require(actualPath); + var directModel = { + couchDbUrl: dataSource.options.couchDbUrl, + dbName: dbName + }; + var setPromise = dataSource.set(directModel, data); + promises.push(setPromise); + }); + }); + + // An sequence with an empty array of promises will automatically be resolved, so we can safely use this construct. + var sequence = fluid.promise.sequence(promises); + sequence.then(function () { + that.events.onDataLoaded.fire(that); + }); + return sequence; +}; + fluid.defaults("gpii.dataLoader.dataSource", { gradeNames: ["kettle.dataSource.URL", "kettle.dataSource.CouchDB"], + writable: true, + writeMethod: "POST", couchDbUrl: null, // Supplied by integrators - url: "%couchDbUrl/_bulk_docs" + url: "%couchDbUrl/%dbName/_bulk_docs" termMap: { - couchDbUrl: "noencode:%couchDbUrl" + couchDbUrl: "noencode:%couchDbUrl", + dbName: "%dbName" } }); + +fluid.defaults("gpii.dataLoader.authDataLoader", { + gradeNames: ["gpii.dataLoader"], + databases: { + auth: { + data: [ + "%universal/testData/security/TestOAuth2DataStore.json", + "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ] + } + }, +}); From 3c0c6caa1f5de9f0a2b4ceb0895f58f1100e468a Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 15 Nov 2016 19:27:56 -0500 Subject: [PATCH 03/35] GPII-1987: In progress of implementing data loaders. --- scripts/authDataLoader.js | 50 ++++++++++++++ scripts/dataLoader.js | 100 ---------------------------- scripts/shared/dataLoader.js | 123 +++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 100 deletions(-) create mode 100644 scripts/authDataLoader.js delete mode 100644 scripts/dataLoader.js create mode 100644 scripts/shared/dataLoader.js diff --git a/scripts/authDataLoader.js b/scripts/authDataLoader.js new file mode 100644 index 000000000..8efc6081d --- /dev/null +++ b/scripts/authDataLoader.js @@ -0,0 +1,50 @@ +/*! +Copyright 2016 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +// The data loader to load GPII authorization server data into CouchDB + +"use strict"; + +require("./shared/dataLoader.js"); + +fluid.defaults("gpii.dataLoader.authDataLoader", { + gradeNames: ["gpii.dataLoader"], + databases: { + auth: { + data: [ + "%universal/testData/security/TestOAuth2DataStore.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ] + } + }, + couchDbUrl: "http://localhost:5984", + listeners: { + listeners: { + "onCreate.load": { + listener: "gpii.dataLoader.authDataLoader.loadData", + args: ["{that}"] + } + } + } +}); + +gpii.dataLoader.authDataLoader.loadData = function (that) { + var promise = that.load(); + promise.then(function () { + console.log("The authorization data has been loaded successfully."); + }, function (err) { + console.log("Error at loading the authorization data: ", err); + }); +}; + +gpii.dataLoader.authDataLoader(); diff --git a/scripts/dataLoader.js b/scripts/dataLoader.js deleted file mode 100644 index 6e6654ab2..000000000 --- a/scripts/dataLoader.js +++ /dev/null @@ -1,100 +0,0 @@ -/*! -Copyright 2016 OCAD University - -Licensed under the New BSD license. You may not use this file except in -compliance with this License. - -The research leading to these results has received funding from the European Union's -Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. - -You may obtain a copy of the License at -https://github.com/GPII/universal/blob/master/LICENSE.txt -*/ - -// The data loader to load GPII test data into CouchDB - -"use strict"; - -fluid.defaults("gpii.dataLoader", { - gradeNames: ["fluid.component"], - // Accepted format: - // dbName1: { - // data: ["pathToDataFile1", "pathToDataFile1"...] - // }, - // dbName2: { - // data: ["pathToDataFile1", "pathToDataFile1"...] - // } - databases: { // Supplied by integrators - }, - couchDbUrl: null, // Supplied by integrators - components: { - dataSource: { - type: "gpii.dataLoader.dataSource" - } - }, - invokers: { - load: { - funcName: "gpii.dataLoader.load", - args: ["{that}", "{that}.dataSource"] - } - }, - events: { - onDataLoaded: null - }, - distributeOptions: { - couchDbUrl: { - record: "{that}.options.couchDbUrl", - target: "{that > dataSource}.options.couchDbUrl" - } - } -}); - -gpii.dataLoader.load = function (that, dataSource) { - var promises = []; - - // Process databases one by one - fluid.each(that.options.databases, function (filePaths, dbName) { - filePaths = fluid.makeArray(filePaths); - fluid.each(filePaths, function (onePath) { - var actualPath = fluid.module.resolvePath(onePath); - var data = require(actualPath); - var directModel = { - couchDbUrl: dataSource.options.couchDbUrl, - dbName: dbName - }; - var setPromise = dataSource.set(directModel, data); - promises.push(setPromise); - }); - }); - - // An sequence with an empty array of promises will automatically be resolved, so we can safely use this construct. - var sequence = fluid.promise.sequence(promises); - sequence.then(function () { - that.events.onDataLoaded.fire(that); - }); - return sequence; -}; - -fluid.defaults("gpii.dataLoader.dataSource", { - gradeNames: ["kettle.dataSource.URL", "kettle.dataSource.CouchDB"], - writable: true, - writeMethod: "POST", - couchDbUrl: null, // Supplied by integrators - url: "%couchDbUrl/%dbName/_bulk_docs" - termMap: { - couchDbUrl: "noencode:%couchDbUrl", - dbName: "%dbName" - } -}); - -fluid.defaults("gpii.dataLoader.authDataLoader", { - gradeNames: ["gpii.dataLoader"], - databases: { - auth: { - data: [ - "%universal/testData/security/TestOAuth2DataStore.json", - "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" - ] - } - }, -}); diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js new file mode 100644 index 000000000..b3f06554d --- /dev/null +++ b/scripts/shared/dataLoader.js @@ -0,0 +1,123 @@ +/*! +Copyright 2016 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +// The underlying data loader implementation shared by ../authDataLoader.js and ../prefsDataLoader.js +// to load GPII test data into CouchDB. +// Steps to load data: +// 1. Check the existence of the database; +// 2. If the database already exists, delete it to ensure a clean load; otherwise, create the database; +// 3. Load data in bulk using CouchDB /_bulk_docs request - http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs + +"use strict"; + +var fluid = require("infusion"), + kettle = require("kettle"), + gpii = fluid.registerNamespace("gpii"); + +// Data Source used to interact with CouchDB +fluid.defaults("gpii.dataLoader.dataSource", { + gradeNames: ["kettle.dataSource.URL", "kettle.dataSource.CouchDB"], + url: "%couchDbUrl/%dbName/_bulk_docs", + termMap: { + couchDbUrl: "noencode:%couchDbUrl", + dbName: "%dbName" + } +}); + +fluid.defaults("gpii.dataLoader.dataSource.writable", { + gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.CouchDB.writable"], + writable: true +}); + +// Data loader +fluid.defaults("gpii.dataLoader", { + gradeNames: ["fluid.component"], + // Accepted format: + // dbName1: { + // data: ["pathToDataFile1", "pathToDataFile1"...] + // }, + // dbName2: { + // data: ["pathToDataFile1", "pathToDataFile1"...] + // } + databases: { // Supplied by integrators + }, + couchDbUrl: null, // Supplied by integrators + components: { + getDbDataSource: { + type: "gpii.dataLoader.dataSource", + options: { + url: "%couchDbUrl/%dbName" + } + }, + deleteDbDataSource: { + type: "gpii.dataLoader.dataSource.writable", + options: { + url: "%couchDbUrl/%dbName", + writeMethod: "DELETE" + } + }, + createDbDataSource: { + type: "gpii.dataLoader.dataSource.writable", + options: { + url: "%couchDbUrl/%dbName", + writeMethod: "PUT" + } + }, + loadDataSource: { + type: "gpii.dataLoader.dataSource.writable", + options: { + url: "%couchDbUrl/%dbName/_bulk_docs", + writeMethod: "POST" + } + } + }, + invokers: { + load: { + funcName: "gpii.dataLoader.load", + args: ["{that}"] + } + } +}); + +gpii.dataLoader.load = function (that) { + var promises = [], + getDbDataSource = that.getDbDataSource, + deleteDbDataSource = that.deleteDbDataSource, + createDbDataSource = that.createDbDataSource; + + // Process databases one by one + fluid.each(that.options.databases, function (dbData, dbName) { + var directModel = { + couchDbUrl: that.options.couchDbUrl, + dbName: dbName + }; + + // check the existence of the database + var getDbPromise = that.getDbDataSource.get(directModel); + getDbPromise.then(function () { + // The database already exists, delete it + gpii.dataLoader.deleteDb(that); + }); + var dataPaths = fluid.makeArray(dbData.data); + fluid.each(dataPaths, function (onePath) { + console.log("=== onePath: ", onePath); + var actualPath = fluid.module.resolvePath(onePath); + var data = require(actualPath); + var setPromise = that.loadDataSource.set(directModel, data); + promises.push(setPromise); + }); + }); + + // An sequence with an empty array of promises will automatically be resolved, so we can safely use this construct. + return fluid.promise.sequence(promises); +}; From 64c175eb433bf163fbfcba983525c040051bb124 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 21 Nov 2016 10:17:52 -0500 Subject: [PATCH 04/35] GPII-1987: In progress of loading the authorization data. --- scripts/authDataLoader.js | 15 +++++++++------ scripts/shared/dataLoader.js | 33 ++++++++++++++++++++------------- 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/scripts/authDataLoader.js b/scripts/authDataLoader.js index 8efc6081d..0f53b1ffa 100644 --- a/scripts/authDataLoader.js +++ b/scripts/authDataLoader.js @@ -15,6 +15,11 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt "use strict"; +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + +fluid.setLogging(true); + require("./shared/dataLoader.js"); fluid.defaults("gpii.dataLoader.authDataLoader", { @@ -27,13 +32,11 @@ fluid.defaults("gpii.dataLoader.authDataLoader", { ] } }, - couchDbUrl: "http://localhost:5984", + couchDbUrl: "http://admin:admin@localhost:5984", listeners: { - listeners: { - "onCreate.load": { - listener: "gpii.dataLoader.authDataLoader.loadData", - args: ["{that}"] - } + "onCreate.load": { + listener: "gpii.dataLoader.authDataLoader.loadData", + args: ["{that}"] } } }); diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index b3f06554d..777a778b9 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -27,7 +27,6 @@ var fluid = require("infusion"), // Data Source used to interact with CouchDB fluid.defaults("gpii.dataLoader.dataSource", { gradeNames: ["kettle.dataSource.URL", "kettle.dataSource.CouchDB"], - url: "%couchDbUrl/%dbName/_bulk_docs", termMap: { couchDbUrl: "noencode:%couchDbUrl", dbName: "%dbName" @@ -35,7 +34,7 @@ fluid.defaults("gpii.dataLoader.dataSource", { }); fluid.defaults("gpii.dataLoader.dataSource.writable", { - gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.CouchDB.writable"], + gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.writable", "kettle.dataSource.CouchDB.writable"], writable: true }); @@ -90,10 +89,7 @@ fluid.defaults("gpii.dataLoader", { }); gpii.dataLoader.load = function (that) { - var promises = [], - getDbDataSource = that.getDbDataSource, - deleteDbDataSource = that.deleteDbDataSource, - createDbDataSource = that.createDbDataSource; + var promises = []; // Process databases one by one fluid.each(that.options.databases, function (dbData, dbName) { @@ -102,15 +98,15 @@ gpii.dataLoader.load = function (that) { dbName: dbName }; - // check the existence of the database - var getDbPromise = that.getDbDataSource.get(directModel); - getDbPromise.then(function () { - // The database already exists, delete it - gpii.dataLoader.deleteDb(that); - }); + // Delete the database if it already exists + promises.push(gpii.dataLoader.cleanUpDb(that, directModel)); + + // Create the database + var createDbPromise = that.createDbDataSource.set(directModel); + promises.push(createDbPromise); + var dataPaths = fluid.makeArray(dbData.data); fluid.each(dataPaths, function (onePath) { - console.log("=== onePath: ", onePath); var actualPath = fluid.module.resolvePath(onePath); var data = require(actualPath); var setPromise = that.loadDataSource.set(directModel, data); @@ -121,3 +117,14 @@ gpii.dataLoader.load = function (that) { // An sequence with an empty array of promises will automatically be resolved, so we can safely use this construct. return fluid.promise.sequence(promises); }; + +// Delete the database if it already exists, otherwise, do nothing +gpii.dataLoader.cleanUpDb = function (that, directModel) { + var promiseTogo = fluid.promise(); + var getDbPromise = that.getDbDataSource.get(directModel); + getDbPromise.then(function () { + // The database already exists, delete it + promiseTogo = that.deleteDbDataSource.set(directModel); + }); + return promiseTogo; +}; From 22e41d1444d9ae7d4cc1a8c6e0c647c44c048cca Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 9 Dec 2016 11:32:28 -0500 Subject: [PATCH 05/35] GPII-1987: Dropped the use of kettle.dataSource.CouchDB by using kettle.dataSource.URL because for creating and loading data into CouchDB, a pre-fetch feature provided by kettle.dataSource.CouchDB for checking on _revision value is not needed. --- scripts/shared/dataLoader.js | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 777a778b9..7409b32ca 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -26,7 +26,8 @@ var fluid = require("infusion"), // Data Source used to interact with CouchDB fluid.defaults("gpii.dataLoader.dataSource", { - gradeNames: ["kettle.dataSource.URL", "kettle.dataSource.CouchDB"], + gradeNames: ["kettle.dataSource.URL"], + readOnlyGrade: "gpii.dataLoader.dataSource", termMap: { couchDbUrl: "noencode:%couchDbUrl", dbName: "%dbName" @@ -34,7 +35,7 @@ fluid.defaults("gpii.dataLoader.dataSource", { }); fluid.defaults("gpii.dataLoader.dataSource.writable", { - gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.writable", "kettle.dataSource.CouchDB.writable"], + gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.writable"], writable: true }); @@ -66,12 +67,24 @@ fluid.defaults("gpii.dataLoader", { } }, createDbDataSource: { - type: "gpii.dataLoader.dataSource.writable", + type: "gpii.dataLoader.dataSource", options: { url: "%couchDbUrl/%dbName", - writeMethod: "PUT" + method: "PUT" } }, + // createDbDataSource: { + // type: "gpii.dataLoader.dataSource.writable", + // options: { + // url: "%couchDbUrl/%dbName", + // writeMethod: "PUT", + // components: { + // encoding: { + // type: "kettle.dataSource.encoding.none" + // } + // } + // } + // }, loadDataSource: { type: "gpii.dataLoader.dataSource.writable", options: { @@ -102,7 +115,7 @@ gpii.dataLoader.load = function (that) { promises.push(gpii.dataLoader.cleanUpDb(that, directModel)); // Create the database - var createDbPromise = that.createDbDataSource.set(directModel); + var createDbPromise = that.createDbDataSource.get(directModel); promises.push(createDbPromise); var dataPaths = fluid.makeArray(dbData.data); From de78b8645afc2f2f98e5b7b7c5c75ae0da353b12 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 16 Dec 2016 13:57:41 -0500 Subject: [PATCH 06/35] GPII-1987: Completed the script for loading authorization data into CouchDB. --- scripts/authDataLoader.js | 8 +++- scripts/shared/dataLoader.js | 74 +++++++++++++++++++----------------- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/scripts/authDataLoader.js b/scripts/authDataLoader.js index 0f53b1ffa..421de2c14 100644 --- a/scripts/authDataLoader.js +++ b/scripts/authDataLoader.js @@ -11,7 +11,11 @@ You may obtain a copy of the License at https://github.com/GPII/universal/blob/master/LICENSE.txt */ -// The data loader to load GPII authorization server data into CouchDB +/** + * The data loader to load GPII authorization server data into CouchDB. + * Before running this script, make sure CouchDB is started and ready for handling + * database operation requests. + */ "use strict"; @@ -46,7 +50,7 @@ gpii.dataLoader.authDataLoader.loadData = function (that) { promise.then(function () { console.log("The authorization data has been loaded successfully."); }, function (err) { - console.log("Error at loading the authorization data: ", err); + console.log("Error at loading the authorization data. Error details: ", err); }); }; diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 7409b32ca..0bf114195 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -15,7 +15,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt // to load GPII test data into CouchDB. // Steps to load data: // 1. Check the existence of the database; -// 2. If the database already exists, delete it to ensure a clean load; otherwise, create the database; +// 2. If the database already exists, delete and recreate the database to ensure a clean data load; otherwise, create the database; // 3. Load data in bulk using CouchDB /_bulk_docs request - http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs "use strict"; @@ -35,7 +35,7 @@ fluid.defaults("gpii.dataLoader.dataSource", { }); fluid.defaults("gpii.dataLoader.dataSource.writable", { - gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.writable"], + gradeNames: ["gpii.dataLoader.dataSource", "kettle.dataSource.URL.writable"], writable: true }); @@ -60,10 +60,10 @@ fluid.defaults("gpii.dataLoader", { } }, deleteDbDataSource: { - type: "gpii.dataLoader.dataSource.writable", + type: "gpii.dataLoader.dataSource", options: { url: "%couchDbUrl/%dbName", - writeMethod: "DELETE" + method: "DELETE" } }, createDbDataSource: { @@ -73,18 +73,6 @@ fluid.defaults("gpii.dataLoader", { method: "PUT" } }, - // createDbDataSource: { - // type: "gpii.dataLoader.dataSource.writable", - // options: { - // url: "%couchDbUrl/%dbName", - // writeMethod: "PUT", - // components: { - // encoding: { - // type: "kettle.dataSource.encoding.none" - // } - // } - // } - // }, loadDataSource: { type: "gpii.dataLoader.dataSource.writable", options: { @@ -102,6 +90,7 @@ fluid.defaults("gpii.dataLoader", { }); gpii.dataLoader.load = function (that) { + var promiseTogo = fluid.promise(); var promises = []; // Process databases one by one @@ -112,32 +101,49 @@ gpii.dataLoader.load = function (that) { }; // Delete the database if it already exists - promises.push(gpii.dataLoader.cleanUpDb(that, directModel)); - - // Create the database - var createDbPromise = that.createDbDataSource.get(directModel); - promises.push(createDbPromise); - - var dataPaths = fluid.makeArray(dbData.data); - fluid.each(dataPaths, function (onePath) { - var actualPath = fluid.module.resolvePath(onePath); - var data = require(actualPath); - var setPromise = that.loadDataSource.set(directModel, data); - promises.push(setPromise); + var prepareDbPromise = gpii.dataLoader.prepareDB(that, directModel); + + prepareDbPromise.then(function () { + var dataPaths = fluid.makeArray(dbData.data); + fluid.each(dataPaths, function (onePath) { + var actualPath = fluid.module.resolvePath(onePath); + + // Convert the couchDB accepted doc format for using /_bulk_docs end point + var data = { + docs: require(actualPath) + }; + var setPromise = that.loadDataSource.set(directModel, data); + promises.push(setPromise); + }); + var loadDataPromise = fluid.promise.sequence(promises); + fluid.promise.follow(loadDataPromise, promiseTogo); }); }); - // An sequence with an empty array of promises will automatically be resolved, so we can safely use this construct. - return fluid.promise.sequence(promises); + return promiseTogo; }; -// Delete the database if it already exists, otherwise, do nothing -gpii.dataLoader.cleanUpDb = function (that, directModel) { +gpii.dataLoader.createDb = function (that, directModel) { + return that.createDbDataSource.get(directModel); +}; + +// If the database already exists, delete and recreate; otherwise, create the database. +gpii.dataLoader.prepareDB = function (that, directModel) { var promiseTogo = fluid.promise(); var getDbPromise = that.getDbDataSource.get(directModel); + getDbPromise.then(function () { - // The database already exists, delete it - promiseTogo = that.deleteDbDataSource.set(directModel); + // The database already exists, delete and recreate to ensure a clean database for loading data + var deleteDbPromise = that.deleteDbDataSource.get(directModel); + deleteDbPromise.then(function () { + var createDbPromise = gpii.dataLoader.createDb(that, directModel); + fluid.promise.follow(createDbPromise, promiseTogo); + }); + }, function () { + // The database does not exist, create it + var createDbPromise = gpii.dataLoader.createDb(that, directModel); + fluid.promise.follow(createDbPromise, promiseTogo); }); + return promiseTogo; }; From 19b8436f9bef60e4c257aaa4c8c9b946748899cc Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 19 Dec 2016 16:24:40 -0500 Subject: [PATCH 07/35] GPII-1987: In progress of creating the preferences data loader. --- scripts/prefsDataLoader.js | 108 ++++++++++++++++++++++++++ scripts/shared/dataLoader.js | 12 ++- tests/scripts/PrefsDataLoaderTests.js | 81 +++++++++++++++++++ tests/scripts/data/carla.json | 16 ++++ tests/scripts/data/franklin.md | 17 ++++ tests/scripts/data/sammy.json | 19 +++++ tests/scripts/data/unused/david.json | 33 ++++++++ 7 files changed, 282 insertions(+), 4 deletions(-) create mode 100644 scripts/prefsDataLoader.js create mode 100644 tests/scripts/PrefsDataLoaderTests.js create mode 100644 tests/scripts/data/carla.json create mode 100644 tests/scripts/data/franklin.md create mode 100644 tests/scripts/data/sammy.json create mode 100644 tests/scripts/data/unused/david.json diff --git a/scripts/prefsDataLoader.js b/scripts/prefsDataLoader.js new file mode 100644 index 000000000..7ef816d2e --- /dev/null +++ b/scripts/prefsDataLoader.js @@ -0,0 +1,108 @@ +/*! +Copyright 2016 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +/** + * The data loader to load GPII preferences data into CouchDB. + * Before running this script, make sure CouchDB is started and ready for handling + * database operation requests. + */ + +"use strict"; + +var fluid = require("infusion"), + fs = require("fs"), + gpii = fluid.registerNamespace("gpii"); + +fluid.setLogging(true); + +require("./shared/dataLoader.js"); + +fluid.defaults("gpii.dataLoader.prefsDataLoader", { + gradeNames: ["gpii.dataLoader"], + couchDbUrl: "http://admin:admin@localhost:5984", + components: { + dataConverter: { + type: "gpii.dataLoader.prefsDataLoader.dataConverter" + }, + loader: { + type: "gpii.dataLoader", + createOnEvent: "onCreateLoader", + databases: "{arguments}.0" + } + }, + events: { + onCreateLoader: null + }, + listeners: { + "onCreate.load": { + listener: "gpii.dataLoader.prefsDataLoader.loadData", + args: ["{that}"] + } + }, + distributeOptions: { + "couchDbUrl": { + source: "{that}.options.couchDbUrl", + target: "{that > loader}.options.couchDbUrl" + } + } +}); + +gpii.dataLoader.prefsDataLoader.loadData = function (that) { + var promise = that.load(); + promise.then(function () { + console.log("The authorization data has been loaded successfully."); + }, function (err) { + console.log("Error at loading the authorization data. Error details: ", err); + }); +}; + +// gpii.dataLoader.prefsDataLoader(); + +/* + * Converts the physical data files into the structure that can be imported into CouchDB + */ +fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { + gradeNames: ["fluid.component"], + dataPath: "%universal/testData/preferences/", + events: { + onPrefsDataStructureConstructed: null + }, + listeners: { + "onCreate.constructPrefsDataStructure": { + listener: "gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure", + args: ["{that}.options.dataPath", "{that}.events.onPrefsDataStructureConstructed"] + } + } + }); + +gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = function (dataPath, onPrefsDataStructureConstructedEvent) { + var prefsDataStructure = []; + var actualDataPath = fluid.module.resolvePath(dataPath); + var prefsDataList = fs.readdirSync(actualDataPath); + + fluid.each(prefsDataList, function (filename) { + var fileExtension = filename.slice(-5); + // Only json files contain preferences data + if (fileExtension === ".json") { + var gpiiToken = filename.substring(0, filename.length - 5); + var fullPath = actualDataPath + filename; + + var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); + var prefsData = {}; + fluid.set(prefsData, gpiiToken, data); + prefsDataStructure.push(prefsData); + } + }); + + onPrefsDataStructureConstructedEvent.fire(prefsDataStructure); +}; diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 0bf114195..eaf54117a 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -105,13 +105,17 @@ gpii.dataLoader.load = function (that) { prepareDbPromise.then(function () { var dataPaths = fluid.makeArray(dbData.data); - fluid.each(dataPaths, function (onePath) { - var actualPath = fluid.module.resolvePath(onePath); - + fluid.each(dataPaths, function (oneData) { + if (fluid.isPrimitive(oneData)) { + // load data from a physical file + var actualPath = fluid.module.resolvePath(oneData); + oneData = require(actualPath); + } // Convert the couchDB accepted doc format for using /_bulk_docs end point var data = { - docs: require(actualPath) + docs: oneData }; + var setPromise = that.loadDataSource.set(directModel, data); promises.push(setPromise); }); diff --git a/tests/scripts/PrefsDataLoaderTests.js b/tests/scripts/PrefsDataLoaderTests.js new file mode 100644 index 000000000..af417b926 --- /dev/null +++ b/tests/scripts/PrefsDataLoaderTests.js @@ -0,0 +1,81 @@ +/*! + * GPII Settings Transformer Tests + * + * Copyright 2016 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = fluid || require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +require("../../scripts/prefsDataLoader.js"); + +(function () { + fluid.registerNamespace("gpii.tests.prefsDataLoader"); + + fluid.defaults("gpii.tests.prefsDataLoader.dataConverter", { + gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], + dataPath: "data/", + expected: [{ + carla: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + } + }, { + sammy: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + } + }] + }); + + jqUnit.asyncTest("Test gpii.dataLoader.prefsDataLoader.dataConverter", function () { + gpii.tests.prefsDataLoader.dataConverter({ + listeners: { + "onPrefsDataStructureConstructed.assert": { + listener: "jqUnit.assertDeepEq", + args: ["The converted data structure is expected", "{that}.options.expected", "{arguments}.0"] + }, + "onPrefsDataStructureConstructed.runTests": { + listener: "jqUnit.start", + priority: "last" + } + } + }); + }); +})(); diff --git a/tests/scripts/data/carla.json b/tests/scripts/data/carla.json new file mode 100644 index 000000000..3bc9c517f --- /dev/null +++ b/tests/scripts/data/carla.json @@ -0,0 +1,16 @@ +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } +} diff --git a/tests/scripts/data/franklin.md b/tests/scripts/data/franklin.md new file mode 100644 index 000000000..18886de46 --- /dev/null +++ b/tests/scripts/data/franklin.md @@ -0,0 +1,17 @@ +# chris.json + +## Weekly Integration Tests: Preference Sets + +This preference set is used for the +[weekly integration tests](http://wiki.gpii.net/w/Weekly_Integration_Test_Plan) and is manually tested on a regular basis. The expected system behaviour on login is described here. + +It has also been used in the [third review of Cloud4all](https://github.com/GPII/universal/blob/master/testData/preferences/review3/review3-preferences.md). + +## Details +`franklin.json`: preference set with a combination of common and application-specific terms for the following settings: + * Read&Write Gold settings: + * App bar width, + * show text on the app bar, + * optional tool bar shows large icons and text, + * SAPI5 speed set to 50, + * Rule-Based Matchmaker. diff --git a/tests/scripts/data/sammy.json b/tests/scripts/data/sammy.json new file mode 100644 index 000000000..459103b01 --- /dev/null +++ b/tests/scripts/data/sammy.json @@ -0,0 +1,19 @@ +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } +} \ No newline at end of file diff --git a/tests/scripts/data/unused/david.json b/tests/scripts/data/unused/david.json new file mode 100644 index 000000000..92125aefe --- /dev/null +++ b/tests/scripts/data/unused/david.json @@ -0,0 +1,33 @@ +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarIconSet.$t": "Professional", + "ApplicationSettings.AppBar.optToolbarButtonGroupNameCurrent.$t": "Reading Features", + "ApplicationSettings.AppBar.DocType.$t": "1", + "ApplicationSettings.AppBar.Width.$t": 788, + "ApplicationSettings.AppBar.ShowText.$t": true, + "ApplicationSettings.AppBar.optToolbarShowText.$t": true, + "ApplicationSettings.AppBar.LargeIcons.$t": true, + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Pitch.$t": 33, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 31, + "ApplicationSettings.Speech.optSAPI5Volume.$t": 90, + "ApplicationSettings.Speech.optSAPI5PauseBetweenWords.$t": 115, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false, + "ApplicationSettings.Speech.optSAPI5Voice.$t": "ScanSoft UK Indian Sangeeta", + "ApplicationSettings.Speech.WebHighlighting.$t": true, + "ApplicationSettings.Translation.ToLanguage.$t": "hi", + "ApplicationSettings.Speech.optSAPI5SpeechHighlightContext.$t": 0, + "ApplicationSettings.Scanning.ScanDestination.$t": "Word", + "ApplicationSettings.Scanning.ScanToFile.$t": true, + "ApplicationSettings.Spelling.SpellAsIType.$t": false + } + } + } + } + } +} \ No newline at end of file From a27423ba97c5a7ee0d60d0d78d2ea7a66718762b Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 20 Dec 2016 20:15:46 -0500 Subject: [PATCH 08/35] GPII-1987: In progress of creating preferences data loader. --- scripts/authDataLoader.js | 2 +- scripts/prefsDataLoader.js | 85 ++----------------- scripts/shared/dataLoader-prefs.js | 127 +++++++++++++++++++++++++++++ scripts/shared/dataLoader.js | 50 ++++++++---- 4 files changed, 167 insertions(+), 97 deletions(-) create mode 100644 scripts/shared/dataLoader-prefs.js diff --git a/scripts/authDataLoader.js b/scripts/authDataLoader.js index 421de2c14..a518c2600 100644 --- a/scripts/authDataLoader.js +++ b/scripts/authDataLoader.js @@ -30,7 +30,7 @@ fluid.defaults("gpii.dataLoader.authDataLoader", { gradeNames: ["gpii.dataLoader"], databases: { auth: { - data: [ + dataFile: [ "%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] diff --git a/scripts/prefsDataLoader.js b/scripts/prefsDataLoader.js index 7ef816d2e..c1e67c606 100644 --- a/scripts/prefsDataLoader.js +++ b/scripts/prefsDataLoader.js @@ -20,89 +20,14 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt "use strict"; var fluid = require("infusion"), - fs = require("fs"), gpii = fluid.registerNamespace("gpii"); fluid.setLogging(true); -require("./shared/dataLoader.js"); +require("./shared/dataLoader-prefs.js"); -fluid.defaults("gpii.dataLoader.prefsDataLoader", { - gradeNames: ["gpii.dataLoader"], - couchDbUrl: "http://admin:admin@localhost:5984", - components: { - dataConverter: { - type: "gpii.dataLoader.prefsDataLoader.dataConverter" - }, - loader: { - type: "gpii.dataLoader", - createOnEvent: "onCreateLoader", - databases: "{arguments}.0" - } - }, - events: { - onCreateLoader: null - }, - listeners: { - "onCreate.load": { - listener: "gpii.dataLoader.prefsDataLoader.loadData", - args: ["{that}"] - } - }, - distributeOptions: { - "couchDbUrl": { - source: "{that}.options.couchDbUrl", - target: "{that > loader}.options.couchDbUrl" - } - } +gpii.dataLoader.prefsDataLoader({ + dataPath: "%universal/tests/scripts/data/", + // dataPath: "%universal/testData/preferences/", + couchDbUrl: "http://admin:admin@localhost:5984" }); - -gpii.dataLoader.prefsDataLoader.loadData = function (that) { - var promise = that.load(); - promise.then(function () { - console.log("The authorization data has been loaded successfully."); - }, function (err) { - console.log("Error at loading the authorization data. Error details: ", err); - }); -}; - -// gpii.dataLoader.prefsDataLoader(); - -/* - * Converts the physical data files into the structure that can be imported into CouchDB - */ -fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { - gradeNames: ["fluid.component"], - dataPath: "%universal/testData/preferences/", - events: { - onPrefsDataStructureConstructed: null - }, - listeners: { - "onCreate.constructPrefsDataStructure": { - listener: "gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure", - args: ["{that}.options.dataPath", "{that}.events.onPrefsDataStructureConstructed"] - } - } - }); - -gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = function (dataPath, onPrefsDataStructureConstructedEvent) { - var prefsDataStructure = []; - var actualDataPath = fluid.module.resolvePath(dataPath); - var prefsDataList = fs.readdirSync(actualDataPath); - - fluid.each(prefsDataList, function (filename) { - var fileExtension = filename.slice(-5); - // Only json files contain preferences data - if (fileExtension === ".json") { - var gpiiToken = filename.substring(0, filename.length - 5); - var fullPath = actualDataPath + filename; - - var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); - var prefsData = {}; - fluid.set(prefsData, gpiiToken, data); - prefsDataStructure.push(prefsData); - } - }); - - onPrefsDataStructureConstructedEvent.fire(prefsDataStructure); -}; diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/dataLoader-prefs.js new file mode 100644 index 000000000..fae3a4516 --- /dev/null +++ b/scripts/shared/dataLoader-prefs.js @@ -0,0 +1,127 @@ +/*! +Copyright 2016 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +/** + * The components required by ../prefsDataLoader.js + */ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"), + fs = require("fs"), + JSON5 = require("json5"); + +fluid.setLogging(true); + +require("./dataLoader.js"); + +fluid.defaults("gpii.dataLoader.prefsDataLoader", { + gradeNames: ["fluid.component"], + components: { + dataConverter: { + type: "gpii.dataLoader.prefsDataLoader.dataConverter", + options: { + listeners: { + "onPrefsDataStructureConstructed.escalate": { + listener: "{prefsDataLoader}.events.onCreateLoader.fire", + args: ["{arguments}.0"] + } + } + } + }, + loader: { + type: "gpii.dataLoader", + createOnEvent: "onCreateLoader", + options: { + databases: { + preferences: { + data: "{arguments}.0" + } + }, + listeners: { + "onCreate.escalate": "{prefsDataLoader}.events.onLoaderReady.fire" + } + } + } + }, + events: { + onCreateLoader: null, + onLoaderReady: null + }, + listeners: { + "onLoaderReady.load": { + listener: "gpii.dataLoader.prefsDataLoader.loadData", + args: ["{that}.loader"] + } + }, + distributeOptions: { + "couchDbUrl": { + source: "{that}.options.couchDbUrl", + target: "{that > loader}.options.couchDbUrl" + }, + "dataPath": { + source: "{that}.options.dataPath", + target: "{that > dataConverter}.options.dataPath" + } + } +}); + +gpii.dataLoader.prefsDataLoader.loadData = function (loader) { + var promise = loader.load(); + promise.then(function () { + console.log("The preferences data has been loaded successfully."); + }, function (err) { + console.log("Error at loading the preferences data. Error details: ", err); + }); +}; + +/* + * Converts the physical data files into the structure that can be imported into CouchDB + */ +fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { + gradeNames: ["fluid.component"], + dataPath: "%universal/testData/preferences/", + events: { + onPrefsDataStructureConstructed: null + }, + listeners: { + "onCreate.constructPrefsDataStructure": { + listener: "gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure", + args: ["{that}.options.dataPath", "{that}.events.onPrefsDataStructureConstructed"] + } + } + }); + +gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = function (dataPath, onPrefsDataStructureConstructedEvent) { + var prefsDataStructure = []; + var actualDataPath = fluid.module.resolvePath(dataPath); + var prefsDataList = fs.readdirSync(actualDataPath); + + fluid.each(prefsDataList, function (filename) { + var fileExtension = filename.slice(-5); + // Only json files contain preferences data + if (fileExtension === ".json") { + var gpiiToken = filename.substring(0, filename.length - 5); + var fullPath = actualDataPath + filename; + + var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); + var prefsData = {}; + // fluid.set(prefsData, "a", data); + fluid.set(prefsData, gpiiToken, data); + prefsDataStructure.push(prefsData); + } + }); + + onPrefsDataStructureConstructedEvent.fire(prefsDataStructure); +}; diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index eaf54117a..1f4b99d7c 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -24,6 +24,8 @@ var fluid = require("infusion"), kettle = require("kettle"), gpii = fluid.registerNamespace("gpii"); +var fs = require("fs"); + // Data Source used to interact with CouchDB fluid.defaults("gpii.dataLoader.dataSource", { gradeNames: ["kettle.dataSource.URL"], @@ -91,7 +93,7 @@ fluid.defaults("gpii.dataLoader", { gpii.dataLoader.load = function (that) { var promiseTogo = fluid.promise(); - var promises = []; + var loadDataPromises = []; // Process databases one by one fluid.each(that.options.databases, function (dbData, dbName) { @@ -104,22 +106,16 @@ gpii.dataLoader.load = function (that) { var prepareDbPromise = gpii.dataLoader.prepareDB(that, directModel); prepareDbPromise.then(function () { - var dataPaths = fluid.makeArray(dbData.data); - fluid.each(dataPaths, function (oneData) { - if (fluid.isPrimitive(oneData)) { - // load data from a physical file - var actualPath = fluid.module.resolvePath(oneData); - oneData = require(actualPath); - } - // Convert the couchDB accepted doc format for using /_bulk_docs end point - var data = { - docs: oneData - }; - - var setPromise = that.loadDataSource.set(directModel, data); - promises.push(setPromise); + // load data files + var dataFilePaths = fluid.makeArray(dbData.dataFile); + fluid.each(dataFilePaths, function (oneDataFile) { + loadDataPromises.push(gpii.dataLoader.loadDataFile(that.loadDataSource, oneDataFile, directModel)); }); - var loadDataPromise = fluid.promise.sequence(promises); + + // load direct data + loadDataPromises.push(gpii.dataLoader.loadData(that.loadDataSource, dbData.data, directModel)); + + var loadDataPromise = fluid.promise.sequence(loadDataPromises); fluid.promise.follow(loadDataPromise, promiseTogo); }); }); @@ -151,3 +147,25 @@ gpii.dataLoader.prepareDB = function (that, directModel) { return promiseTogo; }; + +gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { + if (!data) { + return fluid.promise(); + } + + // Convert to couchDB accepted doc format for using /_bulk_docs end point + var data = { + "docs": data + }; + + console.log("data: ", data.docs[0].condTest); + fs.writeFileSync(__dirname + "/debug.log", data, "utf-8"); + return loadDataSource.set(directModel, data); +}; + +gpii.dataLoader.loadDataFile = function (loadDataSource, dataFile, directModel) { + var actualPath = fluid.module.resolvePath(dataFile); + var data = require(actualPath); + + return gpii.dataLoader.loadData(loadDataSource, data, directModel); +}; From a06801d0ab676d9220a6f5b9c76d88e96ca6b127 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 21 Dec 2016 16:56:27 -0500 Subject: [PATCH 09/35] GPII-1987: A little adjustment on constructing the preferences data structure for CouchDB import. --- scripts/shared/dataLoader-prefs.js | 16 ++++++++++++---- scripts/shared/dataLoader.js | 5 +---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/dataLoader-prefs.js index fae3a4516..27a0e4dcd 100644 --- a/scripts/shared/dataLoader-prefs.js +++ b/scripts/shared/dataLoader-prefs.js @@ -115,11 +115,19 @@ gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = func var gpiiToken = filename.substring(0, filename.length - 5); var fullPath = actualDataPath + filename; - var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); + // var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); + var data = { + "a": "Gerät" + }; + + // The actual preferences data needs to be the value of a field named "value" when being imported into CouchDB var prefsData = {}; - // fluid.set(prefsData, "a", data); - fluid.set(prefsData, gpiiToken, data); - prefsDataStructure.push(prefsData); + fluid.set(prefsData, "value", data); + + var finalData = fluid.extend(prefsData, { + "_id": gpiiToken + }); + prefsDataStructure.push(finalData); } }); diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 1f4b99d7c..1e1e9a263 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -24,8 +24,6 @@ var fluid = require("infusion"), kettle = require("kettle"), gpii = fluid.registerNamespace("gpii"); -var fs = require("fs"); - // Data Source used to interact with CouchDB fluid.defaults("gpii.dataLoader.dataSource", { gradeNames: ["kettle.dataSource.URL"], @@ -158,8 +156,7 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { "docs": data }; - console.log("data: ", data.docs[0].condTest); - fs.writeFileSync(__dirname + "/debug.log", data, "utf-8"); + console.log("data: ", data); return loadDataSource.set(directModel, data); }; From 745bbfcea81ddff677c8a9f6880b836f339beb3f Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 23 Dec 2016 11:40:49 -0500 Subject: [PATCH 10/35] GPII-1987: Completed on the preferences data loader. --- scripts/prefsDataLoader.js | 3 +-- scripts/shared/dataLoader-prefs.js | 12 ++++++++---- scripts/shared/dataLoader.js | 11 +++++++---- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/scripts/prefsDataLoader.js b/scripts/prefsDataLoader.js index c1e67c606..22ea20959 100644 --- a/scripts/prefsDataLoader.js +++ b/scripts/prefsDataLoader.js @@ -27,7 +27,6 @@ fluid.setLogging(true); require("./shared/dataLoader-prefs.js"); gpii.dataLoader.prefsDataLoader({ - dataPath: "%universal/tests/scripts/data/", - // dataPath: "%universal/testData/preferences/", + dataPath: "%universal/testData/preferences/", couchDbUrl: "http://admin:admin@localhost:5984" }); diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/dataLoader-prefs.js index 27a0e4dcd..9e4c299d7 100644 --- a/scripts/shared/dataLoader-prefs.js +++ b/scripts/shared/dataLoader-prefs.js @@ -26,6 +26,13 @@ fluid.setLogging(true); require("./dataLoader.js"); +/** + * This component is composed by two sub-components: + * 1. dataConverter: to read thru the preferences data and construct the json file contents + * into the data structure that can be accepted by CouchDB /_bulk_docs API + * 2. loader: take the data structure created by dataConverter and load into CouchDB. Using + * "gpii.dataLoader" as the underlying grade for loading. + */ fluid.defaults("gpii.dataLoader.prefsDataLoader", { gradeNames: ["fluid.component"], components: { @@ -115,10 +122,7 @@ gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = func var gpiiToken = filename.substring(0, filename.length - 5); var fullPath = actualDataPath + filename; - // var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); - var data = { - "a": "Gerät" - }; + var data = JSON.parse(fs.readFileSync(fullPath, "utf8")); // The actual preferences data needs to be the value of a field named "value" when being imported into CouchDB var prefsData = {}; diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 1e1e9a263..cbbce5e92 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -42,12 +42,16 @@ fluid.defaults("gpii.dataLoader.dataSource.writable", { // Data loader fluid.defaults("gpii.dataLoader", { gradeNames: ["fluid.component"], - // Accepted format: + // Accepted formats: // dbName1: { - // data: ["pathToDataFile1", "pathToDataFile1"...] + // dataFile: ["pathToDataFile1", "pathToDataFile1"...] // }, // dbName2: { - // data: ["pathToDataFile1", "pathToDataFile1"...] + // data: [{ + // ... + // }, { + // ... + // }...] // } databases: { // Supplied by integrators }, @@ -156,7 +160,6 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { "docs": data }; - console.log("data: ", data); return loadDataSource.set(directModel, data); }; From 23762e96e77c16f8256b89e602b5ce3d242b7bc5 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 3 Jan 2017 14:18:14 -0500 Subject: [PATCH 11/35] GPII-1987: Allow to pass the data base name as an option. Added code comments. --- scripts/authDataLoader.js | 36 +++++++++++++---- scripts/prefsDataLoader.js | 1 + scripts/shared/dataLoader-prefs.js | 35 ++++++++++++++-- scripts/shared/dataLoader.js | 64 +++++++++++++++++++++++++----- 4 files changed, 116 insertions(+), 20 deletions(-) diff --git a/scripts/authDataLoader.js b/scripts/authDataLoader.js index a518c2600..2ed6d8e74 100644 --- a/scripts/authDataLoader.js +++ b/scripts/authDataLoader.js @@ -29,14 +29,11 @@ require("./shared/dataLoader.js"); fluid.defaults("gpii.dataLoader.authDataLoader", { gradeNames: ["gpii.dataLoader"], databases: { - auth: { - dataFile: [ - "%universal/testData/security/TestOAuth2DataStore.json", - "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" - ] + expander: { + funcName: "gpii.dataLoader.authDataLoader.constructAuthData", + args: ["{that}.options.dbName", "{that}.options.dataFile"] } }, - couchDbUrl: "http://admin:admin@localhost:5984", listeners: { "onCreate.load": { listener: "gpii.dataLoader.authDataLoader.loadData", @@ -45,6 +42,24 @@ fluid.defaults("gpii.dataLoader.authDataLoader", { } }); +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param dataFile {Array} An array of data paths to files to be loaded into the database. + */ +gpii.dataLoader.authDataLoader.constructAuthData = function (dbName, dataFile) { + var togo = {}; + fluid.set(togo, dbName + ".dataFile", dataFile); + + return togo; +}; + +/** + * Triggers the loading function to load data. + * @param that {Component} An instance of `gpii.dataLoader` (See dataLoader.js). + * @return {None} + */ gpii.dataLoader.authDataLoader.loadData = function (that) { var promise = that.load(); promise.then(function () { @@ -54,4 +69,11 @@ gpii.dataLoader.authDataLoader.loadData = function (that) { }); }; -gpii.dataLoader.authDataLoader(); +gpii.dataLoader.authDataLoader({ + dbName: "auth", + dataFile: [ + "%universal/testData/security/TestOAuth2DataStore.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ], + couchDbUrl: "http://admin:admin@localhost:5984" +}); diff --git a/scripts/prefsDataLoader.js b/scripts/prefsDataLoader.js index 22ea20959..5dd50ddf7 100644 --- a/scripts/prefsDataLoader.js +++ b/scripts/prefsDataLoader.js @@ -27,6 +27,7 @@ fluid.setLogging(true); require("./shared/dataLoader-prefs.js"); gpii.dataLoader.prefsDataLoader({ + dbName: "preferences", dataPath: "%universal/testData/preferences/", couchDbUrl: "http://admin:admin@localhost:5984" }); diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/dataLoader-prefs.js index 9e4c299d7..25715016c 100644 --- a/scripts/shared/dataLoader-prefs.js +++ b/scripts/shared/dataLoader-prefs.js @@ -51,9 +51,11 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { type: "gpii.dataLoader", createOnEvent: "onCreateLoader", options: { + data: "{arguments}.0", databases: { - preferences: { - data: "{arguments}.0" + expander: { + funcName: "gpii.dataLoader.prefsDataLoader.constructDBOption", + args: ["{prefsDataLoader}.options.dbName", "{that}.options.data"] } }, listeners: { @@ -84,6 +86,24 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { } }); +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param data {Object} The data to be loaded into the database. + */ +gpii.dataLoader.prefsDataLoader.constructDBOption = function (dbName, data) { + var togo = {}; + fluid.set(togo, dbName + ".data", data); + + return togo; +}; + +/** + * Triggers the loading function to load data. + * @param loader {Component} An instance of `gpii.dataLoader` (See dataLoader.js). + * @return {None} + */ gpii.dataLoader.prefsDataLoader.loadData = function (loader) { var promise = loader.load(); promise.then(function () { @@ -94,7 +114,7 @@ gpii.dataLoader.prefsDataLoader.loadData = function (loader) { }; /* - * Converts the physical data files into the structure that can be imported into CouchDB + * Converts the physical preferences data files into the structure that can be imported into CouchDB */ fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { gradeNames: ["fluid.component"], @@ -110,6 +130,15 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { } }); +/** + * Read each preferences data from physical json file and construct them into a data structure that is accepted by + * the CouchDB /_bulk_docs API (http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs) + * + * @param dataPath {String} The data path to where the preferences files are located; + * @param onPrefsDataStructureConstructedEvent {Event} fires when the data structure is constructed. The event is fired with + * an argument that is the constructed data structure. + * @return {Event} fires the data ready event with an argument of the constructed data structure. + */ gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = function (dataPath, onPrefsDataStructureConstructedEvent) { var prefsDataStructure = []; var actualDataPath = fluid.module.resolvePath(dataPath); diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index cbbce5e92..da91c89fe 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -16,7 +16,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt // Steps to load data: // 1. Check the existence of the database; // 2. If the database already exists, delete and recreate the database to ensure a clean data load; otherwise, create the database; -// 3. Load data in bulk using CouchDB /_bulk_docs request - http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs +// 3. Load data in bulk using CouchDB /_bulk_docs API - http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs "use strict"; @@ -93,6 +93,15 @@ fluid.defaults("gpii.dataLoader", { } }); +/** + * The main load function that processes the data loading for databases defined in `that.options.databases` one by one. + * The data can be provided in 2 forms as one or both listed sub-options into `that.options.databases`: + * 1. `dataFile` option: an array of paths to data files. For example: dataFile: ["pathA/a.json", "pathB/b.json"]; + * 2. `data` option: the direct data in the structure that is allowed by CouchDB /_bulk_docs API (http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs) + * + * @param that {Component} An instance of "gpii.dataLoader" + * @return {Promise} The promise of the data loading process. + */ gpii.dataLoader.load = function (that) { var promiseTogo = fluid.promise(); var loadDataPromises = []; @@ -105,7 +114,7 @@ gpii.dataLoader.load = function (that) { }; // Delete the database if it already exists - var prepareDbPromise = gpii.dataLoader.prepareDB(that, directModel); + var prepareDbPromise = gpii.dataLoader.prepareDB(that.getDbDataSource, that.createDbDataSource, that.deleteDbDataSource, directModel); prepareDbPromise.then(function () { // load data files @@ -125,31 +134,57 @@ gpii.dataLoader.load = function (that) { return promiseTogo; }; -gpii.dataLoader.createDb = function (that, directModel) { - return that.createDbDataSource.get(directModel); +/** + * To create a database. + * + * @param createDbDataSource {DataSource} The data source for creating a database; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of creating the database. + */ +gpii.dataLoader.createDb = function (createDbDataSource, directModel) { + return createDbDataSource.get(directModel); }; -// If the database already exists, delete and recreate; otherwise, create the database. -gpii.dataLoader.prepareDB = function (that, directModel) { +/** + * Prepare the database before loading the data. If the database already exists, delete and recreate it; otherwise, create the database. + * + * @param getDbDataSource {DataSource} The data source for checking the existence of a database; + * @param createDbDataSource {DataSource} The data source for creating a database; + * @param deleteDbDataSource {DataSource} The data source for deleting a database; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of preparing the database. + */ +gpii.dataLoader.prepareDB = function (getDbDataSource, createDbDataSource, deleteDbDataSource, directModel) { var promiseTogo = fluid.promise(); - var getDbPromise = that.getDbDataSource.get(directModel); + var getDbPromise = getDbDataSource.get(directModel); getDbPromise.then(function () { // The database already exists, delete and recreate to ensure a clean database for loading data - var deleteDbPromise = that.deleteDbDataSource.get(directModel); + var deleteDbPromise = deleteDbDataSource.get(directModel); deleteDbPromise.then(function () { - var createDbPromise = gpii.dataLoader.createDb(that, directModel); + var createDbPromise = gpii.dataLoader.createDb(createDbDataSource, directModel); fluid.promise.follow(createDbPromise, promiseTogo); }); }, function () { // The database does not exist, create it - var createDbPromise = gpii.dataLoader.createDb(that, directModel); + var createDbPromise = gpii.dataLoader.createDb(createDbDataSource, directModel); fluid.promise.follow(createDbPromise, promiseTogo); }); return promiseTogo; }; +/** + * Load the data into a database. + * + * @param loadDataSource {DataSource} The data source for loading the data; + * @param data {DataSource} The data to be loaded; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of loading the data. + */ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { if (!data) { return fluid.promise(); @@ -163,6 +198,15 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { return loadDataSource.set(directModel, data); }; +/** + * Load a data file into a database. + * + * @param loadDataSource {DataSource} The data source for loading the data; + * @param dataFile {String} The path to a data file; + * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched + * + * @return {Promise} A promise with the response of loading the data file. + */ gpii.dataLoader.loadDataFile = function (loadDataSource, dataFile, directModel) { var actualPath = fluid.module.resolvePath(dataFile); var data = require(actualPath); From 6ce660cb732239027fa6e07160e8f3c5f0ae1648 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 3 Jan 2017 15:50:58 -0500 Subject: [PATCH 12/35] GPII-1987: Splitted authDataLoader.js into 2 js files that one contains the actual implementation of the auth loader component and the other is to call the loader component to perform the loading so that the latter can be enhanced to accept an configuration. --- scripts/authDataLoader.js | 47 ++--------------------------- scripts/prefsDataLoader.js | 2 +- scripts/shared/dataLoader-auth.js | 48 ++++++++++++++++++++++++++++++ scripts/shared/dataLoader-prefs.js | 20 ++----------- scripts/shared/dataLoader.js | 20 +++++++++++-- 5 files changed, 72 insertions(+), 65 deletions(-) create mode 100644 scripts/shared/dataLoader-auth.js diff --git a/scripts/authDataLoader.js b/scripts/authDataLoader.js index 2ed6d8e74..c30bf82b9 100644 --- a/scripts/authDataLoader.js +++ b/scripts/authDataLoader.js @@ -1,5 +1,5 @@ /*! -Copyright 2016 OCAD University +Copyright 2016-2017 OCAD University Licensed under the New BSD license. You may not use this file except in compliance with this License. @@ -24,50 +24,7 @@ var fluid = require("infusion"), fluid.setLogging(true); -require("./shared/dataLoader.js"); - -fluid.defaults("gpii.dataLoader.authDataLoader", { - gradeNames: ["gpii.dataLoader"], - databases: { - expander: { - funcName: "gpii.dataLoader.authDataLoader.constructAuthData", - args: ["{that}.options.dbName", "{that}.options.dataFile"] - } - }, - listeners: { - "onCreate.load": { - listener: "gpii.dataLoader.authDataLoader.loadData", - args: ["{that}"] - } - } -}); - -/** - * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). - * - * @param dbName {String} The database name; - * @param dataFile {Array} An array of data paths to files to be loaded into the database. - */ -gpii.dataLoader.authDataLoader.constructAuthData = function (dbName, dataFile) { - var togo = {}; - fluid.set(togo, dbName + ".dataFile", dataFile); - - return togo; -}; - -/** - * Triggers the loading function to load data. - * @param that {Component} An instance of `gpii.dataLoader` (See dataLoader.js). - * @return {None} - */ -gpii.dataLoader.authDataLoader.loadData = function (that) { - var promise = that.load(); - promise.then(function () { - console.log("The authorization data has been loaded successfully."); - }, function (err) { - console.log("Error at loading the authorization data. Error details: ", err); - }); -}; +require("./shared/dataLoader-auth.js"); gpii.dataLoader.authDataLoader({ dbName: "auth", diff --git a/scripts/prefsDataLoader.js b/scripts/prefsDataLoader.js index 5dd50ddf7..faf751f59 100644 --- a/scripts/prefsDataLoader.js +++ b/scripts/prefsDataLoader.js @@ -1,5 +1,5 @@ /*! -Copyright 2016 OCAD University +Copyright 2016-2017 OCAD University Licensed under the New BSD license. You may not use this file except in compliance with this License. diff --git a/scripts/shared/dataLoader-auth.js b/scripts/shared/dataLoader-auth.js new file mode 100644 index 000000000..41b5207e3 --- /dev/null +++ b/scripts/shared/dataLoader-auth.js @@ -0,0 +1,48 @@ +/*! +Copyright 2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +The research leading to these results has received funding from the European Union's +Seventh Framework Programme (FP7/2007-2013) under grant agreement no. 289016. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +"use strict"; + +var fluid = require("infusion"), + gpii = fluid.registerNamespace("gpii"); + +require("./dataLoader.js"); + +fluid.defaults("gpii.dataLoader.authDataLoader", { + gradeNames: ["gpii.dataLoader"], + databases: { + expander: { + funcName: "gpii.dataLoader.authDataLoader.constructAuthData", + args: ["{that}.options.dbName", "{that}.options.dataFile"] + } + }, + listeners: { + "onCreate.load": { + listener: "gpii.dataLoader.performLoad", + args: ["{that}.options.dbName", "{that}.load"] + } + } +}); + +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param dataFile {Array} An array of data paths to files to be loaded into the database. + */ +gpii.dataLoader.authDataLoader.constructAuthData = function (dbName, dataFile) { + var togo = {}; + fluid.set(togo, dbName + ".dataFile", dataFile); + + return togo; +}; diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/dataLoader-prefs.js index 25715016c..2fb6bf70d 100644 --- a/scripts/shared/dataLoader-prefs.js +++ b/scripts/shared/dataLoader-prefs.js @@ -1,5 +1,5 @@ /*! -Copyright 2016 OCAD University +Copyright 2016-2017 OCAD University Licensed under the New BSD license. You may not use this file except in compliance with this License. @@ -70,8 +70,8 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { }, listeners: { "onLoaderReady.load": { - listener: "gpii.dataLoader.prefsDataLoader.loadData", - args: ["{that}.loader"] + listener: "gpii.dataLoader.performLoad", + args: ["{that}.options.dbName", "{that}.loader.load"] } }, distributeOptions: { @@ -99,20 +99,6 @@ gpii.dataLoader.prefsDataLoader.constructDBOption = function (dbName, data) { return togo; }; -/** - * Triggers the loading function to load data. - * @param loader {Component} An instance of `gpii.dataLoader` (See dataLoader.js). - * @return {None} - */ -gpii.dataLoader.prefsDataLoader.loadData = function (loader) { - var promise = loader.load(); - promise.then(function () { - console.log("The preferences data has been loaded successfully."); - }, function (err) { - console.log("Error at loading the preferences data. Error details: ", err); - }); -}; - /* * Converts the physical preferences data files into the structure that can be imported into CouchDB */ diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index da91c89fe..098f41c9f 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -1,5 +1,5 @@ /*! -Copyright 2016 OCAD University +Copyright 2016-2017 OCAD University Licensed under the New BSD license. You may not use this file except in compliance with this License. @@ -48,7 +48,8 @@ fluid.defaults("gpii.dataLoader", { // }, // dbName2: { // data: [{ - // ... + // "_id": {String}, + // [Other object key-value pairs] // }, { // ... // }...] @@ -213,3 +214,18 @@ gpii.dataLoader.loadDataFile = function (loadDataSource, dataFile, directModel) return gpii.dataLoader.loadData(loadDataSource, data, directModel); }; + +/** + * The utility function to trigger the loading function to load data. + * @param dbName {String} The database name; + * @param loadFunc {Function} The load() invoker defined for an instance of `gpii.dataLoader` component (See dataLoader.js); + * @return {None} The success or fail result of the loading is reported on the console. + */ +gpii.dataLoader.performLoad = function (dbName, loadFunc) { + var promise = loadFunc(); + promise.then(function () { + console.log("The " + dbName + " data has been loaded successfully."); + }, function (err) { + console.log("Error at loading the " + dbName + " data. Error details: ", err); + }); +}; From 7798a4279b02a14ac6d9576b820c3608bc3dfa5f Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 30 Jan 2017 19:54:52 -0500 Subject: [PATCH 13/35] GPII-1987: Refactored data loaders as well as adding test for the auth loader. --- scripts/shared/dataLoader-auth.js | 2 +- scripts/shared/dataLoader-prefs.js | 10 ++- scripts/shared/dataLoader.js | 59 +++++++++--- tests/scripts/AuthDataLoaderTests.js | 123 ++++++++++++++++++++++++++ tests/scripts/DataLoaderTestsUtils.js | 28 ++++++ tests/scripts/PrefsDataLoaderTests.js | 2 +- 6 files changed, 210 insertions(+), 14 deletions(-) create mode 100644 tests/scripts/AuthDataLoaderTests.js create mode 100644 tests/scripts/DataLoaderTestsUtils.js diff --git a/scripts/shared/dataLoader-auth.js b/scripts/shared/dataLoader-auth.js index 41b5207e3..4aaa63fae 100644 --- a/scripts/shared/dataLoader-auth.js +++ b/scripts/shared/dataLoader-auth.js @@ -29,7 +29,7 @@ fluid.defaults("gpii.dataLoader.authDataLoader", { listeners: { "onCreate.load": { listener: "gpii.dataLoader.performLoad", - args: ["{that}.options.dbName", "{that}.load"] + args: ["{that}.options.dbName", "{that}"] } } }); diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/dataLoader-prefs.js index 2fb6bf70d..05bae4beb 100644 --- a/scripts/shared/dataLoader-prefs.js +++ b/scripts/shared/dataLoader-prefs.js @@ -58,6 +58,10 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { args: ["{prefsDataLoader}.options.dbName", "{that}.options.data"] } }, + events: { + onDataLoaded: "{prefsDataLoader}.events.onDataLoaded", + onDataLoadedError: "{prefsDataLoader}.events.onDataLoadedError" + }, listeners: { "onCreate.escalate": "{prefsDataLoader}.events.onLoaderReady.fire" } @@ -66,12 +70,14 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { }, events: { onCreateLoader: null, - onLoaderReady: null + onLoaderReady: null, + onDataLoaded: null, + onDataLoadedError: null }, listeners: { "onLoaderReady.load": { listener: "gpii.dataLoader.performLoad", - args: ["{that}.options.dbName", "{that}.loader.load"] + args: ["{that}.options.dbName", "{that}.loader"] } }, distributeOptions: { diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 098f41c9f..a8f9bfb72 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -22,7 +22,8 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt var fluid = require("infusion"), kettle = require("kettle"), - gpii = fluid.registerNamespace("gpii"); + gpii = fluid.registerNamespace("gpii"), + fs = require("fs"); // Data Source used to interact with CouchDB fluid.defaults("gpii.dataLoader.dataSource", { @@ -91,6 +92,10 @@ fluid.defaults("gpii.dataLoader", { funcName: "gpii.dataLoader.load", args: ["{that}"] } + }, + events: { + onDataLoaded: null, + onDataLoadedError: null } }); @@ -120,12 +125,12 @@ gpii.dataLoader.load = function (that) { prepareDbPromise.then(function () { // load data files var dataFilePaths = fluid.makeArray(dbData.dataFile); - fluid.each(dataFilePaths, function (oneDataFile) { - loadDataPromises.push(gpii.dataLoader.loadDataFile(that.loadDataSource, oneDataFile, directModel)); - }); + loadDataPromises = gpii.dataLoader.loadDataFiles(that.loadDataSource, dataFilePaths, directModel); // load direct data - loadDataPromises.push(gpii.dataLoader.loadData(that.loadDataSource, dbData.data, directModel)); + if (dbData.data) { + loadDataPromises.push(gpii.dataLoader.loadData(that.loadDataSource, dbData.data, directModel)); + } var loadDataPromise = fluid.promise.sequence(loadDataPromises); fluid.promise.follow(loadDataPromise, promiseTogo); @@ -208,24 +213,58 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { * * @return {Promise} A promise with the response of loading the data file. */ +gpii.dataLoader.loadDataFiles = function (loadDataSource, dataFilePaths, directModel) { + // Capture non-found file paths + var unfoundFiles = []; + // Capture data from files if files exist. + var dataContents = []; + var loadDataPromises = []; + + fluid.each(dataFilePaths, function (oneDataFile) { + var actualPath = fluid.module.resolvePath(oneDataFile); + if (!fs.existsSync(actualPath)) { + unfoundFiles.push(actualPath); + } else { + dataContents.push(require(actualPath)); + } + }); + + if (unfoundFiles.length > 0) { + var errorPromise = fluid.promise().reject("Data file(s) not found in the file system: " + unfoundFiles.join()); + loadDataPromises.push(errorPromise); + } else { + fluid.each(dataContents, function (data) { + loadDataPromises.push(gpii.dataLoader.loadData(loadDataSource, data, directModel)); + }); + } + return loadDataPromises; +}; + gpii.dataLoader.loadDataFile = function (loadDataSource, dataFile, directModel) { var actualPath = fluid.module.resolvePath(dataFile); - var data = require(actualPath); + if (!fs.existsSync(actualPath)) { + return fluid.promise().reject("Data file is not found in the file system: " + actualPath); + } else { + var data = require(actualPath); + var promiseTogo = gpii.dataLoader.loadData(loadDataSource, data, directModel); - return gpii.dataLoader.loadData(loadDataSource, data, directModel); + return promiseTogo; + } }; /** - * The utility function to trigger the loading function to load data. + * The API function to trigger the loading function to load data. * @param dbName {String} The database name; * @param loadFunc {Function} The load() invoker defined for an instance of `gpii.dataLoader` component (See dataLoader.js); * @return {None} The success or fail result of the loading is reported on the console. */ -gpii.dataLoader.performLoad = function (dbName, loadFunc) { - var promise = loadFunc(); +gpii.dataLoader.performLoad = function (dbName, loaderComponent) { + var promise = loaderComponent.load(); promise.then(function () { console.log("The " + dbName + " data has been loaded successfully."); + loaderComponent.events.onDataLoaded.fire(); }, function (err) { console.log("Error at loading the " + dbName + " data. Error details: ", err); + loaderComponent.events.onDataLoadedError.fire(err); }); }; diff --git a/tests/scripts/AuthDataLoaderTests.js b/tests/scripts/AuthDataLoaderTests.js new file mode 100644 index 000000000..586800884 --- /dev/null +++ b/tests/scripts/AuthDataLoaderTests.js @@ -0,0 +1,123 @@ +/*! + * GPII Auth Data Loader Tests + * + * Copyright 2017 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = fluid || require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +require("gpii-pouchdb"); +gpii.pouch.loadTestingSupport(); + +require("../../scripts/shared/dataLoader-auth.js"); +require("./DataLoaderTestsUtils.js"); + +fluid.defaults("gpii.tests.authDataLoader", { + gradeNames: ["gpii.dataLoader.authDataLoader"], + dbName: "auth", + dataFile: [ + "%universal/testData/security/TestOAuth2DataStore.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ], + couchDbUrl: "http://localhost:1234" +}); + +fluid.defaults("gpii.tests.authTestCaseHolder", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + expected: { + total: { + doc_count: 17 + } + }, + rawModules: [{ + name: "Testing Auth Data Loader", + tests: [{ + name: "Testing the load results", + type: "test", + sequence: [{ + func: "{totalRequest}.send" + }, { + listener: "gpii.tests.authDataLoader.checkResponse", + event: "{totalRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{totalRequest}.nativeResponse", "{arguments}.0", 200, "{testCaseHolder}.options.expected.total"] + }] + }] + }], + components: { + totalRequest: { + type: "kettle.test.request.http", + options: { + path: "/auth", + port: 1234, + method: "GET" + } + } + } +}); + +gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expectedStatus, expected, bodyPath) { + expectedStatus = expectedStatus ? expectedStatus : 200; + + var bodyData = JSON.parse(body); + + if (bodyPath) { + bodyData = fluid.get(bodyData, bodyPath); + } + gpii.test.express.helpers.isSaneResponse(response, body, expectedStatus); + + if (expected === undefined) { + jqUnit.assertUndefined(msg, bodyData); + } else { + jqUnit.assertLeftHand(msg, expected, bodyData); + } +}; + +fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { + gradeNames: ["gpii.test.pouch.environment"], + port: 1234, + components: { + authDataLoader: { + type: "gpii.tests.authDataLoader", + createOnEvent: "onHarnessReady", + options: { + listeners: { + "onDataLoaded.escalate": "{authDataLoaderTests}.events.onAuthDataLoaded.fire", + "onDataLoaded.debug": { + listener: "console.log", + args: ["authDataLoader onDataLoaded fired"] + } + } + } + } + }, + events: { + onAuthDataLoaded: null, + onFixturesConstructed: { + events: { + onAuthDataLoaded: "onAuthDataLoaded" + } + } + } +}); + +//*********** Combine Test Environment and Test Case Holder ***********// +fluid.defaults("gpii.tests.authDataLoaderTests", { + gradeNames: ["gpii.tests.authDataLoader.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.authTestCaseHolder" + } + } +}); + +fluid.test.runTests("gpii.tests.authDataLoaderTests"); diff --git a/tests/scripts/DataLoaderTestsUtils.js b/tests/scripts/DataLoaderTestsUtils.js new file mode 100644 index 000000000..698715be1 --- /dev/null +++ b/tests/scripts/DataLoaderTestsUtils.js @@ -0,0 +1,28 @@ +/** +GPII Data Loader Tests Utils + +Copyright 2017 OCAD University + +Licensed under the New BSD license. You may not use this file except in +compliance with this License. + +You may obtain a copy of the License at +https://github.com/GPII/universal/blob/master/LICENSE.txt +*/ + +"use strict"; + +var fluid = require("infusion"); + +//require("./js/DataStoreTestsUtils.js"); + +fluid.defaults("gpii.tests.dataLoader.baseTestCaseHolder", { + gradeNames: ["gpii.test.express.caseHolder"], + sequenceEnd: [{ + func: "{gpii.test.pouch.environment}.events.onCleanup.fire" + }, { + event: "{gpii.test.pouch.environment}.events.onCleanupComplete", + listener: "fluid.log", + args: ["Database cleanup complete"] + }] +}); diff --git a/tests/scripts/PrefsDataLoaderTests.js b/tests/scripts/PrefsDataLoaderTests.js index af417b926..ba4d0a993 100644 --- a/tests/scripts/PrefsDataLoaderTests.js +++ b/tests/scripts/PrefsDataLoaderTests.js @@ -1,5 +1,5 @@ /*! - * GPII Settings Transformer Tests + * GPII Preferences Data Loader Tests * * Copyright 2016 OCAD University * From eef277d615e80db0fa63c76be905c363927e6016 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 2 Feb 2017 11:55:38 -0500 Subject: [PATCH 14/35] GPII-1987: Added the validation process before the loading starts. --- scripts/shared/dataLoader.js | 135 +++++++++---------- tests/scripts/DataLoaderTests.js | 200 +++++++++++++++++++++++++++++ tests/scripts/data/views-test.json | 10 ++ 3 files changed, 278 insertions(+), 67 deletions(-) create mode 100644 tests/scripts/DataLoaderTests.js create mode 100644 tests/scripts/data/views-test.json diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index a8f9bfb72..4fef83955 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -112,34 +112,83 @@ gpii.dataLoader.load = function (that) { var promiseTogo = fluid.promise(); var loadDataPromises = []; - // Process databases one by one - fluid.each(that.options.databases, function (dbData, dbName) { - var directModel = { - couchDbUrl: that.options.couchDbUrl, - dbName: dbName - }; + // Validate and extract data from data files + var dataFileResults = gpii.dataLoader.processDataFiles(that.options.databases); - // Delete the database if it already exists - var prepareDbPromise = gpii.dataLoader.prepareDB(that.getDbDataSource, that.createDbDataSource, that.deleteDbDataSource, directModel); + if (dataFileResults.errors.length > 0) { + promiseTogo.reject("Data file(s) not found in the file system: " + errors.join()); + } else { + var databases = dataFileResults.databases; + // Process databases one by one + fluid.each(databases, function (dbData, dbName) { + var directModel = { + couchDbUrl: that.options.couchDbUrl, + dbName: dbName + }; - prepareDbPromise.then(function () { - // load data files - var dataFilePaths = fluid.makeArray(dbData.dataFile); - loadDataPromises = gpii.dataLoader.loadDataFiles(that.loadDataSource, dataFilePaths, directModel); + // Delete the database if it already exists + var prepareDbPromise = gpii.dataLoader.prepareDB(that.getDbDataSource, that.createDbDataSource, that.deleteDbDataSource, directModel); - // load direct data - if (dbData.data) { - loadDataPromises.push(gpii.dataLoader.loadData(that.loadDataSource, dbData.data, directModel)); - } + prepareDbPromise.then(function () { + // load data + if (dbData) { + loadDataPromises.push(gpii.dataLoader.loadData(that.loadDataSource, dbData, directModel)); + } - var loadDataPromise = fluid.promise.sequence(loadDataPromises); - fluid.promise.follow(loadDataPromise, promiseTogo); + var loadDataPromise = fluid.promise.sequence(loadDataPromises); + fluid.promise.follow(loadDataPromise, promiseTogo); + }); }); - }); + } return promiseTogo; }; +/** + * Loop thru each database, to ensure the existence of data files in database.dataFile option. If files exist, + * extract data from data files, concat with data from database.data, and return the concatenated data for each database. + * Otherwise, report errors on non-existent files. + * + * @param databases {Object} The same structure as options.databases of the "gpii.dataLoader" component; + * + * @return {Object} In a structure of {errors: [{String}, ...], databases: [{dbName1: [{data}]}, ...]}. + */ +gpii.dataLoader.processDataFiles = function (databases) { + var errors = []; + // Structure: {database1: {data: []}, ...} + var allData = {}; + + fluid.each(databases, function (database, dbName) { + var data = []; + if (database.dataFile) { + // Loop thru data files to check their existence. + // If doesn't exist, report error. Otherwise, extract the data + fluid.each(database.dataFile, function (oneDataFile) { + var actualPath = fluid.module.resolvePath(oneDataFile); + if (!fs.existsSync(actualPath)) { + errors.push(actualPath); + } else { + data = data.concat(fluid.makeArray(require(actualPath))); + } + }); + } + + // Concat the extracted data from data files with the provided input at database.data + if (database.data) { + data = data.concat(fluid.makeArray(database.data)); + } + + if (data.length > 0) { + fluid.set(allData, [dbName], data); + } + }); + + return { + errors: errors, + databases: allData + }; +}; + /** * To create a database. * @@ -204,54 +253,6 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { return loadDataSource.set(directModel, data); }; -/** - * Load a data file into a database. - * - * @param loadDataSource {DataSource} The data source for loading the data; - * @param dataFile {String} The path to a data file; - * @param directModel {Object} The direct model expressing the "coordinates" of the model to be fetched - * - * @return {Promise} A promise with the response of loading the data file. - */ -gpii.dataLoader.loadDataFiles = function (loadDataSource, dataFilePaths, directModel) { - // Capture non-found file paths - var unfoundFiles = []; - // Capture data from files if files exist. - var dataContents = []; - var loadDataPromises = []; - - fluid.each(dataFilePaths, function (oneDataFile) { - var actualPath = fluid.module.resolvePath(oneDataFile); - if (!fs.existsSync(actualPath)) { - unfoundFiles.push(actualPath); - } else { - dataContents.push(require(actualPath)); - } - }); - - if (unfoundFiles.length > 0) { - var errorPromise = fluid.promise().reject("Data file(s) not found in the file system: " + unfoundFiles.join()); - loadDataPromises.push(errorPromise); - } else { - fluid.each(dataContents, function (data) { - loadDataPromises.push(gpii.dataLoader.loadData(loadDataSource, data, directModel)); - }); - } - return loadDataPromises; -}; - -gpii.dataLoader.loadDataFile = function (loadDataSource, dataFile, directModel) { - var actualPath = fluid.module.resolvePath(dataFile); - if (!fs.existsSync(actualPath)) { - return fluid.promise().reject("Data file is not found in the file system: " + actualPath); - } else { - var data = require(actualPath); - var promiseTogo = gpii.dataLoader.loadData(loadDataSource, data, directModel); - - return promiseTogo; - } -}; - /** * The API function to trigger the loading function to load data. * @param dbName {String} The database name; diff --git a/tests/scripts/DataLoaderTests.js b/tests/scripts/DataLoaderTests.js new file mode 100644 index 000000000..034d84717 --- /dev/null +++ b/tests/scripts/DataLoaderTests.js @@ -0,0 +1,200 @@ +/*! + * GPII Preferences Data Loader Tests + * + * Copyright 2016 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = fluid || require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +require("../../scripts/shared/dataLoader.js"); + +fluid.logObjectRenderChars = 10240; + +(function () { + fluid.registerNamespace("gpii.tests.dataLoader"); + + fluid.defaults("gpii.tests.dataLoader.dataConverter", { + gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], + dataPath: "data/", + expected: [{ + carla: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + } + }, { + sammy: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + } + }] + }); + + jqUnit.test("Test gpii.dataLoader.processDataFiles", function () { + var testData = { + case1: { + db1: { + dataFile: [ + "%universal/tests/scripts/data/carla.json", + "%universal/tests/scripts/data/views-test.json" + ], + data: { + "test1": "test1" + } + }, + db2: { + data: { + "test2": "test2" + } + } + }, + case2: { + db1: { + dataFile: [ + "%universal/tests/scripts/data/nonExistent1.json", + "%universal/tests/scripts/data/sammy.json" + ], + data: { + "test1": "test1" + } + }, + db2: { + dataFile: [ + "%universal/tests/scripts/data/nonExistent2.json" + ], + data: { + "test2": "test2" + } + } + } + }; + + var expected = { + case1: { + errors: [], + databases: { + db1: [{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + }, { + "_id": "_design/views", + "views": { + "findUserByName": { + "map": "function(doc) {if (doc.type === 'user') emit(doc.name, doc)}" + } + } + }, { + "test1": "test1" + }], + db2: [{ + "test2": "test2" + }] + } + }, + case2: { + // The errors array contains the regex patterns to match actual returned errors + // since the actual errors are full paths of files that vary when this test runs + // on different machines. + errors: [ + ".*tests\/scripts\/data\/nonExistent1.json", + ".*tests\/scripts\/data\/nonExistent2.json" + ], + databases: { + db1: [{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + }, { + "test1": "test1" + }], + db2: [{ + "test2": "test2" + }] + } + } + }; + + var matchedErrors = []; + + fluid.each(testData, function (data, caseName) { + var result = gpii.dataLoader.processDataFiles(data); + + // Verify result.data + jqUnit.assertDeepEq("The returned result is expected", expected[caseName].databases, result.databases); + + // Verify result.errors + fluid.each(result.errors, function (error, i) { + var matchResult = fluid.find_if(expected[caseName].errors, function (errorPattern) { + var matchResult = error.match(errorPattern); + return matchResult && matchResult.length > 0; + }); + if (matchResult) { + matchedErrors.push(error); + } + }); + jqUnit.assertDeepEq("All errors are reported", result.errors, matchedErrors); + }); + }); +})(); diff --git a/tests/scripts/data/views-test.json b/tests/scripts/data/views-test.json new file mode 100644 index 000000000..e314ed60f --- /dev/null +++ b/tests/scripts/data/views-test.json @@ -0,0 +1,10 @@ +[ + { + "_id": "_design/views", + "views": { + "findUserByName": { + "map": "function(doc) {if (doc.type === 'user') emit(doc.name, doc)}" + } + } + } +] From 1bd667d5406eb12d09ea24e5c092521c02d9fa30 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 2 Feb 2017 13:49:13 -0500 Subject: [PATCH 15/35] GPII-1987: File renaming and tests fixing. --- scripts/{authDataLoader.js => dataLoader-auth.js} | 2 +- scripts/{prefsDataLoader.js => dataLoader-prefs.js} | 2 +- .../shared/{dataLoader-auth.js => authDataLoader.js} | 0 .../{dataLoader-prefs.js => prefsDataLoader.js} | 0 tests/scripts/AuthDataLoaderTests.js | 5 ++++- tests/scripts/DataLoaderTests.js | 6 +++--- tests/scripts/PrefsDataLoaderTests.js | 12 +++++++----- tests/scripts/data/{ => prefs}/carla.json | 0 tests/scripts/data/{ => prefs}/franklin.md | 0 tests/scripts/data/{ => prefs}/sammy.json | 0 10 files changed, 16 insertions(+), 11 deletions(-) rename scripts/{authDataLoader.js => dataLoader-auth.js} (96%) rename scripts/{prefsDataLoader.js => dataLoader-prefs.js} (95%) rename scripts/shared/{dataLoader-auth.js => authDataLoader.js} (100%) rename scripts/shared/{dataLoader-prefs.js => prefsDataLoader.js} (100%) rename tests/scripts/data/{ => prefs}/carla.json (100%) rename tests/scripts/data/{ => prefs}/franklin.md (100%) rename tests/scripts/data/{ => prefs}/sammy.json (100%) diff --git a/scripts/authDataLoader.js b/scripts/dataLoader-auth.js similarity index 96% rename from scripts/authDataLoader.js rename to scripts/dataLoader-auth.js index c30bf82b9..150f29326 100644 --- a/scripts/authDataLoader.js +++ b/scripts/dataLoader-auth.js @@ -24,7 +24,7 @@ var fluid = require("infusion"), fluid.setLogging(true); -require("./shared/dataLoader-auth.js"); +require("./shared/authDataLoader.js"); gpii.dataLoader.authDataLoader({ dbName: "auth", diff --git a/scripts/prefsDataLoader.js b/scripts/dataLoader-prefs.js similarity index 95% rename from scripts/prefsDataLoader.js rename to scripts/dataLoader-prefs.js index faf751f59..847b66464 100644 --- a/scripts/prefsDataLoader.js +++ b/scripts/dataLoader-prefs.js @@ -24,7 +24,7 @@ var fluid = require("infusion"), fluid.setLogging(true); -require("./shared/dataLoader-prefs.js"); +require("./shared/prefsDataLoader.js"); gpii.dataLoader.prefsDataLoader({ dbName: "preferences", diff --git a/scripts/shared/dataLoader-auth.js b/scripts/shared/authDataLoader.js similarity index 100% rename from scripts/shared/dataLoader-auth.js rename to scripts/shared/authDataLoader.js diff --git a/scripts/shared/dataLoader-prefs.js b/scripts/shared/prefsDataLoader.js similarity index 100% rename from scripts/shared/dataLoader-prefs.js rename to scripts/shared/prefsDataLoader.js diff --git a/tests/scripts/AuthDataLoaderTests.js b/tests/scripts/AuthDataLoaderTests.js index 586800884..8630aa8c0 100644 --- a/tests/scripts/AuthDataLoaderTests.js +++ b/tests/scripts/AuthDataLoaderTests.js @@ -19,7 +19,7 @@ var fluid = fluid || require("infusion"), require("gpii-pouchdb"); gpii.pouch.loadTestingSupport(); -require("../../scripts/shared/dataLoader-auth.js"); +require("../../scripts/shared/authDataLoader.js"); require("./DataLoaderTestsUtils.js"); fluid.defaults("gpii.tests.authDataLoader", { @@ -82,6 +82,9 @@ gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expecte } }; +// This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" +// looks up this compnent name for port and events. +// See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { gradeNames: ["gpii.test.pouch.environment"], port: 1234, diff --git a/tests/scripts/DataLoaderTests.js b/tests/scripts/DataLoaderTests.js index 034d84717..60d28631a 100644 --- a/tests/scripts/DataLoaderTests.js +++ b/tests/scripts/DataLoaderTests.js @@ -1,7 +1,7 @@ /*! * GPII Preferences Data Loader Tests * - * Copyright 2016 OCAD University + * Copyright 2016-2017 OCAD University * * Licensed under the New BSD license. You may not use this file except in * compliance with this License. @@ -71,7 +71,7 @@ fluid.logObjectRenderChars = 10240; case1: { db1: { dataFile: [ - "%universal/tests/scripts/data/carla.json", + "%universal/tests/scripts/data/prefs/carla.json", "%universal/tests/scripts/data/views-test.json" ], data: { @@ -88,7 +88,7 @@ fluid.logObjectRenderChars = 10240; db1: { dataFile: [ "%universal/tests/scripts/data/nonExistent1.json", - "%universal/tests/scripts/data/sammy.json" + "%universal/tests/scripts/data/prefs/sammy.json" ], data: { "test1": "test1" diff --git a/tests/scripts/PrefsDataLoaderTests.js b/tests/scripts/PrefsDataLoaderTests.js index ba4d0a993..fc451fde5 100644 --- a/tests/scripts/PrefsDataLoaderTests.js +++ b/tests/scripts/PrefsDataLoaderTests.js @@ -1,7 +1,7 @@ /*! * GPII Preferences Data Loader Tests * - * Copyright 2016 OCAD University + * Copyright 2016-2017 OCAD University * * Licensed under the New BSD license. You may not use this file except in * compliance with this License. @@ -16,16 +16,17 @@ var fluid = fluid || require("infusion"), jqUnit = fluid.require("node-jqunit", require, "jqUnit"), gpii = fluid.registerNamespace("gpii"); -require("../../scripts/prefsDataLoader.js"); +require("../../scripts/shared/prefsDataLoader.js"); (function () { fluid.registerNamespace("gpii.tests.prefsDataLoader"); fluid.defaults("gpii.tests.prefsDataLoader.dataConverter", { gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], - dataPath: "data/", + dataPath: "data/prefs/", expected: [{ - carla: { + "_id": "carla", + "value": { "flat": { "contexts": { "gpii-default": { @@ -42,7 +43,8 @@ require("../../scripts/prefsDataLoader.js"); } } }, { - sammy: { + "_id": "sammy", + "value": { "flat": { "contexts": { "gpii-default": { diff --git a/tests/scripts/data/carla.json b/tests/scripts/data/prefs/carla.json similarity index 100% rename from tests/scripts/data/carla.json rename to tests/scripts/data/prefs/carla.json diff --git a/tests/scripts/data/franklin.md b/tests/scripts/data/prefs/franklin.md similarity index 100% rename from tests/scripts/data/franklin.md rename to tests/scripts/data/prefs/franklin.md diff --git a/tests/scripts/data/sammy.json b/tests/scripts/data/prefs/sammy.json similarity index 100% rename from tests/scripts/data/sammy.json rename to tests/scripts/data/prefs/sammy.json From cf907b9d892e39283ae9d9c76fc9d22a3cef087b Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 3 Feb 2017 16:36:11 -0500 Subject: [PATCH 16/35] GPII-1987: Adjusted auth data loader tests. --- tests/scripts/AuthDataLoaderTests.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/scripts/AuthDataLoaderTests.js b/tests/scripts/AuthDataLoaderTests.js index 8630aa8c0..8f331a4c8 100644 --- a/tests/scripts/AuthDataLoaderTests.js +++ b/tests/scripts/AuthDataLoaderTests.js @@ -32,7 +32,7 @@ fluid.defaults("gpii.tests.authDataLoader", { couchDbUrl: "http://localhost:1234" }); -fluid.defaults("gpii.tests.authTestCaseHolder", { +fluid.defaults("gpii.tests.authTestCaseHolder.success", { gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], expected: { total: { @@ -85,7 +85,7 @@ gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expecte // This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" // looks up this compnent name for port and events. // See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 -fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { +fluid.defaults("gpii.tests.authDataLoader.success.testEnvironment", { gradeNames: ["gpii.test.pouch.environment"], port: 1234, components: { @@ -94,7 +94,7 @@ fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { createOnEvent: "onHarnessReady", options: { listeners: { - "onDataLoaded.escalate": "{authDataLoaderTests}.events.onAuthDataLoaded.fire", + "onDataLoaded.escalate": "{testEnvironment}.events.onAuthDataLoaded.fire", "onDataLoaded.debug": { listener: "console.log", args: ["authDataLoader onDataLoaded fired"] @@ -114,13 +114,13 @@ fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { }); //*********** Combine Test Environment and Test Case Holder ***********// -fluid.defaults("gpii.tests.authDataLoaderTests", { - gradeNames: ["gpii.tests.authDataLoader.testEnvironment"], +fluid.defaults("gpii.tests.authDataLoaderTests.success", { + gradeNames: ["gpii.tests.authDataLoader.success.testEnvironment"], components: { testCaseHolder: { - type: "gpii.tests.authTestCaseHolder" + type: "gpii.tests.authTestCaseHolder.success" } } }); -fluid.test.runTests("gpii.tests.authDataLoaderTests"); +fluid.test.runTests("gpii.tests.authDataLoaderTests.success"); From 47764a88c02487899563c5a8a36ab98e86ceff2b Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 6 Feb 2017 16:27:58 -0500 Subject: [PATCH 17/35] GPII-1987: Refactored auth data loader test to reuse the code for testing the error case. --- tests/scripts/AuthDataLoaderTests.js | 101 ++++++++++++++++----------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/tests/scripts/AuthDataLoaderTests.js b/tests/scripts/AuthDataLoaderTests.js index 8f331a4c8..64903e479 100644 --- a/tests/scripts/AuthDataLoaderTests.js +++ b/tests/scripts/AuthDataLoaderTests.js @@ -22,7 +22,43 @@ gpii.pouch.loadTestingSupport(); require("../../scripts/shared/authDataLoader.js"); require("./DataLoaderTestsUtils.js"); -fluid.defaults("gpii.tests.authDataLoader", { +// The base testEnvironment grade used by all successful and failed test cases. +// This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" +// looks up this compnent name for port and events. +// See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 +fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { + gradeNames: ["gpii.test.pouch.environment"], + port: 1234, + // Must be supplied with a "gpii.dataLoader.authDataLoader" grade to ensure what's defined in "authDataLoader" sub-component makes sense + authDataLoaderGrade: null, + distributeOptions: { + source: "{that}.options.authDataLoaderGrade", + target: "{that > authDataLoader}.options.gradeNames" + }, + components: { + authDataLoader: { + type: "fluid.component", + createOnEvent: "onHarnessReady", + options: { + listeners: { + "onDataLoaded.escalate": "{testEnvironment}.events.onAuthDataLoaded.fire", + "onDataLoadedError.escalate": "{testEnvironment}.events.onAuthDataLoadedError.fire", + "onCreate.debug": { + listener: "console.log", + args: ["============= ", "{that}.options.gradeNames"] + } + } + } + } + }, + events: { + onAuthDataLoaded: null, + onAuthDataLoadedError: null + } +}); + +//*********** The successful data loading case ***********// +fluid.defaults("gpii.tests.authDataLoader.success", { gradeNames: ["gpii.dataLoader.authDataLoader"], dbName: "auth", dataFile: [ @@ -32,6 +68,19 @@ fluid.defaults("gpii.tests.authDataLoader", { couchDbUrl: "http://localhost:1234" }); +// This component must be name as "testEnvironment" because of the reason described at line 25 +fluid.defaults("gpii.tests.authDataLoader.success.testEnvironment", { + gradeNames: ["gpii.tests.authDataLoader.testEnvironment"], + authDataLoaderGrade: "gpii.tests.authDataLoader.success", + events: { + onFixturesConstructed: { + events: { + onAuthDataLoaded: "onAuthDataLoaded" + } + } + } +}); + fluid.defaults("gpii.tests.authTestCaseHolder.success", { gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], expected: { @@ -65,6 +114,15 @@ fluid.defaults("gpii.tests.authTestCaseHolder.success", { } }); +fluid.defaults("gpii.tests.authDataLoaderTests.success", { + gradeNames: ["gpii.tests.authDataLoader.success.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.authTestCaseHolder.success" + } + } +}); + gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expectedStatus, expected, bodyPath) { expectedStatus = expectedStatus ? expectedStatus : 200; @@ -82,45 +140,4 @@ gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expecte } }; -// This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" -// looks up this compnent name for port and events. -// See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 -fluid.defaults("gpii.tests.authDataLoader.success.testEnvironment", { - gradeNames: ["gpii.test.pouch.environment"], - port: 1234, - components: { - authDataLoader: { - type: "gpii.tests.authDataLoader", - createOnEvent: "onHarnessReady", - options: { - listeners: { - "onDataLoaded.escalate": "{testEnvironment}.events.onAuthDataLoaded.fire", - "onDataLoaded.debug": { - listener: "console.log", - args: ["authDataLoader onDataLoaded fired"] - } - } - } - } - }, - events: { - onAuthDataLoaded: null, - onFixturesConstructed: { - events: { - onAuthDataLoaded: "onAuthDataLoaded" - } - } - } -}); - -//*********** Combine Test Environment and Test Case Holder ***********// -fluid.defaults("gpii.tests.authDataLoaderTests.success", { - gradeNames: ["gpii.tests.authDataLoader.success.testEnvironment"], - components: { - testCaseHolder: { - type: "gpii.tests.authTestCaseHolder.success" - } - } -}); - fluid.test.runTests("gpii.tests.authDataLoaderTests.success"); From 908f9ff7975b807b37c88d28d262ba17bb04ddfe Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 7 Feb 2017 15:57:56 -0500 Subject: [PATCH 18/35] GPII-1987: Added error test case for auth data loader. --- scripts/shared/dataLoader.js | 2 +- tests/scripts/AuthDataLoaderTests.js | 87 +++++++++++++++++++++------ tests/scripts/DataLoaderTests.js | 15 +---- tests/scripts/DataLoaderTestsUtils.js | 27 ++++++++- 4 files changed, 98 insertions(+), 33 deletions(-) diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 4fef83955..45e7b920d 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -116,7 +116,7 @@ gpii.dataLoader.load = function (that) { var dataFileResults = gpii.dataLoader.processDataFiles(that.options.databases); if (dataFileResults.errors.length > 0) { - promiseTogo.reject("Data file(s) not found in the file system: " + errors.join()); + promiseTogo.reject("Data file(s) not found in the file system: " + dataFileResults.errors.join()); } else { var databases = dataFileResults.databases; // Process databases one by one diff --git a/tests/scripts/AuthDataLoaderTests.js b/tests/scripts/AuthDataLoaderTests.js index 64903e479..247d73806 100644 --- a/tests/scripts/AuthDataLoaderTests.js +++ b/tests/scripts/AuthDataLoaderTests.js @@ -42,11 +42,7 @@ fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { options: { listeners: { "onDataLoaded.escalate": "{testEnvironment}.events.onAuthDataLoaded.fire", - "onDataLoadedError.escalate": "{testEnvironment}.events.onAuthDataLoadedError.fire", - "onCreate.debug": { - listener: "console.log", - args: ["============= ", "{that}.options.gradeNames"] - } + "onDataLoadedError.escalate": "{testEnvironment}.events.onAuthDataLoadedError.fire" } } } @@ -57,7 +53,7 @@ fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { } }); -//*********** The successful data loading case ***********// +//*********** The successful data loading test case ***********// fluid.defaults("gpii.tests.authDataLoader.success", { gradeNames: ["gpii.dataLoader.authDataLoader"], dbName: "auth", @@ -123,21 +119,78 @@ fluid.defaults("gpii.tests.authDataLoaderTests.success", { } }); -gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expectedStatus, expected, bodyPath) { - expectedStatus = expectedStatus ? expectedStatus : 200; +//*********** The error data loading test case ***********// +fluid.defaults("gpii.tests.authDataLoader.error", { + gradeNames: ["gpii.dataLoader.authDataLoader"], + dbName: "auth", + dataFile: [ + "%universal/nonExistent.json", + "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" + ], + couchDbUrl: "http://localhost:1234" +}); - var bodyData = JSON.parse(body); +// This component must be name as "testEnvironment" because of the reason described at line 25 +fluid.defaults("gpii.tests.authDataLoader.error.testEnvironment", { + gradeNames: ["gpii.tests.authDataLoader.testEnvironment"], + authDataLoaderGrade: "gpii.tests.authDataLoader.error" +}); - if (bodyPath) { - bodyData = fluid.get(bodyData, bodyPath); +fluid.defaults("gpii.tests.authTestCaseHolder.error", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + rawModules: [{ + name: "Testing Auth Data Loader", + tests: [{ + name: "Testing the load results", + type: "test", + sequence: [{ + event: "{testEnvironment}.events.onAuthDataLoadedError", + listener: "gpii.tests.authTestCaseHolder.error.verifyError", + args: ["{arguments}.0", "Data file\\(s\\) not found in the file system\:.*nonExistent.json"] + }, { + func: "{verifyDbRequest}.send" + }, { + listener: "jqUnit.assertEquals", + event: "{verifyDbRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{verifyDbRequest}.nativeResponse.statusCode", 404] + }] + }] + }], + components: { + verifyDbRequest: { + type: "kettle.test.request.http", + options: { + path: "/auth", + port: 1234, + method: "GET" + } + } } - gpii.test.express.helpers.isSaneResponse(response, body, expectedStatus); +}); - if (expected === undefined) { - jqUnit.assertUndefined(msg, bodyData); - } else { - jqUnit.assertLeftHand(msg, expected, bodyData); +fluid.defaults("gpii.tests.authDataLoaderTests.error", { + gradeNames: ["gpii.tests.authDataLoader.error.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.authTestCaseHolder.error" + } } +}); + +gpii.tests.authTestCaseHolder.error.verifyError = function (msg, expectedMatchedError) { + var isErrorMatch = gpii.tests.dataLoader.verifyMatches(msg, expectedMatchedError); + jqUnit.assertTrue("The expected error is reported", isErrorMatch); +}; + +gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expectedStatus, expected) { + expectedStatus = expectedStatus ? expectedStatus : 200; + var bodyData = JSON.parse(body); + + gpii.test.express.helpers.isSaneResponse(response, body, expectedStatus); + jqUnit.assertLeftHand(msg, expected, bodyData); }; -fluid.test.runTests("gpii.tests.authDataLoaderTests.success"); +fluid.test.runTests([ + "gpii.tests.authDataLoaderTests.success", + "gpii.tests.authDataLoaderTests.error" +]); diff --git a/tests/scripts/DataLoaderTests.js b/tests/scripts/DataLoaderTests.js index 60d28631a..0722d8843 100644 --- a/tests/scripts/DataLoaderTests.js +++ b/tests/scripts/DataLoaderTests.js @@ -17,6 +17,7 @@ var fluid = fluid || require("infusion"), gpii = fluid.registerNamespace("gpii"); require("../../scripts/shared/dataLoader.js"); +require("./DataLoaderTestsUtils.js"); fluid.logObjectRenderChars = 10240; @@ -176,8 +177,6 @@ fluid.logObjectRenderChars = 10240; } }; - var matchedErrors = []; - fluid.each(testData, function (data, caseName) { var result = gpii.dataLoader.processDataFiles(data); @@ -185,16 +184,8 @@ fluid.logObjectRenderChars = 10240; jqUnit.assertDeepEq("The returned result is expected", expected[caseName].databases, result.databases); // Verify result.errors - fluid.each(result.errors, function (error, i) { - var matchResult = fluid.find_if(expected[caseName].errors, function (errorPattern) { - var matchResult = error.match(errorPattern); - return matchResult && matchResult.length > 0; - }); - if (matchResult) { - matchedErrors.push(error); - } - }); - jqUnit.assertDeepEq("All errors are reported", result.errors, matchedErrors); + var isErrorMatch = gpii.tests.dataLoader.verifyMatches(result.errors, expected[caseName].errors); + jqUnit.assertTrue("All expected errors are reported", isErrorMatch); }); }); })(); diff --git a/tests/scripts/DataLoaderTestsUtils.js b/tests/scripts/DataLoaderTestsUtils.js index 698715be1..db669882d 100644 --- a/tests/scripts/DataLoaderTestsUtils.js +++ b/tests/scripts/DataLoaderTestsUtils.js @@ -12,9 +12,8 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt "use strict"; -var fluid = require("infusion"); - -//require("./js/DataStoreTestsUtils.js"); +var fluid = fluid || require("infusion"), + gpii = fluid.registerNamespace("gpii"); fluid.defaults("gpii.tests.dataLoader.baseTestCaseHolder", { gradeNames: ["gpii.test.express.caseHolder"], @@ -26,3 +25,25 @@ fluid.defaults("gpii.tests.dataLoader.baseTestCaseHolder", { args: ["Database cleanup complete"] }] }); + +/** + * The utility function that uses regex to compare whether a string or each string in an given array + * match the regex pattern provided in the expected string or array. + * @param toCompare {String|Array} A string or an array of strings to be compared + * @param expectedMatches {String|Array} A string or an array of regex patterns to be matched + * @return {Boolean} true if all strings in the toCompare match regex patterns in expectedMatches in order, otherwise, return false. + */ +gpii.tests.dataLoader.verifyMatches = function (toCompare, expectedMatches) { + toCompare = fluid.makeArray(toCompare); + expectedMatches = fluid.makeArray(expectedMatches); + + var matched = []; + fluid.each(toCompare, function (one, i) { + var matchResult = one.match(expectedMatches[i]); + if (matchResult && matchResult.length > 0) { + matched.push(one); + } + }); + + return toCompare.length === matched.length; +}; From e03612346b824b942375cccdae2d70363487422f Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 7 Feb 2017 17:37:33 -0500 Subject: [PATCH 19/35] GPII-1987: Created "gpii.dataLoader.reporter" grade to report success and error loading messages to the console. --- scripts/shared/authDataLoader.js | 2 +- scripts/shared/dataLoader.js | 39 +++++++++++++++++++++++++++++-- scripts/shared/prefsDataLoader.js | 2 +- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/scripts/shared/authDataLoader.js b/scripts/shared/authDataLoader.js index 4aaa63fae..7cc94b27b 100644 --- a/scripts/shared/authDataLoader.js +++ b/scripts/shared/authDataLoader.js @@ -19,7 +19,7 @@ var fluid = require("infusion"), require("./dataLoader.js"); fluid.defaults("gpii.dataLoader.authDataLoader", { - gradeNames: ["gpii.dataLoader"], + gradeNames: ["gpii.dataLoader", "gpii.dataLoader.reporter"], databases: { expander: { funcName: "gpii.dataLoader.authDataLoader.constructAuthData", diff --git a/scripts/shared/dataLoader.js b/scripts/shared/dataLoader.js index 45e7b920d..5c7bee5e1 100644 --- a/scripts/shared/dataLoader.js +++ b/scripts/shared/dataLoader.js @@ -262,10 +262,45 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { gpii.dataLoader.performLoad = function (dbName, loaderComponent) { var promise = loaderComponent.load(); promise.then(function () { - console.log("The " + dbName + " data has been loaded successfully."); loaderComponent.events.onDataLoaded.fire(); }, function (err) { - console.log("Error at loading the " + dbName + " data. Error details: ", err); loaderComponent.events.onDataLoadedError.fire(err); }); }; + +// A base grade to report the data loading results to the console. +// This grade should be used as a base grade for auth and prefs data loaders +// where the db name is defined as well as proper events. +fluid.defaults("gpii.dataLoader.reporter", { + gradeNames: ["fluid.component"], + dbName: "dbName", // Must be provided + events: { + onDataLoaded: null, + onDataLoadedError: null + }, + listeners: { + "onDataLoaded.report": { + listener: "console.log", + args: { + expander: { + funcName: "fluid.stringTemplate", + args: ["The %dbName data has been loaded successfully.", { + dbName: "{that}.options.dbName" + }] + } + } + }, + "onDataLoadedError.report": { + listener: "console.log", + args: { + expander: { + funcName: "fluid.stringTemplate", + args: ["Error at loading the %dbName data. Error details: %err", { + dbName: "{that}.options.dbName", + err: "{arguments}.0" + }] + } + } + } + } +}); diff --git a/scripts/shared/prefsDataLoader.js b/scripts/shared/prefsDataLoader.js index 05bae4beb..6e09391c8 100644 --- a/scripts/shared/prefsDataLoader.js +++ b/scripts/shared/prefsDataLoader.js @@ -34,7 +34,7 @@ require("./dataLoader.js"); * "gpii.dataLoader" as the underlying grade for loading. */ fluid.defaults("gpii.dataLoader.prefsDataLoader", { - gradeNames: ["fluid.component"], + gradeNames: ["gpii.dataLoader.reporter"], components: { dataConverter: { type: "gpii.dataLoader.prefsDataLoader.dataConverter", From 89c14cf5f79e1cdbbb1fdd565970898c2f48169a Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 9 Feb 2017 16:11:13 -0500 Subject: [PATCH 20/35] GPII-1987: Added tests for preferences data loader. --- scripts/shared/prefsDataLoader.js | 36 ++++ tests/all-tests.js | 3 + tests/scripts/AuthDataLoaderTests.js | 75 ++------- tests/scripts/DataLoaderTestsUtils.js | 61 +++++++ tests/scripts/PrefsDataLoaderTests.js | 230 ++++++++++++++++++++------ 5 files changed, 294 insertions(+), 111 deletions(-) diff --git a/scripts/shared/prefsDataLoader.js b/scripts/shared/prefsDataLoader.js index 6e09391c8..3367be5fe 100644 --- a/scripts/shared/prefsDataLoader.js +++ b/scripts/shared/prefsDataLoader.js @@ -27,6 +27,7 @@ fluid.setLogging(true); require("./dataLoader.js"); /** + * Preferences Data Loader: to load the preferences (json files) from a specific path into CouchDB * This component is composed by two sub-components: * 1. dataConverter: to read thru the preferences data and construct the json file contents * into the data structure that can be accepted by CouchDB /_bulk_docs API @@ -35,9 +36,16 @@ require("./dataLoader.js"); */ fluid.defaults("gpii.dataLoader.prefsDataLoader", { gradeNames: ["gpii.dataLoader.reporter"], + + // Below is the list of options for configing the preferences data loader + dbName: "preferences", + dataPath: "%universal/testData/preferences/", + couchDbUrl: "http://localhost:5984", + components: { dataConverter: { type: "gpii.dataLoader.prefsDataLoader.dataConverter", + createOnEvent: "onConvertData", options: { listeners: { "onPrefsDataStructureConstructed.escalate": { @@ -69,12 +77,22 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { } }, events: { + // fired to create the subcomponent "dataConverter" after ensuring the given options are valid + onConvertData: null, + // fired to create the subcomponent "loader" once the data is converted and ready to load onCreateLoader: null, + // fired to trigger the load process when "loader" is ready onLoaderReady: null, + // fired after the data is successfully loaded onDataLoaded: null, + // fired with an error message if the load fails onDataLoadedError: null }, listeners: { + "onCreate.validateInputs": { + listener: "gpii.dataLoader.prefsDataLoader.validateInputs", + args: ["{that}.options.dataPath", "{that}.events.onConvertData", "{that}.events.onDataLoadedError"] + }, "onLoaderReady.load": { listener: "gpii.dataLoader.performLoad", args: ["{that}.options.dbName", "{that}.loader"] @@ -92,6 +110,21 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader", { } }); +/** + * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). + * + * @param dbName {String} The database name; + * @param data {Object} The data to be loaded into the database. + */ +gpii.dataLoader.prefsDataLoader.validateInputs = function (dataPath, successEvt, errorEvt) { + var actualPath = fluid.module.resolvePath(dataPath); + if (fs.existsSync(actualPath)) { + successEvt.fire(); + } else { + errorEvt.fire("The provided data path does not exist: " + actualPath); + } +}; + /** * Construct the value of `options.databases` that to be accepted by `gpii.dataLoader` (See dataLoader.js). * @@ -132,6 +165,9 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { * @return {Event} fires the data ready event with an argument of the constructed data structure. */ gpii.dataLoader.prefsDataLoader.dataConverter.constructPrefsDataStructure = function (dataPath, onPrefsDataStructureConstructedEvent) { + // Pad the dataPath with an ending "/" + dataPath = dataPath.substr(dataPath.length - 1) === "/" ? dataPath : dataPath + "/"; + var prefsDataStructure = []; var actualDataPath = fluid.module.resolvePath(dataPath); var prefsDataList = fs.readdirSync(actualDataPath); diff --git a/tests/all-tests.js b/tests/all-tests.js index 8728a48ac..7256a2f15 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -50,6 +50,9 @@ var testIncludes = [ "./UntrustedBrowserChannelTests.js", "./UntrustedDevelopmentTests.js", "./UntrustedUserLogonStateChangeTests.js", + "./scripts/AuthDataLoaderTests.js", + "./scripts/DataLoaderTests.js", + "./scripts/PrefsDataLoaderTests.js", "../gpii/node_modules/flowManager/test/SaveTests.js", "../gpii/node_modules/flowManager/test/UpdateTests.js", "../gpii/node_modules/flowManager/test/BrowserChannelTests.js", diff --git a/tests/scripts/AuthDataLoaderTests.js b/tests/scripts/AuthDataLoaderTests.js index 247d73806..c402eae77 100644 --- a/tests/scripts/AuthDataLoaderTests.js +++ b/tests/scripts/AuthDataLoaderTests.js @@ -13,7 +13,6 @@ "use strict"; var fluid = fluid || require("infusion"), - jqUnit = fluid.require("node-jqunit", require, "jqUnit"), gpii = fluid.registerNamespace("gpii"); require("gpii-pouchdb"); @@ -22,38 +21,7 @@ gpii.pouch.loadTestingSupport(); require("../../scripts/shared/authDataLoader.js"); require("./DataLoaderTestsUtils.js"); -// The base testEnvironment grade used by all successful and failed test cases. -// This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" -// looks up this compnent name for port and events. -// See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 -fluid.defaults("gpii.tests.authDataLoader.testEnvironment", { - gradeNames: ["gpii.test.pouch.environment"], - port: 1234, - // Must be supplied with a "gpii.dataLoader.authDataLoader" grade to ensure what's defined in "authDataLoader" sub-component makes sense - authDataLoaderGrade: null, - distributeOptions: { - source: "{that}.options.authDataLoaderGrade", - target: "{that > authDataLoader}.options.gradeNames" - }, - components: { - authDataLoader: { - type: "fluid.component", - createOnEvent: "onHarnessReady", - options: { - listeners: { - "onDataLoaded.escalate": "{testEnvironment}.events.onAuthDataLoaded.fire", - "onDataLoadedError.escalate": "{testEnvironment}.events.onAuthDataLoadedError.fire" - } - } - } - }, - events: { - onAuthDataLoaded: null, - onAuthDataLoadedError: null - } -}); - -//*********** The successful data loading test case ***********// +//*********** 1. The successful data loading test case ***********// fluid.defaults("gpii.tests.authDataLoader.success", { gradeNames: ["gpii.dataLoader.authDataLoader"], dbName: "auth", @@ -66,12 +34,12 @@ fluid.defaults("gpii.tests.authDataLoader.success", { // This component must be name as "testEnvironment" because of the reason described at line 25 fluid.defaults("gpii.tests.authDataLoader.success.testEnvironment", { - gradeNames: ["gpii.tests.authDataLoader.testEnvironment"], - authDataLoaderGrade: "gpii.tests.authDataLoader.success", + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.authDataLoader.success", events: { onFixturesConstructed: { events: { - onAuthDataLoaded: "onAuthDataLoaded" + onDataLoaded: "onDataLoaded" } } } @@ -85,14 +53,14 @@ fluid.defaults("gpii.tests.authTestCaseHolder.success", { } }, rawModules: [{ - name: "Testing Auth Data Loader", + name: "Testing Auth Data Loader - success", tests: [{ - name: "Testing the load results", + name: "Testing the successful loading", type: "test", sequence: [{ func: "{totalRequest}.send" }, { - listener: "gpii.tests.authDataLoader.checkResponse", + listener: "gpii.tests.dataLoader.checkResponse", event: "{totalRequest}.events.onComplete", args: ["The total number of records should be as expected", "{totalRequest}.nativeResponse", "{arguments}.0", 200, "{testCaseHolder}.options.expected.total"] }] @@ -119,12 +87,12 @@ fluid.defaults("gpii.tests.authDataLoaderTests.success", { } }); -//*********** The error data loading test case ***********// +//*********** 2. The error data loading test case ***********// fluid.defaults("gpii.tests.authDataLoader.error", { gradeNames: ["gpii.dataLoader.authDataLoader"], dbName: "auth", dataFile: [ - "%universal/nonExistent.json", + "%universal/tests/nonExistent.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ], couchDbUrl: "http://localhost:1234" @@ -132,20 +100,20 @@ fluid.defaults("gpii.tests.authDataLoader.error", { // This component must be name as "testEnvironment" because of the reason described at line 25 fluid.defaults("gpii.tests.authDataLoader.error.testEnvironment", { - gradeNames: ["gpii.tests.authDataLoader.testEnvironment"], - authDataLoaderGrade: "gpii.tests.authDataLoader.error" + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.authDataLoader.error" }); fluid.defaults("gpii.tests.authTestCaseHolder.error", { gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], rawModules: [{ - name: "Testing Auth Data Loader", + name: "Testing Auth Data Loader - error", tests: [{ - name: "Testing the load results", + name: "Testing the non-existence of data files", type: "test", sequence: [{ - event: "{testEnvironment}.events.onAuthDataLoadedError", - listener: "gpii.tests.authTestCaseHolder.error.verifyError", + event: "{testEnvironment}.events.onDataLoadedError", + listener: "gpii.tests.dataLoader.checkError", args: ["{arguments}.0", "Data file\\(s\\) not found in the file system\:.*nonExistent.json"] }, { func: "{verifyDbRequest}.send" @@ -177,19 +145,6 @@ fluid.defaults("gpii.tests.authDataLoaderTests.error", { } }); -gpii.tests.authTestCaseHolder.error.verifyError = function (msg, expectedMatchedError) { - var isErrorMatch = gpii.tests.dataLoader.verifyMatches(msg, expectedMatchedError); - jqUnit.assertTrue("The expected error is reported", isErrorMatch); -}; - -gpii.tests.authDataLoader.checkResponse = function (msg, response, body, expectedStatus, expected) { - expectedStatus = expectedStatus ? expectedStatus : 200; - var bodyData = JSON.parse(body); - - gpii.test.express.helpers.isSaneResponse(response, body, expectedStatus); - jqUnit.assertLeftHand(msg, expected, bodyData); -}; - fluid.test.runTests([ "gpii.tests.authDataLoaderTests.success", "gpii.tests.authDataLoaderTests.error" diff --git a/tests/scripts/DataLoaderTestsUtils.js b/tests/scripts/DataLoaderTestsUtils.js index db669882d..fa1c177be 100644 --- a/tests/scripts/DataLoaderTestsUtils.js +++ b/tests/scripts/DataLoaderTestsUtils.js @@ -13,8 +13,40 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt "use strict"; var fluid = fluid || require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), gpii = fluid.registerNamespace("gpii"); +// The base testEnvironment grade used by all successful and failed test cases. +// This component must be name as "testEnvironment" because its base grade "gpii.test.pouch.environment" +// looks up this compnent name for port and events. +// See: https://github.com/GPII/gpii-pouchdb/blob/master/src/test/environment.js#L55 +fluid.defaults("gpii.tests.dataLoader.testEnvironment", { + gradeNames: ["gpii.test.pouch.environment"], + port: 1234, + // Must be supplied with a "gpii.dataLoader.dataLoader" grade to ensure what's defined in "dataLoader" sub-component makes sense + dataLoaderGrade: null, + distributeOptions: { + source: "{that}.options.dataLoaderGrade", + target: "{that > dataLoader}.options.gradeNames" + }, + components: { + dataLoader: { + type: "fluid.component", + createOnEvent: "onHarnessReady", + options: { + listeners: { + "onDataLoaded.escalate": "{testEnvironment}.events.onDataLoaded.fire", + "onDataLoadedError.escalate": "{testEnvironment}.events.onDataLoadedError.fire" + } + } + } + }, + events: { + onDataLoaded: null, + onDataLoadedError: null + } +}); + fluid.defaults("gpii.tests.dataLoader.baseTestCaseHolder", { gradeNames: ["gpii.test.express.caseHolder"], sequenceEnd: [{ @@ -47,3 +79,32 @@ gpii.tests.dataLoader.verifyMatches = function (toCompare, expectedMatches) { return toCompare.length === matched.length; }; + +/** + * The kettle testing assert function to verify the response of a kettle request + * @param msg {String} A message to be displayed by jqUnit assertion + * @param response {Object} A http response + * @param body {Object} The response carried by the `onComplete` event of a kettle request + * @param expectedStatus {Number} The expected http status code + * @param expected {Object} The expected response data + * @return None + */ +gpii.tests.dataLoader.checkResponse = function (msg, response, body, expectedStatus, expected) { + expectedStatus = expectedStatus ? expectedStatus : 200; + var bodyData = JSON.parse(body); + + gpii.test.express.helpers.isSaneResponse(response, body, expectedStatus); + jqUnit.assertLeftHand(msg, expected, bodyData); +}; + +/** + * The testing assert function to verify an error message + * @param msg {String} A message to verify + * @param expectedMatchedError {String} A regex match pattern to match the msg against. Using regex + * is to avoid the verification of the real machine path that is different when a test runs on different machines. + * @return None + */ +gpii.tests.dataLoader.checkError = function (msg, expectedMatchedError) { + var isErrorMatch = gpii.tests.dataLoader.verifyMatches(msg, expectedMatchedError); + jqUnit.assertTrue("The expected error is reported", isErrorMatch); +}; diff --git a/tests/scripts/PrefsDataLoaderTests.js b/tests/scripts/PrefsDataLoaderTests.js index fc451fde5..8317d7724 100644 --- a/tests/scripts/PrefsDataLoaderTests.js +++ b/tests/scripts/PrefsDataLoaderTests.js @@ -16,68 +16,196 @@ var fluid = fluid || require("infusion"), jqUnit = fluid.require("node-jqunit", require, "jqUnit"), gpii = fluid.registerNamespace("gpii"); +require("gpii-pouchdb"); +gpii.pouch.loadTestingSupport(); + require("../../scripts/shared/prefsDataLoader.js"); +require("./DataLoaderTestsUtils.js"); + +fluid.registerNamespace("gpii.tests.prefsDataLoader"); -(function () { - fluid.registerNamespace("gpii.tests.prefsDataLoader"); - - fluid.defaults("gpii.tests.prefsDataLoader.dataConverter", { - gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], - dataPath: "data/prefs/", - expected: [{ - "_id": "carla", - "value": { - "flat": { - "contexts": { - "gpii-default": { - "name": "Default preferences", - "preferences": { - "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { - "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, - "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, - "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false - } +//*********** Test gpii.dataLoader.prefsDataLoader.dataConverter ***********// +fluid.defaults("gpii.tests.prefsDataLoader.dataConverter", { + gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], + dataPath: "%universal/tests/scripts/data/prefs/", + expected: [{ + "_id": "carla", + "value": { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false } } } } } - }, { - "_id": "sammy", - "value": { - "flat": { - "contexts": { - "gpii-default": { - "name": "Default preferences", - "preferences": { - "http://registry.gpii.net/common/fontSize": 24, - "http://registry.gpii.net/common/foregroundColor": "white", - "http://registry.gpii.net/common/backgroundColor": "black", - "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], - "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", - "http://registry.gpii.net/common/magnification": 2.0, - "http://registry.gpii.net/common/tracking": ["mouse"], - "http://registry.gpii.net/common/invertColours": true - } + } + }, { + "_id": "sammy", + "value": { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true } } } } - }] - }); + } + }] +}); - jqUnit.asyncTest("Test gpii.dataLoader.prefsDataLoader.dataConverter", function () { - gpii.tests.prefsDataLoader.dataConverter({ - listeners: { - "onPrefsDataStructureConstructed.assert": { - listener: "jqUnit.assertDeepEq", - args: ["The converted data structure is expected", "{that}.options.expected", "{arguments}.0"] - }, - "onPrefsDataStructureConstructed.runTests": { - listener: "jqUnit.start", - priority: "last" - } +jqUnit.asyncTest("Test gpii.dataLoader.prefsDataLoader.dataConverter", function () { + gpii.tests.prefsDataLoader.dataConverter({ + listeners: { + "onPrefsDataStructureConstructed.assert": { + listener: "jqUnit.assertDeepEq", + args: ["The converted data structure is expected", "{that}.options.expected", "{arguments}.0"] + }, + "onPrefsDataStructureConstructed.runTests": { + listener: "jqUnit.start", + priority: "last" } - }); + } }); -})(); +}); + +//*********** Test gpii.dataLoader.prefsDataLoader ***********// + +//*********** 1. The successful data loading test case ***********// +fluid.defaults("gpii.tests.prefsDataLoader.success", { + gradeNames: ["gpii.dataLoader.prefsDataLoader"], + dbName: "preferences", + dataPath: "%universal/tests/scripts/data/prefs", + couchDbUrl: "http://localhost:1234" +}); + +// This component must be name as "testEnvironment" because of the reason described at line 25 +fluid.defaults("gpii.tests.prefsDataLoader.success.testEnvironment", { + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.prefsDataLoader.success", + events: { + onFixturesConstructed: { + events: { + onDataLoaded: "onDataLoaded" + } + } + } +}); + +fluid.defaults("gpii.tests.prefsTestCaseHolder.success", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + expected: { + total: { + doc_count: 2 + } + }, + rawModules: [{ + name: "Testing Prefs Data Loader - success", + tests: [{ + name: "Testing a successful preferences data loading process", + type: "test", + sequence: [{ + func: "{totalRequest}.send" + }, { + listener: "gpii.tests.dataLoader.checkResponse", + event: "{totalRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{totalRequest}.nativeResponse", "{arguments}.0", 200, "{testCaseHolder}.options.expected.total"] + }] + }] + }], + components: { + totalRequest: { + type: "kettle.test.request.http", + options: { + path: "/preferences", + port: 1234, + method: "GET" + } + } + } +}); + +fluid.defaults("gpii.tests.prefsDataLoaderTests.success", { + gradeNames: ["gpii.tests.prefsDataLoader.success.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.prefsTestCaseHolder.success" + } + } +}); + +//*********** 2. The error data loading test case ***********// +fluid.defaults("gpii.tests.prefsDataLoader.error", { + gradeNames: ["gpii.dataLoader.prefsDataLoader"], + dbName: "preferences", + dataPath: "%universal/nonExistent/", + couchDbUrl: "http://localhost:1234" +}); + +// This component must be name as "testEnvironment" because of the reason described at line 25 +fluid.defaults("gpii.tests.prefsDataLoader.error.testEnvironment", { + gradeNames: ["gpii.tests.dataLoader.testEnvironment"], + dataLoaderGrade: "gpii.tests.prefsDataLoader.error" +}); + +fluid.defaults("gpii.tests.authTestCaseHolder.error", { + gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], + rawModules: [{ + name: "Testing Preferences Data Loader - error", + tests: [{ + name: "Testing the non-existence of a data path", + type: "test", + sequence: [{ + event: "{testEnvironment}.events.onDataLoadedError", + listener: "gpii.tests.dataLoader.checkError", + args: ["{arguments}.0", "The provided data path does not exist\:.*\/nonExistent\/"] + }, { + func: "{verifyDbRequest}.send" + }, { + listener: "jqUnit.assertEquals", + event: "{verifyDbRequest}.events.onComplete", + args: ["The total number of records should be as expected", "{verifyDbRequest}.nativeResponse.statusCode", 404] + }] + }] + }], + components: { + verifyDbRequest: { + type: "kettle.test.request.http", + options: { + path: "/auth", + port: 1234, + method: "GET" + } + } + } +}); + +fluid.defaults("gpii.tests.prefsDataLoaderTests.error", { + gradeNames: ["gpii.tests.prefsDataLoader.error.testEnvironment"], + components: { + testCaseHolder: { + type: "gpii.tests.authTestCaseHolder.error" + } + } +}); + +fluid.test.runTests([ + "gpii.tests.prefsDataLoaderTests.success", + "gpii.tests.prefsDataLoaderTests.error" +]); From 86bb6669582b8893244148957dc928687e6f05c3 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 9 Feb 2017 16:31:28 -0500 Subject: [PATCH 21/35] GPII-1987: Corrected a duplicate grade name that causes running all-tests.js to fail. --- tests/scripts/PrefsDataLoaderTests.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scripts/PrefsDataLoaderTests.js b/tests/scripts/PrefsDataLoaderTests.js index 8317d7724..e87ac01ee 100644 --- a/tests/scripts/PrefsDataLoaderTests.js +++ b/tests/scripts/PrefsDataLoaderTests.js @@ -164,7 +164,7 @@ fluid.defaults("gpii.tests.prefsDataLoader.error.testEnvironment", { dataLoaderGrade: "gpii.tests.prefsDataLoader.error" }); -fluid.defaults("gpii.tests.authTestCaseHolder.error", { +fluid.defaults("gpii.tests.prefsTestCaseHolder.error", { gradeNames: ["gpii.tests.dataLoader.baseTestCaseHolder"], rawModules: [{ name: "Testing Preferences Data Loader - error", @@ -200,7 +200,7 @@ fluid.defaults("gpii.tests.prefsDataLoaderTests.error", { gradeNames: ["gpii.tests.prefsDataLoader.error.testEnvironment"], components: { testCaseHolder: { - type: "gpii.tests.authTestCaseHolder.error" + type: "gpii.tests.prefsTestCaseHolder.error" } } }); From c88e17cb438ae4d269d7dfbd9b633836fe83b312 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 10 Feb 2017 15:56:37 -0500 Subject: [PATCH 22/35] GPII-1987: Added README for "scripts" directory. --- scripts/README.md | 83 +++++++++++++++++++++++++++++++++++++ scripts/dataLoader-auth.js | 2 +- scripts/dataLoader-prefs.js | 2 +- 3 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 scripts/README.md diff --git a/scripts/README.md b/scripts/README.md new file mode 100644 index 000000000..3cfa1d574 --- /dev/null +++ b/scripts/README.md @@ -0,0 +1,83 @@ +This directory contains these scripts: + +* **browserifyTestDependency.js**: To browserify test dependencies required for running some web tests. +* **dataLoader-auth.js**: To load authorization test data into CouchDB. +* **dataLoader-prefs.js**: To load preferences test data into CouchDB. + +## Browserify Test Dependencies: + +**Script name:** `browserifyTestDependency.js` + +This script runs as a npm postinstall step to browserify some node modules that are required for running a few web tests. The creation of this script is to work around the issue that some of these node modules are only pulled in as `devDependencies`(see [GPII package.json](../package.json)), which means the browserifying should only occurs when GPII runs in the development mode. This script ensures the present of these node modules before proceeding with the browserifying. It prevents the failure of running `npm install` when setting up GPII in the production mode. + +To start this script, run the following command in the `universal` directory: + +``` +node scripts/browserifyTestDependency.js +``` + +## Data Loaders + +Data loaders are used for setting up GPII production environment by loading authorization and/or preferences test data into CouchDB. See [Data Loader issue ticket](https://issues.gpii.net/browse/GPII-1987) for details. + +### Preferences Data Loader + +**Script name:** `dataLoader-prefs.js` + +Preferences Data Loader reads all json files from the given directory. Each json file is loaded as a individual CouchDB document with its file name as the corresponding document id. + +To start the data loading, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-prefs.js +``` + +#### How to config Preferences Data Loader + +`dataLoader-prefs.js` instantiates the infusion component `gpii.dataLoader.prefsDataLoader` to load preferences data. + +#### Component Options + +| Option | Type | Description | Default | +| ----------------- | ---------- | ----------- | ------- | +| `dbName` | String | Required. The name of the CouchDB database to be created for loading preferences data. If the dabase already exists, delete it and recreate. | preferences | +| `dataPath` | String | Required. The directory where all preference json files reside. | %universal/testData/preferences/ | +| `couchDbUrl` | String | Required. The URL to the CouchDB. | http://localhost:5984 | + +#### Supported Events + +| Event | Description | Parameters | Parameters Description | +| ----- | ----------- | ---------- | ---------------------- | +| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | +| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | + +### Authorization Data Loader + +**Script name:** `dataLoader-auth.js` + +Authorization Data Loader reads given json files from the file system and load them into CouchDB as they are. + +To start the data loading, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-auth.js +``` + +#### How to config Authorization Data Loader + +`dataLoader-auth.js` instantiates the infusion component `gpii.dataLoader.authDataLoader` to load authorization data. + +#### Component Options + +| Option | Type | Description | Default | +| ----------------- | ---------- | ----------- | ------- | +| `dbName` | String | Required. The name of the CouchDB database to be created for loading authorization data. If the dabase already exists, delete it and recreate. | auth | +| `dataFile` | Array | Required. An array of data files to be loaded. | ["%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json"] | +| `couchDbUrl` | String | Required. The URL to the CouchDB. | `http://localhost:5984` | + +#### Supported Events + +| Event | Description | Parameters | Parameters Description | +| ----- | ----------- | ---------- | ---------------------- | +| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | +| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | diff --git a/scripts/dataLoader-auth.js b/scripts/dataLoader-auth.js index 150f29326..542a1e67c 100644 --- a/scripts/dataLoader-auth.js +++ b/scripts/dataLoader-auth.js @@ -32,5 +32,5 @@ gpii.dataLoader.authDataLoader({ "%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ], - couchDbUrl: "http://admin:admin@localhost:5984" + couchDbUrl: "http://localhost:5984" }); diff --git a/scripts/dataLoader-prefs.js b/scripts/dataLoader-prefs.js index 847b66464..8103bb4dc 100644 --- a/scripts/dataLoader-prefs.js +++ b/scripts/dataLoader-prefs.js @@ -29,5 +29,5 @@ require("./shared/prefsDataLoader.js"); gpii.dataLoader.prefsDataLoader({ dbName: "preferences", dataPath: "%universal/testData/preferences/", - couchDbUrl: "http://admin:admin@localhost:5984" + couchDbUrl: "http://localhost:5984" }); From 521f72c2bdc5c1e0f781150cce92c19d2933fb39 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 13 Feb 2017 11:15:52 -0500 Subject: [PATCH 23/35] GPII-1987: In progress of using data loaders to set up GPII production VM. --- vagrant-configs/provisioning/couchdb-vars.yml | 2 -- vagrant-configs/provisioning/deploy.sh | 11 ++++++++--- vagrant-configs/provisioning/preferences-server.yml | 2 -- vagrant-configs/provisioning/requirements.yml | 3 --- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/vagrant-configs/provisioning/couchdb-vars.yml b/vagrant-configs/provisioning/couchdb-vars.yml index 1f678a4fe..2cf380ea1 100644 --- a/vagrant-configs/provisioning/couchdb-vars.yml +++ b/vagrant-configs/provisioning/couchdb-vars.yml @@ -4,8 +4,6 @@ # and 'couchdb_admin_password' couchdb_databases: - - preferences - - auth couchdb_delete_existing_databases: true diff --git a/vagrant-configs/provisioning/deploy.sh b/vagrant-configs/provisioning/deploy.sh index 205db0f1a..d6fde734b 100644 --- a/vagrant-configs/provisioning/deploy.sh +++ b/vagrant-configs/provisioning/deploy.sh @@ -14,7 +14,12 @@ sudo PYTHONUNBUFFERED=1 ansible-playbook /home/vagrant/sync/vagrant-configs/prov # Install Flow Manager sudo PYTHONUNBUFFERED=1 ansible-playbook /home/vagrant/sync/vagrant-configs/provisioning/flow-manager.yml --tags="install,configure,deploy" -# Add additional CouchDB data -kanso upload /home/vagrant/sync/testData/security/TestOAuth2DataStore.json http://localhost:5984/auth -kanso upload /home/vagrant/sync/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json http://localhost:5984/auth +# Load Preferences Test Data into CouchDB +node /home/vagrant/sync/scripts/dataLoader-prefs.js + +# Load Authorization Test Data into CouchDB +node /home/vagrant/sync/scripts/dataLoader-auth.js + +# kanso upload /home/vagrant/sync/testData/security/TestOAuth2DataStore.json http://localhost:5984/auth +# kanso upload /home/vagrant/sync/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json http://localhost:5984/auth diff --git a/vagrant-configs/provisioning/preferences-server.yml b/vagrant-configs/provisioning/preferences-server.yml index 159e157f9..be8861029 100644 --- a/vagrant-configs/provisioning/preferences-server.yml +++ b/vagrant-configs/provisioning/preferences-server.yml @@ -10,5 +10,3 @@ - facts - nodejs - preferences-server - - preferences-server-data-loader - diff --git a/vagrant-configs/provisioning/requirements.yml b/vagrant-configs/provisioning/requirements.yml index 31fce8e40..271ccc7bc 100644 --- a/vagrant-configs/provisioning/requirements.yml +++ b/vagrant-configs/provisioning/requirements.yml @@ -10,8 +10,5 @@ - src: https://github.com/gpii-ops/ansible-preferences-server name: preferences-server -- src: https://github.com/gpii-ops/ansible-preferences-server-data-loader - name: preferences-server-data-loader - - src: https://github.com/gpii-ops/ansible-flow-manager name: flow-manager From 025cba4330cc4425170b53219adbd6daef56fd74 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 15 Feb 2017 15:19:11 -0500 Subject: [PATCH 24/35] GPII-1987: Uses avtar's fix on ansible-preferences-server that allows the GPII production VM to start properly. --- vagrant-configs/provisioning/deploy.sh | 4 ---- vagrant-configs/provisioning/requirements.yml | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/vagrant-configs/provisioning/deploy.sh b/vagrant-configs/provisioning/deploy.sh index d6fde734b..3f9cdfdaa 100644 --- a/vagrant-configs/provisioning/deploy.sh +++ b/vagrant-configs/provisioning/deploy.sh @@ -19,7 +19,3 @@ node /home/vagrant/sync/scripts/dataLoader-prefs.js # Load Authorization Test Data into CouchDB node /home/vagrant/sync/scripts/dataLoader-auth.js - -# kanso upload /home/vagrant/sync/testData/security/TestOAuth2DataStore.json http://localhost:5984/auth -# kanso upload /home/vagrant/sync/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json http://localhost:5984/auth - diff --git a/vagrant-configs/provisioning/requirements.yml b/vagrant-configs/provisioning/requirements.yml index 271ccc7bc..ef7ed0a60 100644 --- a/vagrant-configs/provisioning/requirements.yml +++ b/vagrant-configs/provisioning/requirements.yml @@ -7,7 +7,8 @@ - src: https://github.com/idi-ops/ansible-nodejs name: nodejs -- src: https://github.com/gpii-ops/ansible-preferences-server +- src: https://github.com/avtar/ansible-preferences-server + version: remotes/origin/GPII-1884 name: preferences-server - src: https://github.com/gpii-ops/ansible-flow-manager From 67715b9f516c0135f0e5a811011e9e61d1634d44 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 16 Feb 2017 21:06:35 -0500 Subject: [PATCH 25/35] GPII-1987: Addressed code review comments. --- documentation/DataLoader.md | 65 ++++++ gpii/node_modules/dataLoader/README.md | 10 + gpii/node_modules/dataLoader/index.js | 9 + .../dataLoader/src}/authDataLoader.js | 2 +- .../dataLoader/src}/dataLoader.js | 0 .../dataLoader/src}/prefsDataLoader.js | 3 +- .../dataLoader/test}/AuthDataLoaderTests.js | 4 +- .../dataLoader/test/DataLoaderTests.js | 187 +++++++++++++++++ .../dataLoader/test}/DataLoaderTestsUtils.js | 2 +- .../dataLoader/test}/PrefsDataLoaderTests.js | 8 +- .../dataLoader/test}/data/prefs/carla.json | 0 .../dataLoader/test}/data/prefs/franklin.md | 0 .../dataLoader/test}/data/prefs/sammy.json | 0 .../dataLoader/test}/data/unused/david.json | 0 .../dataLoader/test}/data/views-test.json | 0 scripts/README.md | 50 +---- tests/all-tests.js | 6 +- tests/scripts/DataLoaderTests.js | 191 ------------------ 18 files changed, 287 insertions(+), 250 deletions(-) create mode 100644 documentation/DataLoader.md create mode 100644 gpii/node_modules/dataLoader/README.md create mode 100644 gpii/node_modules/dataLoader/index.js rename {scripts/shared => gpii/node_modules/dataLoader/src}/authDataLoader.js (96%) rename {scripts/shared => gpii/node_modules/dataLoader/src}/dataLoader.js (100%) rename {scripts/shared => gpii/node_modules/dataLoader/src}/prefsDataLoader.js (99%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/AuthDataLoaderTests.js (98%) create mode 100644 gpii/node_modules/dataLoader/test/DataLoaderTests.js rename {tests/scripts => gpii/node_modules/dataLoader/test}/DataLoaderTestsUtils.js (99%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/PrefsDataLoaderTests.js (97%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/data/prefs/carla.json (100%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/data/prefs/franklin.md (100%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/data/prefs/sammy.json (100%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/data/unused/david.json (100%) rename {tests/scripts => gpii/node_modules/dataLoader/test}/data/views-test.json (100%) delete mode 100644 tests/scripts/DataLoaderTests.js diff --git a/documentation/DataLoader.md b/documentation/DataLoader.md new file mode 100644 index 000000000..e59749a5c --- /dev/null +++ b/documentation/DataLoader.md @@ -0,0 +1,65 @@ +## Data Loaders + +Data loaders are used for setting up GPII production environment by loading authorization and/or preferences test data into CouchDB. See [Data Loader issue ticket](https://issues.gpii.net/browse/GPII-1987) for details. + +### Preferences Data Loader + +**Script Location:** `scripts/dataLoader-prefs.js` + +**Infusion Component:** `gpii.dataLoader.prefsDataLoader` + +**Component Source Code:** `gpii/node_modules/dataLoader/src/prefsDataLoader.js` + +Preferences Data Loader reads all JSON files from the given directory. Each JSON file is loaded as a individual CouchDB document with its file name as the corresponding document id. + +To start the data loading of the preferences data, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-prefs.js +``` + +#### Component Options + +| Option | Type | Description | Default | +| ----------------- | ---------- | ----------- | ------- | +| `dbName` | String | Required. The name of the CouchDB database to be created for loading preferences data. If the database already exists, delete it and recreate. | preferences | +| `dataPath` | String | Required. The directory where all preference JSON files reside. | %universal/testData/preferences/ | +| `couchDbUrl` | String | Required. The URL to the CouchDB. | http://localhost:5984 | + +#### Supported Events + +| Event | Description | Parameters | Parameters Description | +| ----- | ----------- | ---------- | ---------------------- | +| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | +| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | + +### Authorization Data Loader + +**Script Location:** `scripts/dataLoader-auth.js` + +**Infusion Component:** `gpii.dataLoader.authDataLoader` + +**Component Source Code:** `gpii/node_modules/dataLoader/src/authDataLoader.js` + +Authorization Data Loader reads given JSON files from the file system and load them into CouchDB as they are. + +To start the data loading of the authorization data, run the following command in the `universal` directory: + +``` +node scripts/dataLoader-auth.js +``` + +#### Component Options + +| Option | Type | Description | Default | +| ----------------- | ---------- | ----------- | ------- | +| `dbName` | String | Required. The name of the CouchDB database to be created for loading authorization data. If the database already exists, delete it and recreate. | auth | +| `dataFile` | Array | Required. An array of data files to be loaded. | ["%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json"] | +| `couchDbUrl` | String | Required. The URL to the CouchDB. | `http://localhost:5984` | + +#### Supported Events + +| Event | Description | Parameters | Parameters Description | +| ----- | ----------- | ---------- | ---------------------- | +| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | +| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | diff --git a/gpii/node_modules/dataLoader/README.md b/gpii/node_modules/dataLoader/README.md new file mode 100644 index 000000000..5a6f13d50 --- /dev/null +++ b/gpii/node_modules/dataLoader/README.md @@ -0,0 +1,10 @@ +Data Loader +=========== + +Data Loaders to load given data or JSON files into CouchDB. + +--- + +### Dependencies + +[infusion](https://github.com/fluid-project/infusion) framework. diff --git a/gpii/node_modules/dataLoader/index.js b/gpii/node_modules/dataLoader/index.js new file mode 100644 index 000000000..fe6057087 --- /dev/null +++ b/gpii/node_modules/dataLoader/index.js @@ -0,0 +1,9 @@ +"use strict"; + +var fluid = require("infusion"); + +fluid.module.register("dataLoader", __dirname, require); + +require("./src/authDataLoader.js"); +require("./src/dataLoader.js"); +require("./src/prefsDataLoader.js"); diff --git a/scripts/shared/authDataLoader.js b/gpii/node_modules/dataLoader/src/authDataLoader.js similarity index 96% rename from scripts/shared/authDataLoader.js rename to gpii/node_modules/dataLoader/src/authDataLoader.js index 7cc94b27b..138abc2e9 100644 --- a/scripts/shared/authDataLoader.js +++ b/gpii/node_modules/dataLoader/src/authDataLoader.js @@ -42,7 +42,7 @@ fluid.defaults("gpii.dataLoader.authDataLoader", { */ gpii.dataLoader.authDataLoader.constructAuthData = function (dbName, dataFile) { var togo = {}; - fluid.set(togo, dbName + ".dataFile", dataFile); + fluid.set(togo, [dbName, "dataFile"], dataFile); return togo; }; diff --git a/scripts/shared/dataLoader.js b/gpii/node_modules/dataLoader/src/dataLoader.js similarity index 100% rename from scripts/shared/dataLoader.js rename to gpii/node_modules/dataLoader/src/dataLoader.js diff --git a/scripts/shared/prefsDataLoader.js b/gpii/node_modules/dataLoader/src/prefsDataLoader.js similarity index 99% rename from scripts/shared/prefsDataLoader.js rename to gpii/node_modules/dataLoader/src/prefsDataLoader.js index 3367be5fe..82887a6a7 100644 --- a/scripts/shared/prefsDataLoader.js +++ b/gpii/node_modules/dataLoader/src/prefsDataLoader.js @@ -19,8 +19,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt var fluid = require("infusion"), gpii = fluid.registerNamespace("gpii"), - fs = require("fs"), - JSON5 = require("json5"); + fs = require("fs"); fluid.setLogging(true); diff --git a/tests/scripts/AuthDataLoaderTests.js b/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js similarity index 98% rename from tests/scripts/AuthDataLoaderTests.js rename to gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js index c402eae77..e16b0c715 100644 --- a/tests/scripts/AuthDataLoaderTests.js +++ b/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js @@ -12,13 +12,13 @@ "use strict"; -var fluid = fluid || require("infusion"), +var fluid = require("infusion"), gpii = fluid.registerNamespace("gpii"); require("gpii-pouchdb"); gpii.pouch.loadTestingSupport(); -require("../../scripts/shared/authDataLoader.js"); +require("dataLoader"); require("./DataLoaderTestsUtils.js"); //*********** 1. The successful data loading test case ***********// diff --git a/gpii/node_modules/dataLoader/test/DataLoaderTests.js b/gpii/node_modules/dataLoader/test/DataLoaderTests.js new file mode 100644 index 000000000..fd39ca0f1 --- /dev/null +++ b/gpii/node_modules/dataLoader/test/DataLoaderTests.js @@ -0,0 +1,187 @@ +/*! + * GPII Preferences Data Loader Tests + * + * Copyright 2016-2017 OCAD University + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"), + gpii = fluid.registerNamespace("gpii"); + +require("dataLoader"); +require("./DataLoaderTestsUtils.js"); + +fluid.registerNamespace("gpii.tests.dataLoader"); + +fluid.defaults("gpii.tests.dataLoader.dataConverter", { + gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], + dataPath: "data/", + expected: [{ + carla: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + } + }, { + sammy: { + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + } + }] +}); + +jqUnit.test("Test gpii.dataLoader.processDataFiles", function () { + var testData = { + case1: { + db1: { + dataFile: [ + "%dataLoader/test/data/prefs/carla.json", + "%dataLoader/test/data/views-test.json" + ], + data: { + "test1": "test1" + } + }, + db2: { + data: { + "test2": "test2" + } + } + }, + case2: { + db1: { + dataFile: [ + "%dataLoader/test/data/nonExistent1.json", + "%dataLoader/test/data/prefs/sammy.json" + ], + data: { + "test1": "test1" + } + }, + db2: { + dataFile: [ + "%dataLoader/test/data/nonExistent2.json" + ], + data: { + "test2": "test2" + } + } + } + }; + + var expected = { + case1: { + errors: [], + databases: { + db1: [{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { + "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, + "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, + "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false + } + } + } + } + } + }, { + "_id": "_design/views", + "views": { + "findUserByName": { + "map": "function(doc) {if (doc.type === 'user') emit(doc.name, doc)}" + } + } + }, { + "test1": "test1" + }], + db2: [{ + "test2": "test2" + }] + } + }, + case2: { + // The errors array contains the regex patterns to match actual returned errors + // since the actual errors are full paths of files that vary when this test runs + // on different machines. + errors: [ + ".*test\/data\/nonExistent1.json", + ".*test\/data\/nonExistent2.json" + ], + databases: { + db1: [{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/common/fontSize": 24, + "http://registry.gpii.net/common/foregroundColor": "white", + "http://registry.gpii.net/common/backgroundColor": "black", + "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], + "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", + "http://registry.gpii.net/common/magnification": 2.0, + "http://registry.gpii.net/common/tracking": ["mouse"], + "http://registry.gpii.net/common/invertColours": true + } + } + } + } + }, { + "test1": "test1" + }], + db2: [{ + "test2": "test2" + }] + } + } + }; + + fluid.each(testData, function (data, caseName) { + var result = gpii.dataLoader.processDataFiles(data); + + // Verify result.data + jqUnit.assertDeepEq("The returned result is expected", expected[caseName].databases, result.databases); + + // Verify result.errors + var isErrorMatch = gpii.tests.dataLoader.verifyMatches(result.errors, expected[caseName].errors); + jqUnit.assertTrue("All expected errors are reported", isErrorMatch); + }); +}); diff --git a/tests/scripts/DataLoaderTestsUtils.js b/gpii/node_modules/dataLoader/test/DataLoaderTestsUtils.js similarity index 99% rename from tests/scripts/DataLoaderTestsUtils.js rename to gpii/node_modules/dataLoader/test/DataLoaderTestsUtils.js index fa1c177be..270dca0ae 100644 --- a/tests/scripts/DataLoaderTestsUtils.js +++ b/gpii/node_modules/dataLoader/test/DataLoaderTestsUtils.js @@ -12,7 +12,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt "use strict"; -var fluid = fluid || require("infusion"), +var fluid = require("infusion"), jqUnit = fluid.require("node-jqunit", require, "jqUnit"), gpii = fluid.registerNamespace("gpii"); diff --git a/tests/scripts/PrefsDataLoaderTests.js b/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js similarity index 97% rename from tests/scripts/PrefsDataLoaderTests.js rename to gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js index e87ac01ee..fea090394 100644 --- a/tests/scripts/PrefsDataLoaderTests.js +++ b/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js @@ -12,14 +12,14 @@ "use strict"; -var fluid = fluid || require("infusion"), +var fluid = require("infusion"), jqUnit = fluid.require("node-jqunit", require, "jqUnit"), gpii = fluid.registerNamespace("gpii"); require("gpii-pouchdb"); gpii.pouch.loadTestingSupport(); -require("../../scripts/shared/prefsDataLoader.js"); +require("dataLoader"); require("./DataLoaderTestsUtils.js"); fluid.registerNamespace("gpii.tests.prefsDataLoader"); @@ -27,7 +27,7 @@ fluid.registerNamespace("gpii.tests.prefsDataLoader"); //*********** Test gpii.dataLoader.prefsDataLoader.dataConverter ***********// fluid.defaults("gpii.tests.prefsDataLoader.dataConverter", { gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], - dataPath: "%universal/tests/scripts/data/prefs/", + dataPath: "%dataLoader/test/data/prefs/", expected: [{ "_id": "carla", "value": { @@ -91,7 +91,7 @@ jqUnit.asyncTest("Test gpii.dataLoader.prefsDataLoader.dataConverter", function fluid.defaults("gpii.tests.prefsDataLoader.success", { gradeNames: ["gpii.dataLoader.prefsDataLoader"], dbName: "preferences", - dataPath: "%universal/tests/scripts/data/prefs", + dataPath: "%dataLoader/test/data/prefs", couchDbUrl: "http://localhost:1234" }); diff --git a/tests/scripts/data/prefs/carla.json b/gpii/node_modules/dataLoader/test/data/prefs/carla.json similarity index 100% rename from tests/scripts/data/prefs/carla.json rename to gpii/node_modules/dataLoader/test/data/prefs/carla.json diff --git a/tests/scripts/data/prefs/franklin.md b/gpii/node_modules/dataLoader/test/data/prefs/franklin.md similarity index 100% rename from tests/scripts/data/prefs/franklin.md rename to gpii/node_modules/dataLoader/test/data/prefs/franklin.md diff --git a/tests/scripts/data/prefs/sammy.json b/gpii/node_modules/dataLoader/test/data/prefs/sammy.json similarity index 100% rename from tests/scripts/data/prefs/sammy.json rename to gpii/node_modules/dataLoader/test/data/prefs/sammy.json diff --git a/tests/scripts/data/unused/david.json b/gpii/node_modules/dataLoader/test/data/unused/david.json similarity index 100% rename from tests/scripts/data/unused/david.json rename to gpii/node_modules/dataLoader/test/data/unused/david.json diff --git a/tests/scripts/data/views-test.json b/gpii/node_modules/dataLoader/test/data/views-test.json similarity index 100% rename from tests/scripts/data/views-test.json rename to gpii/node_modules/dataLoader/test/data/views-test.json diff --git a/scripts/README.md b/scripts/README.md index 3cfa1d574..b1a63db4c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -18,66 +18,24 @@ node scripts/browserifyTestDependency.js ## Data Loaders -Data loaders are used for setting up GPII production environment by loading authorization and/or preferences test data into CouchDB. See [Data Loader issue ticket](https://issues.gpii.net/browse/GPII-1987) for details. +See [Data loader documentation](../documentation/DataLoader.md) to understand how to config data loaders. -### Preferences Data Loader +## Preferences Data Loader **Script name:** `dataLoader-prefs.js` -Preferences Data Loader reads all json files from the given directory. Each json file is loaded as a individual CouchDB document with its file name as the corresponding document id. - -To start the data loading, run the following command in the `universal` directory: +To load the preferences data into CouchDB, run the following command in the `universal` directory: ``` node scripts/dataLoader-prefs.js ``` -#### How to config Preferences Data Loader - -`dataLoader-prefs.js` instantiates the infusion component `gpii.dataLoader.prefsDataLoader` to load preferences data. - -#### Component Options - -| Option | Type | Description | Default | -| ----------------- | ---------- | ----------- | ------- | -| `dbName` | String | Required. The name of the CouchDB database to be created for loading preferences data. If the dabase already exists, delete it and recreate. | preferences | -| `dataPath` | String | Required. The directory where all preference json files reside. | %universal/testData/preferences/ | -| `couchDbUrl` | String | Required. The URL to the CouchDB. | http://localhost:5984 | - -#### Supported Events - -| Event | Description | Parameters | Parameters Description | -| ----- | ----------- | ---------- | ---------------------- | -| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | -| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | - ### Authorization Data Loader **Script name:** `dataLoader-auth.js` -Authorization Data Loader reads given json files from the file system and load them into CouchDB as they are. - -To start the data loading, run the following command in the `universal` directory: +To load the authorization data into CouchDB, run the following command in the `universal` directory: ``` node scripts/dataLoader-auth.js ``` - -#### How to config Authorization Data Loader - -`dataLoader-auth.js` instantiates the infusion component `gpii.dataLoader.authDataLoader` to load authorization data. - -#### Component Options - -| Option | Type | Description | Default | -| ----------------- | ---------- | ----------- | ------- | -| `dbName` | String | Required. The name of the CouchDB database to be created for loading authorization data. If the dabase already exists, delete it and recreate. | auth | -| `dataFile` | Array | Required. An array of data files to be loaded. | ["%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json"] | -| `couchDbUrl` | String | Required. The URL to the CouchDB. | `http://localhost:5984` | - -#### Supported Events - -| Event | Description | Parameters | Parameters Description | -| ----- | ----------- | ---------- | ---------------------- | -| `onDataLoaded` | Fires when all preference data has been loaded into the CouchDB. | None | | -| `onDataLoadedError` | Fires when an error occurs at loading preference data. | errorMsg | The error message. | diff --git a/tests/all-tests.js b/tests/all-tests.js index 41c7f8c1a..fe644b828 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -51,9 +51,9 @@ var testIncludes = [ "./UntrustedBrowserChannelTests.js", "./UntrustedDevelopmentTests.js", "./UntrustedUserLogonStateChangeTests.js", - "./scripts/AuthDataLoaderTests.js", - "./scripts/DataLoaderTests.js", - "./scripts/PrefsDataLoaderTests.js", + "../gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js", + "../gpii/node_modules/dataLoader/test/DataLoaderTests.js", + "../gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js", "../gpii/node_modules/flowManager/test/SaveTests.js", "../gpii/node_modules/flowManager/test/UpdateTests.js", "../gpii/node_modules/flowManager/test/BrowserChannelTests.js", diff --git a/tests/scripts/DataLoaderTests.js b/tests/scripts/DataLoaderTests.js deleted file mode 100644 index 0722d8843..000000000 --- a/tests/scripts/DataLoaderTests.js +++ /dev/null @@ -1,191 +0,0 @@ -/*! - * GPII Preferences Data Loader Tests - * - * Copyright 2016-2017 OCAD University - * - * Licensed under the New BSD license. You may not use this file except in - * compliance with this License. - * - * You may obtain a copy of the License at - * https://github.com/GPII/universal/blob/master/LICENSE.txt - */ - -"use strict"; - -var fluid = fluid || require("infusion"), - jqUnit = fluid.require("node-jqunit", require, "jqUnit"), - gpii = fluid.registerNamespace("gpii"); - -require("../../scripts/shared/dataLoader.js"); -require("./DataLoaderTestsUtils.js"); - -fluid.logObjectRenderChars = 10240; - -(function () { - fluid.registerNamespace("gpii.tests.dataLoader"); - - fluid.defaults("gpii.tests.dataLoader.dataConverter", { - gradeNames: ["gpii.dataLoader.prefsDataLoader.dataConverter"], - dataPath: "data/", - expected: [{ - carla: { - "flat": { - "contexts": { - "gpii-default": { - "name": "Default preferences", - "preferences": { - "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { - "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, - "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, - "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false - } - } - } - } - } - } - }, { - sammy: { - "flat": { - "contexts": { - "gpii-default": { - "name": "Default preferences", - "preferences": { - "http://registry.gpii.net/common/fontSize": 24, - "http://registry.gpii.net/common/foregroundColor": "white", - "http://registry.gpii.net/common/backgroundColor": "black", - "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], - "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", - "http://registry.gpii.net/common/magnification": 2.0, - "http://registry.gpii.net/common/tracking": ["mouse"], - "http://registry.gpii.net/common/invertColours": true - } - } - } - } - } - }] - }); - - jqUnit.test("Test gpii.dataLoader.processDataFiles", function () { - var testData = { - case1: { - db1: { - dataFile: [ - "%universal/tests/scripts/data/prefs/carla.json", - "%universal/tests/scripts/data/views-test.json" - ], - data: { - "test1": "test1" - } - }, - db2: { - data: { - "test2": "test2" - } - } - }, - case2: { - db1: { - dataFile: [ - "%universal/tests/scripts/data/nonExistent1.json", - "%universal/tests/scripts/data/prefs/sammy.json" - ], - data: { - "test1": "test1" - } - }, - db2: { - dataFile: [ - "%universal/tests/scripts/data/nonExistent2.json" - ], - data: { - "test2": "test2" - } - } - } - }; - - var expected = { - case1: { - errors: [], - databases: { - db1: [{ - "flat": { - "contexts": { - "gpii-default": { - "name": "Default preferences", - "preferences": { - "http://registry.gpii.net/applications/com.texthelp.readWriteGold": { - "ApplicationSettings.AppBar.optToolbarLargeIcons.$t": true, - "ApplicationSettings.Speech.optSAPI5Speed.$t": 50, - "ApplicationSettings.Speech.optAutoUseScreenReading.$t": false - } - } - } - } - } - }, { - "_id": "_design/views", - "views": { - "findUserByName": { - "map": "function(doc) {if (doc.type === 'user') emit(doc.name, doc)}" - } - } - }, { - "test1": "test1" - }], - db2: [{ - "test2": "test2" - }] - } - }, - case2: { - // The errors array contains the regex patterns to match actual returned errors - // since the actual errors are full paths of files that vary when this test runs - // on different machines. - errors: [ - ".*tests\/scripts\/data\/nonExistent1.json", - ".*tests\/scripts\/data\/nonExistent2.json" - ], - databases: { - db1: [{ - "flat": { - "contexts": { - "gpii-default": { - "name": "Default preferences", - "preferences": { - "http://registry.gpii.net/common/fontSize": 24, - "http://registry.gpii.net/common/foregroundColor": "white", - "http://registry.gpii.net/common/backgroundColor": "black", - "http://registry.gpii.net/common/fontFaceFontName": ["Comic Sans"], - "http://registry.gpii.net/common/fontFaceGenericFontFace": "sans serif", - "http://registry.gpii.net/common/magnification": 2.0, - "http://registry.gpii.net/common/tracking": ["mouse"], - "http://registry.gpii.net/common/invertColours": true - } - } - } - } - }, { - "test1": "test1" - }], - db2: [{ - "test2": "test2" - }] - } - } - }; - - fluid.each(testData, function (data, caseName) { - var result = gpii.dataLoader.processDataFiles(data); - - // Verify result.data - jqUnit.assertDeepEq("The returned result is expected", expected[caseName].databases, result.databases); - - // Verify result.errors - var isErrorMatch = gpii.tests.dataLoader.verifyMatches(result.errors, expected[caseName].errors); - jqUnit.assertTrue("All expected errors are reported", isErrorMatch); - }); - }); -})(); From 0e75218f8af5ed7f5597c1853b0b81219a224372 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 16 Feb 2017 21:18:27 -0500 Subject: [PATCH 26/35] GPII-1987: Adjusted data loader scripts to use the new gpii dataLoader module. --- scripts/dataLoader-auth.js | 4 +--- scripts/dataLoader-prefs.js | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/scripts/dataLoader-auth.js b/scripts/dataLoader-auth.js index 542a1e67c..c2e21f5cd 100644 --- a/scripts/dataLoader-auth.js +++ b/scripts/dataLoader-auth.js @@ -22,9 +22,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt var fluid = require("infusion"), gpii = fluid.registerNamespace("gpii"); -fluid.setLogging(true); - -require("./shared/authDataLoader.js"); +require("../gpii/node_modules/dataLoader/index.js"); gpii.dataLoader.authDataLoader({ dbName: "auth", diff --git a/scripts/dataLoader-prefs.js b/scripts/dataLoader-prefs.js index 8103bb4dc..77119ad07 100644 --- a/scripts/dataLoader-prefs.js +++ b/scripts/dataLoader-prefs.js @@ -22,9 +22,7 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt var fluid = require("infusion"), gpii = fluid.registerNamespace("gpii"); -fluid.setLogging(true); - -require("./shared/prefsDataLoader.js"); +require("../gpii/node_modules/dataLoader/index.js"); gpii.dataLoader.prefsDataLoader({ dbName: "preferences", From 1af51a9a70da8c9ab6b06d9cf40528b4346c7a5c Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 16 Feb 2017 21:40:41 -0500 Subject: [PATCH 27/35] GPII-1987: Use the correct branch to pull out ansible-preferences-server for setting up production VM. --- vagrant-configs/provisioning/requirements.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vagrant-configs/provisioning/requirements.yml b/vagrant-configs/provisioning/requirements.yml index ef7ed0a60..427d135f7 100644 --- a/vagrant-configs/provisioning/requirements.yml +++ b/vagrant-configs/provisioning/requirements.yml @@ -8,7 +8,7 @@ name: nodejs - src: https://github.com/avtar/ansible-preferences-server - version: remotes/origin/GPII-1884 + version: remotes/origin/fixes-paths-names name: preferences-server - src: https://github.com/gpii-ops/ansible-flow-manager From 72aab6857a55c536b81fd44ac3b6dae88b5896e0 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 21 Feb 2017 12:54:07 -0500 Subject: [PATCH 28/35] GPII-1987: Use the latest gpii-pouchdb changes to include the improvements on the pouchdb cleanup cycle. Also improved the data loader documentation. --- documentation/DataLoader.md | 19 ++++++++++++++++++- package.json | 2 +- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/documentation/DataLoader.md b/documentation/DataLoader.md index e59749a5c..3e8830d69 100644 --- a/documentation/DataLoader.md +++ b/documentation/DataLoader.md @@ -54,9 +54,26 @@ node scripts/dataLoader-auth.js | Option | Type | Description | Default | | ----------------- | ---------- | ----------- | ------- | | `dbName` | String | Required. The name of the CouchDB database to be created for loading authorization data. If the database already exists, delete it and recreate. | auth | -| `dataFile` | Array | Required. An array of data files to be loaded. | ["%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json"] | +| `dataFile` | Array | Required. An array of JSON data files to be loaded. See the section below for an example of a JSON data file.| ["%universal/testData/security/TestOAuth2DataStore.json", "%universal/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json"] | | `couchDbUrl` | String | Required. The URL to the CouchDB. | `http://localhost:5984` | +**Example of A JSON Data File** +``` +[ + { + "_id": "user-1", + "type": "user", + "name": "chromehc", + "defaultGpiiToken": "review3_chrome_high_contrast" + }, { + "_id": "user-2", + "type": "user", + "name": "ma1", + "defaultGpiiToken": "review3_ma1" + } +] +``` + #### Supported Events | Event | Description | Parameters | Parameters Description | diff --git a/package.json b/package.json index c6c993f58..856c71c6b 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "glob": "7.1.1", "write-file-atomic": "1.3.1", "json5": "0.5.1", - "gpii-pouchdb": "git://github.com/the-t-in-rtf/gpii-pouchdb#3bf92dbb99570154b880e6254601b532861b1ee4", + "gpii-pouchdb": "git://github.com/the-t-in-rtf/gpii-pouchdb#83f5aca07b2835c216da766caee5bde568db1411", "fluid-resolve": "1.2.0", "rimraf": "2.5.4", "mkdirp": "0.5.1" From 34e211e40198489c4f08ffb3d8df5d2017f3c9a9 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Tue, 21 Feb 2017 14:36:42 -0500 Subject: [PATCH 29/35] GPII-1987: Sync up infusion versions used by universal and gpii-pouchdb to fix the web test failure. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 856c71c6b..889ec6448 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "bugs": "http://issues.gpii.net/browse/GPII", "homepage": "http://gpii.net/", "dependencies": { - "infusion": "3.0.0-dev.20170127T130413Z.103de6e", + "infusion": "3.0.0-dev.20170131T153243Z.6aab53a", "node-jqunit": "1.1.4", "ini": "git://github.com/GPII/ini.git#be8a04aa22f5ad9321ebcbba7740314a53bc8ab2", "node-uuid": "1.4.7", From 4f7f072798987728c0d26835a50c5ef686862a8a Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 22 Feb 2017 10:32:05 -0500 Subject: [PATCH 30/35] GPII-1987: Use the correct gpii-pouchdb version after checking with Tony. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 889ec6448..86013f183 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "glob": "7.1.1", "write-file-atomic": "1.3.1", "json5": "0.5.1", - "gpii-pouchdb": "git://github.com/the-t-in-rtf/gpii-pouchdb#83f5aca07b2835c216da766caee5bde568db1411", + "gpii-pouchdb": "1.0.1-dev.20170208T180039Z.83f5aca", "fluid-resolve": "1.2.0", "rimraf": "2.5.4", "mkdirp": "0.5.1" From 3907f0c3a9d099f5e655a1fdfb60cd857d1c2dac Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Thu, 23 Feb 2017 11:42:54 -0500 Subject: [PATCH 31/35] GPII-1987: Improved code comments and linted. --- .../node_modules/dataLoader/src/dataLoader.js | 19 ++++++++++++------- .../dataLoader/src/prefsDataLoader.js | 2 +- .../dataLoader/test/AuthDataLoaderTests.js | 4 ++-- .../dataLoader/test/PrefsDataLoaderTests.js | 4 ++-- 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/gpii/node_modules/dataLoader/src/dataLoader.js b/gpii/node_modules/dataLoader/src/dataLoader.js index 5c7bee5e1..cd12eac9f 100644 --- a/gpii/node_modules/dataLoader/src/dataLoader.js +++ b/gpii/node_modules/dataLoader/src/dataLoader.js @@ -11,7 +11,7 @@ You may obtain a copy of the License at https://github.com/GPII/universal/blob/master/LICENSE.txt */ -// The underlying data loader implementation shared by ../authDataLoader.js and ../prefsDataLoader.js +// The underlying data loader implementation shared by ./authDataLoader.js and ./prefsDataLoader.js // to load GPII test data into CouchDB. // Steps to load data: // 1. Check the existence of the database; @@ -21,10 +21,11 @@ https://github.com/GPII/universal/blob/master/LICENSE.txt "use strict"; var fluid = require("infusion"), - kettle = require("kettle"), gpii = fluid.registerNamespace("gpii"), fs = require("fs"); +fluid.require("kettle", require); + // Data Source used to interact with CouchDB fluid.defaults("gpii.dataLoader.dataSource", { gradeNames: ["kettle.dataSource.URL"], @@ -101,9 +102,13 @@ fluid.defaults("gpii.dataLoader", { /** * The main load function that processes the data loading for databases defined in `that.options.databases` one by one. - * The data can be provided in 2 forms as one or both listed sub-options into `that.options.databases`: - * 1. `dataFile` option: an array of paths to data files. For example: dataFile: ["pathA/a.json", "pathB/b.json"]; - * 2. `data` option: the direct data in the structure that is allowed by CouchDB /_bulk_docs API (http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs) + * The structure of `that.options.databases`: + * { + * dbName1: [data], + * dbName2: [data] + * ... + * } + * The format of [data] is in the structure allowed by CouchDB /_bulk_docs API (http://docs.couchdb.org/en/2.0.0/api/database/bulk-api.html#db-bulk-docs) * * @param that {Component} An instance of "gpii.dataLoader" * @return {Promise} The promise of the data loading process. @@ -126,7 +131,7 @@ gpii.dataLoader.load = function (that) { dbName: dbName }; - // Delete the database if it already exists + // Delete and recreate the database if it already exists; otherwise, create the database var prepareDbPromise = gpii.dataLoader.prepareDB(that.getDbDataSource, that.createDbDataSource, that.deleteDbDataSource, directModel); prepareDbPromise.then(function () { @@ -246,7 +251,7 @@ gpii.dataLoader.loadData = function (loadDataSource, data, directModel) { } // Convert to couchDB accepted doc format for using /_bulk_docs end point - var data = { + data = { "docs": data }; diff --git a/gpii/node_modules/dataLoader/src/prefsDataLoader.js b/gpii/node_modules/dataLoader/src/prefsDataLoader.js index 82887a6a7..47b36bd55 100644 --- a/gpii/node_modules/dataLoader/src/prefsDataLoader.js +++ b/gpii/node_modules/dataLoader/src/prefsDataLoader.js @@ -152,7 +152,7 @@ fluid.defaults("gpii.dataLoader.prefsDataLoader.dataConverter", { args: ["{that}.options.dataPath", "{that}.events.onPrefsDataStructureConstructed"] } } - }); +}); /** * Read each preferences data from physical json file and construct them into a data structure that is accepted by diff --git a/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js b/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js index e16b0c715..50d445b2f 100644 --- a/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js +++ b/gpii/node_modules/dataLoader/test/AuthDataLoaderTests.js @@ -32,7 +32,7 @@ fluid.defaults("gpii.tests.authDataLoader.success", { couchDbUrl: "http://localhost:1234" }); -// This component must be name as "testEnvironment" because of the reason described at line 25 +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 fluid.defaults("gpii.tests.authDataLoader.success.testEnvironment", { gradeNames: ["gpii.tests.dataLoader.testEnvironment"], dataLoaderGrade: "gpii.tests.authDataLoader.success", @@ -98,7 +98,7 @@ fluid.defaults("gpii.tests.authDataLoader.error", { couchDbUrl: "http://localhost:1234" }); -// This component must be name as "testEnvironment" because of the reason described at line 25 +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 fluid.defaults("gpii.tests.authDataLoader.error.testEnvironment", { gradeNames: ["gpii.tests.dataLoader.testEnvironment"], dataLoaderGrade: "gpii.tests.authDataLoader.error" diff --git a/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js b/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js index fea090394..42b8f5ddb 100644 --- a/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js +++ b/gpii/node_modules/dataLoader/test/PrefsDataLoaderTests.js @@ -95,7 +95,7 @@ fluid.defaults("gpii.tests.prefsDataLoader.success", { couchDbUrl: "http://localhost:1234" }); -// This component must be name as "testEnvironment" because of the reason described at line 25 +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 fluid.defaults("gpii.tests.prefsDataLoader.success.testEnvironment", { gradeNames: ["gpii.tests.dataLoader.testEnvironment"], dataLoaderGrade: "gpii.tests.prefsDataLoader.success", @@ -158,7 +158,7 @@ fluid.defaults("gpii.tests.prefsDataLoader.error", { couchDbUrl: "http://localhost:1234" }); -// This component must be name as "testEnvironment" because of the reason described at line 25 +// This component must be name as "testEnvironment" because of the reason described at ./DataLoaderTestsUtils.js line 19 fluid.defaults("gpii.tests.prefsDataLoader.error.testEnvironment", { gradeNames: ["gpii.tests.dataLoader.testEnvironment"], dataLoaderGrade: "gpii.tests.prefsDataLoader.error" From 28182003c4be3d5716934017ba5e25b69f0416b7 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 24 Feb 2017 14:32:06 -0500 Subject: [PATCH 32/35] GPII-1987: Switched to use gpii-ops/ansible-preferences-server master branch since avtar's pull request has been merged in. --- vagrant-configs/provisioning/requirements.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vagrant-configs/provisioning/requirements.yml b/vagrant-configs/provisioning/requirements.yml index 427d135f7..271ccc7bc 100644 --- a/vagrant-configs/provisioning/requirements.yml +++ b/vagrant-configs/provisioning/requirements.yml @@ -7,8 +7,7 @@ - src: https://github.com/idi-ops/ansible-nodejs name: nodejs -- src: https://github.com/avtar/ansible-preferences-server - version: remotes/origin/fixes-paths-names +- src: https://github.com/gpii-ops/ansible-preferences-server name: preferences-server - src: https://github.com/gpii-ops/ansible-flow-manager From f6a1d79fca4787eddca051116a725abb7dc36fb3 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Wed, 26 Apr 2017 15:14:43 -0400 Subject: [PATCH 33/35] GPII-1987: Experimenting with the lastest gpii-pouchdb release. --- .../gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js | 5 ++++- gpii/node_modules/pouchManager/test/pouchManagerTests.js | 5 ++++- gpii/node_modules/testing/src/PouchTestCaseHolder.js | 5 ++++- package.json | 4 ++-- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js b/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js index 85e0971d4..1883ab02e 100644 --- a/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js +++ b/gpii/node_modules/gpii-oauth2/gpii-oauth2-datastore/test/DbDataStoreTestsUtils.js @@ -28,7 +28,10 @@ fluid.defaults("gpii.tests.dbDataStore.environment", { "%gpii-oauth2/gpii-oauth2-datastore/test/data/gpiiAuthTestData.json", "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] - } + }, + _users: {}, + _replicator: {}, + pouch__all_dbs__: {} } }, components: { diff --git a/gpii/node_modules/pouchManager/test/pouchManagerTests.js b/gpii/node_modules/pouchManager/test/pouchManagerTests.js index 8bb1e600a..ce5f3a814 100644 --- a/gpii/node_modules/pouchManager/test/pouchManagerTests.js +++ b/gpii/node_modules/pouchManager/test/pouchManagerTests.js @@ -38,7 +38,10 @@ fluid.defaults("gpii.tests.pouchManager", { data: [ "%universal/gpii/node_modules/pouchManager/test/data/oauth2TestData.json" ] - } + }, + _users: {}, + _replicator: {}, + pouch__all_dbs__: {} } } }); diff --git a/gpii/node_modules/testing/src/PouchTestCaseHolder.js b/gpii/node_modules/testing/src/PouchTestCaseHolder.js index e7ed7b8c9..dd3a2c2a4 100644 --- a/gpii/node_modules/testing/src/PouchTestCaseHolder.js +++ b/gpii/node_modules/testing/src/PouchTestCaseHolder.js @@ -45,7 +45,10 @@ fluid.defaults("gpii.test.pouch.pouchTestCaseHolder", { "%universal/tests/platform/cloud/OAuth2AcceptanceDataStore.json", "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] - } + }, + _users: {}, + _replicator: {}, + pouch__all_dbs__: {} } }, events: { diff --git a/package.json b/package.json index 42614b301..f8c9ec089 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "bugs": "http://issues.gpii.net/browse/GPII", "homepage": "http://gpii.net/", "dependencies": { - "infusion": "3.0.0-dev.20170131T153243Z.6aab53a", + "infusion": "3.0.0-dev.20170322T234120Z.278de35", "node-jqunit": "1.1.4", "ini": "git://github.com/GPII/ini.git#be8a04aa22f5ad9321ebcbba7740314a53bc8ab2", "node-uuid": "1.4.7", @@ -25,7 +25,7 @@ "glob": "7.1.1", "write-file-atomic": "1.3.1", "json5": "0.5.1", - "gpii-pouchdb": "1.0.1-dev.20170208T180039Z.83f5aca", + "gpii-pouchdb": "1.0.1", "fluid-resolve": "1.2.0", "rimraf": "2.5.4", "mkdirp": "0.5.1" From 2da973e451d1a9f3d59761f18a99bd634d6a2333 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Fri, 28 Apr 2017 14:49:06 -0400 Subject: [PATCH 34/35] GPII-1987: Updated to use the latest gpii-pouchdb repo commits. --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index f8c9ec089..ad61fe95c 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "glob": "7.1.1", "write-file-atomic": "1.3.1", "json5": "0.5.1", - "gpii-pouchdb": "1.0.1", + "gpii-pouchdb": "1.0.2-dev.20170428T084916Z.058ac5a", "fluid-resolve": "1.2.0", "rimraf": "2.5.4", "mkdirp": "0.5.1" From f02e7d855fb2e253211147e73d298637aae2c640 Mon Sep 17 00:00:00 2001 From: Cindy Qi Li Date: Mon, 1 May 2017 11:40:45 -0400 Subject: [PATCH 35/35] GPII-1987: Updated to use the latest gpii-pouchdb release. --- .../pouchManager/configs/gpii.pouchManager.config.base.json | 5 ++++- package.json | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json index 251505a47..f937142e5 100644 --- a/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json +++ b/gpii/node_modules/pouchManager/configs/gpii.pouchManager.config.base.json @@ -11,7 +11,10 @@ "%universal/testData/security/TestOAuth2DataStore.json", "%gpii-oauth2/gpii-oauth2-datastore/dbViews/views.json" ] - } + }, + "_users": {}, + "_replicator": {}, + "pouch__all_dbs__": {} } } }, diff --git a/package.json b/package.json index ad61fe95c..d0badaedc 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,7 @@ "glob": "7.1.1", "write-file-atomic": "1.3.1", "json5": "0.5.1", - "gpii-pouchdb": "1.0.2-dev.20170428T084916Z.058ac5a", + "gpii-pouchdb": "1.0.2", "fluid-resolve": "1.2.0", "rimraf": "2.5.4", "mkdirp": "0.5.1"