From db15f62f9f2d0c11b1cdc1a385f8891616dbaf58 Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 2 May 2018 14:25:17 +0100 Subject: [PATCH 01/56] GPII-2971: The beginning of IoD. --- gpii/node_modules/installOnDemand/README.md | 32 ++ gpii/node_modules/installOnDemand/index.js | 22 ++ .../node_modules/installOnDemand/package.json | 13 + .../installOnDemand/scripts/setup.ps1 | 3 + .../installOnDemand/src/installOnDemand.js | 372 ++++++++++++++++++ .../installOnDemand/src/packageInstaller.js | 36 ++ .../test/installOnDemandTests.js | 39 ++ index.js | 1 + 8 files changed, 518 insertions(+) create mode 100644 gpii/node_modules/installOnDemand/README.md create mode 100644 gpii/node_modules/installOnDemand/index.js create mode 100644 gpii/node_modules/installOnDemand/package.json create mode 100644 gpii/node_modules/installOnDemand/scripts/setup.ps1 create mode 100644 gpii/node_modules/installOnDemand/src/installOnDemand.js create mode 100644 gpii/node_modules/installOnDemand/src/packageInstaller.js create mode 100644 gpii/node_modules/installOnDemand/test/installOnDemandTests.js diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/installOnDemand/README.md new file mode 100644 index 000000000..6860bfb3a --- /dev/null +++ b/gpii/node_modules/installOnDemand/README.md @@ -0,0 +1,32 @@ +# Install on Demand + +Provides the ability to install software on demand. + +## `gpii.iod` component + +The stages of installation: + +### getInfo +Retrieves the package information (eg download location, installation instructions) + +### download +Downloads the package. + +### check +Checks the downloaded package. + +### prepareInstall +Generates the installation commands. + +Finds the `packageInstaller` component that handles this type of package. + +### install +Installs the package. + +### cleanup +Cleans the files. + + +## `gpii.iod.packageInstaller` + +Base component of the package installers, which perform the work that's specific to the type of package being installed. diff --git a/gpii/node_modules/installOnDemand/index.js b/gpii/node_modules/installOnDemand/index.js new file mode 100644 index 000000000..73345c021 --- /dev/null +++ b/gpii/node_modules/installOnDemand/index.js @@ -0,0 +1,22 @@ +/* + * Install on Demand. + * + * Copyright 2018 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +require("./src/installOnDemand.js"); +require("./src/packageInstaller.js"); diff --git a/gpii/node_modules/installOnDemand/package.json b/gpii/node_modules/installOnDemand/package.json new file mode 100644 index 000000000..5781321d0 --- /dev/null +++ b/gpii/node_modules/installOnDemand/package.json @@ -0,0 +1,13 @@ +{ + "name": "installOnDemand", + "description": "Install on Demand", + "version": "0.3.0", + "author": "GPII", + "bugs": "http://issues.gpii.net/browse/GPII", + "homepage": "http://gpii.net/", + "dependencies": {}, + "license" : "BSD-3-Clause", + "repository": "git://github.com/GPII/windows.git", + "main": "./index.js", + "engines": { "node" : ">=4.2.1" } +} diff --git a/gpii/node_modules/installOnDemand/scripts/setup.ps1 b/gpii/node_modules/installOnDemand/scripts/setup.ps1 new file mode 100644 index 000000000..badaf0500 --- /dev/null +++ b/gpii/node_modules/installOnDemand/scripts/setup.ps1 @@ -0,0 +1,3 @@ + +Set-ExecutionPolicy Bypass -Scope Process -Force +iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js new file mode 100644 index 000000000..69b02d138 --- /dev/null +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -0,0 +1,372 @@ +/* + * Install on Demand. + * + * Copyright 2018 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); + +var path = require("path"), + os = require("os"), + fs = require("fs"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.iod"); + +require("./packageInstaller.js"); + +/** + * The stages of installation: + * - getInfo: Retrieves the package information (eg download location, installation instructions) + * - download: Downloads the package. + * - check: Checks the downloaded package. + * - prepareInstall: Generates the installation commands. + * - install: Installs the package. + * - cleanup: Cleans the files. + */ + +fluid.defaults("gpii.iod", { + gradeNames: ["fluid.component"], + contextAwareness: { + platform: { + checks: { + windows: { + contextValue: "{gpii.contexts.windows}", + gradeNames: ["gpii.windows.iod"] + } + } + } + }, + invokers: { + requirePackage: { + funcName: "gpii.iod.requirePackage", + args: ["{that}", "{arguments}.0"] + }, + startRemoval: { + funcName: "gpii.iod.startRemoval", + args: ["{that}", "{arguments}.0"] + }, + findInstaller: { + funcName: "gpii.iod.findInstaller", + args: ["{that}", "{arguments}.0"] + }, + getWorkingPath: { + funcName: "gpii.iod.getWorkingPath", + args: ["{arguments}.0"] + }, + // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns + // a installation, either directly or via a promise. + getPackageInfo: { + funcName: "gpii.iod.getPackageInfo", + args: ["{that}", "{arguments}.0"] + }, + downloadPackage: { + funcName: "gpii.iod.downloadPackage", + args: ["{that}", "{arguments}.0"] + }, + checkPackage: { + funcName: "gpii.iod.checkPackage", + args: ["{that}", "{arguments}.0"] + }, + prepareInstall: { + funcName: "gpii.iod.prepareInstall", + args: ["{that}", "{arguments}.0"] + }, + installPackage: { + funcName: "gpii.iod.installPackage", + args: ["{that}", "{arguments}.0"] + }, + cleanup: { + funcName: "gpii.iod.cleanup", + args: ["{that}", "{arguments}.0", "{arguments}.1"] + }, + uninstallPackage: { + funcName: "gpii.iod.uninstallPackage", + args: ["{that}", "{arguments}.0"] + } + }, + events: { + // Dummy events for the installation pipe-lines + onRequirePackage: null, + onRemovePackage: null + }, + listeners: { + "onRequirePackage.getInfo": { + func: "{that}.getPackageInfo", + priority: "first" + }, + "onRequirePackage.download": { + func: "{that}.downloadPackage", + priority: "after:getInfo" + }, + "onRequirePackage.check": { + func: "{that}.checkPackage", + priority: "after:download" + }, + "onRequirePackage.prepareInstall": { + func: "{that}.prepareInstall", + priority: "after:check" + }, + "onRequirePackage.install": { + func: "{that}.installPackage", + priority: "after:prepareInstall" + }, + "onRequirePackage.cleanup": { + func: "{that}.cleanup", + priority: "after:install" + }, + "onRemovePackage.uninstallPackage": { + func: "{that}.uninstallPackage", + priority: "first" + } + }, + + members: { + installations: {} + } +}); + +/** + * Create a directory where packages are temporarily stored. + * @param packageName {String} Name of the package for which the directory is being created. + */ +gpii.iod.getWorkingPath = function (packageName) { + var parts = [ + os.tmpdir(), + "gpii-iod", + packageName && packageName.replace(/[^a-z0-9]/, "_"), + Math.random().toString(36) + ]; + + return parts.reduce(function (parent, child) { + var dir = path.join(parent, child); + try { + fs.mkdirSync(dir); + } catch (e) { + if (e.code !== "EEXIST") { + throw e; + } + } + return dir; + }, ""); +}; + +/** + * Finds a package installer component that handles the given type of package. + * + * @param that {Component} The gpii.iod instance. + * @param packageType {string} The package type identifier. + * @return {Component} A gpii.iod.installer component that handles the requested type of package. + */ +gpii.iod.findInstaller = function (that, packageType) { + var packageInstallers = fluid.queryIoCSelector(that, "gpii.iod.packageInstaller"); + + return fluid.find(packageInstallers, function (installer) { + var packageTypes = fluid.makeArray(installer.options.packageTypes); + return packageTypes.indexOf(packageType) >= 0 + ? installer + : undefined; + }); +}; + + +/** + * Starts the process of installing a package. + * + * @param that {Component} The gpii.iod instance. + * @param packageName {string} The package name. + */ +gpii.iod.requirePackage = function (that, packageName) { + fluid.log("IoD: Requiring " + packageName); + + var installation = { + id: "aaa", + packageName: packageName + }; + that.installations[installation.id] = installation; + + var promise = fluid.promise.fireTransformEvent(that.events.onRequirePackage, installation); + + return promise.then(function () { + fluid.log("IoD: Installation complete"); + }, function (err) { + fluid.log("IoD: Installation failed:", err.error || err); + var installation = err.installation; + if (!installation) { + installation = that.installations[packageName]; + } + if (installation) { + installation.failed = true; + that.cleanup(installation); + } + }); +}; + +/** + * Retrieve the package metadata. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.getPackageInfo = function (that, installation) { + fluid.log("IoD: Getting package info for " + installation.packageName); + + var packages = { + "wget": { + name: "wget", + url: "e:\\Wget.1.19.4.nupkg", + //url: "https://chocolatey.org/api/v2/package/Wget/1.19.4", + filename: "Wget.1.19.4.nupkg", + packageType: "chocolatey" + } + }; + + installation.packageInfo = Object.assign({}, packages[installation.packageName]); + + var promise = fluid.promise(); + if (installation.packageInfo) { + promise.resolve(installation); + } else { + promise.reject({ + isError: true, + error: "no such package: " + installation.packageName + }); + } + + return promise; +}; + +/** + * Downloads a package from the server. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.downloadPackage = function (that, installation) { + fluid.log("IoD: Downloading package " + installation.packageInfo.url); + + var promise = fluid.promise(); + + installation.tempDir = that.getWorkingPath(installation.packageName); + installation.localPackage = path.join(installation.tempDir, installation.packageInfo.filename); + + fs.copyFile(installation.packageInfo.url, installation.localPackage, function (err) { + if (err) { + promise.reject({ + isError: true, + message: "Unable to download package" + }); + } else { + promise.resolve(installation); + } + }); + + return promise; +}; + +/** + * Checks that a downloaded package is ok. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.checkPackage = function (that, installation) { + fluid.log("IoD: Checking downloaded package file " + installation.packageInfo.filename); + return installation; +}; + +/** + * Generate the installation instructions. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.prepareInstall = function (that, installation) { + fluid.log("IoD: Preparing installation for " + installation.packageName); + + installation.installer = that.findInstaller(installation.packageInfo.packageType); + + var promise = fluid.promise(); + if (installation.installer) { + promise = installation.installer.prepareInstall(installation); + } else { + promise = fluid.promise(); + promise.reject({ + isError: true, + error: "Unable to find a package installer for packageType '" + installation.packageInfo.packageType + "'" + }); + } + + return promise; +}; + +/** + * Installs the package. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.installPackage = function (that, installation) { + fluid.log("IoD: Installing package " + installation.packageInfo.filename); + return installation.installer.installPackage(installation); +}; + +/** + * Cleans up things that are no longer required. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.cleanup = function (that, installation) { + fluid.log("IoD: Cleaning installation of " + installation.packageName); + + var result = installation.installer.cleanup(installation); + return fluid.toPromise(result).then(function () { + delete that.installations[installation]; + }); +}; + +/** + * Starts the package removal routine. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + * @return {Promise} Resolves to an object containing package information and installation state. + */ +gpii.iod.startRemoval = function (that, installation) { + if (typeof(installation) === "string") { + installation = that.installations[installation]; + } + + if (!installation.installer) { + installation.installer = that.findInstaller(installation.packageInfo.packageType); + } + + fluid.log("IoD: Removing installation of " + installation.packageName); + var promise = fluid.promise.fireTransformEvent(that.events.onRequirePackage, installation); + return promise; +}; + +gpii.iod.uninstallPackage = function (that, installation) { + installation.installer.uninstallPackage(installation); +}; diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js new file mode 100644 index 000000000..1d8dde8ca --- /dev/null +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -0,0 +1,36 @@ +/* + * Abstraction of something that installs packages. + * + * Copyright 2018 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); +fluid.registerNamespace("gpii.iod"); + +fluid.defaults("gpii.iod.packageInstaller", { + gradeNames: ["fluid.component"], + + invokers: { + prepareInstall: "fluid.notImplemented", + installPackage: "fluid.notImplemented", + cleanup: "fluid.notImplemented", + uninstallPackage: "fluid.notImplemented" + }, + + packageTypes: null +}); + diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js new file mode 100644 index 000000000..3ec69ce89 --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -0,0 +1,39 @@ +/* + * IoD Tests. + * + * Copyright 2018 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("gpii-universal"); + +var jqUnit = fluid.require("node-jqunit"); +var gpii = fluid.registerNamespace("gpii"); + +fluid.registerNamespace("gpii.tests.iod"); + +require("../index.js"); + +jqUnit.module("gpii.tests.iod"); + + +jqUnit.asyncTest("install tests", function () { + + var iod = gpii.iod(); + + iod.requirePackage("wget"); + +}); diff --git a/index.js b/index.js index f44e3cc1b..2acb72957 100644 --- a/index.js +++ b/index.js @@ -41,6 +41,7 @@ require("./gpii/node_modules/pouchManager"); require("./gpii/node_modules/eventLog"); require("./gpii/node_modules/processReporter"); require("./gpii/node_modules/userListeners"); +require("./gpii/node_modules/installOnDemand"); gpii.loadTestingSupport = function () { fluid.contextAware.makeChecks({ From 6ad68cfb816c859b01f3c12c49fcfa43379a51d8 Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 2 May 2018 15:35:45 +0100 Subject: [PATCH 02/56] GPII-2971: Moved installation routing into installer component. --- .../installOnDemand/src/installOnDemand.js | 205 +++--------------- .../installOnDemand/src/packageInstaller.js | 149 ++++++++++++- 2 files changed, 172 insertions(+), 182 deletions(-) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 69b02d138..af087652a 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -29,16 +29,6 @@ fluid.registerNamespace("gpii.iod"); require("./packageInstaller.js"); -/** - * The stages of installation: - * - getInfo: Retrieves the package information (eg download location, installation instructions) - * - download: Downloads the package. - * - check: Checks the downloaded package. - * - prepareInstall: Generates the installation commands. - * - install: Installs the package. - * - cleanup: Cleans the files. - */ - fluid.defaults("gpii.iod", { gradeNames: ["fluid.component"], contextAwareness: { @@ -56,6 +46,10 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.requirePackage", args: ["{that}", "{arguments}.0"] }, + getPackageInfo: { + funcName: "gpii.iod.getPackageInfo", + args: ["{that}", "{arguments}.0"] + }, startRemoval: { funcName: "gpii.iod.startRemoval", args: ["{that}", "{arguments}.0"] @@ -67,71 +61,6 @@ fluid.defaults("gpii.iod", { getWorkingPath: { funcName: "gpii.iod.getWorkingPath", args: ["{arguments}.0"] - }, - // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns - // a installation, either directly or via a promise. - getPackageInfo: { - funcName: "gpii.iod.getPackageInfo", - args: ["{that}", "{arguments}.0"] - }, - downloadPackage: { - funcName: "gpii.iod.downloadPackage", - args: ["{that}", "{arguments}.0"] - }, - checkPackage: { - funcName: "gpii.iod.checkPackage", - args: ["{that}", "{arguments}.0"] - }, - prepareInstall: { - funcName: "gpii.iod.prepareInstall", - args: ["{that}", "{arguments}.0"] - }, - installPackage: { - funcName: "gpii.iod.installPackage", - args: ["{that}", "{arguments}.0"] - }, - cleanup: { - funcName: "gpii.iod.cleanup", - args: ["{that}", "{arguments}.0", "{arguments}.1"] - }, - uninstallPackage: { - funcName: "gpii.iod.uninstallPackage", - args: ["{that}", "{arguments}.0"] - } - }, - events: { - // Dummy events for the installation pipe-lines - onRequirePackage: null, - onRemovePackage: null - }, - listeners: { - "onRequirePackage.getInfo": { - func: "{that}.getPackageInfo", - priority: "first" - }, - "onRequirePackage.download": { - func: "{that}.downloadPackage", - priority: "after:getInfo" - }, - "onRequirePackage.check": { - func: "{that}.checkPackage", - priority: "after:download" - }, - "onRequirePackage.prepareInstall": { - func: "{that}.prepareInstall", - priority: "after:check" - }, - "onRequirePackage.install": { - func: "{that}.installPackage", - priority: "after:prepareInstall" - }, - "onRequirePackage.cleanup": { - func: "{that}.cleanup", - priority: "after:install" - }, - "onRemovePackage.uninstallPackage": { - func: "{that}.uninstallPackage", - priority: "first" } }, @@ -183,12 +112,12 @@ gpii.iod.findInstaller = function (that, packageType) { }); }; - /** * Starts the process of installing a package. * * @param that {Component} The gpii.iod instance. * @param packageName {string} The package name. + * @return {Promise} Resolves when the installation is complete. */ gpii.iod.requirePackage = function (that, packageName) { fluid.log("IoD: Requiring " + packageName); @@ -199,7 +128,20 @@ gpii.iod.requirePackage = function (that, packageName) { }; that.installations[installation.id] = installation; - var promise = fluid.promise.fireTransformEvent(that.events.onRequirePackage, installation); + var promise = fluid.promise(); + + that.getPackageInfo(packageName).then(function (packageInfo) { + var installer = that.findInstaller(packageInfo.packageType); + if (installer) { + var result = installer.startInstaller(packageInfo); + fluid.promise.follow(result, promise); + } else { + promise.reject({ + isError: true, + error: "Unable to find an installer for package type " + packageInfo.packageTypes + }); + } + }); return promise.then(function () { fluid.log("IoD: Installation complete"); @@ -221,10 +163,10 @@ gpii.iod.requirePackage = function (that, packageName) { * * @param that {Component} The gpii.iod instance. * @param installation {object} The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. + * @return {Promise} Resolves to an object containing package information. */ -gpii.iod.getPackageInfo = function (that, installation) { - fluid.log("IoD: Getting package info for " + installation.packageName); +gpii.iod.getPackageInfo = function (that, packageName) { + fluid.log("IoD: Getting package info for " + packageName); var packages = { "wget": { @@ -236,116 +178,21 @@ gpii.iod.getPackageInfo = function (that, installation) { } }; - installation.packageInfo = Object.assign({}, packages[installation.packageName]); - - var promise = fluid.promise(); - if (installation.packageInfo) { - promise.resolve(installation); - } else { - promise.reject({ - isError: true, - error: "no such package: " + installation.packageName - }); - } - - return promise; -}; - -/** - * Downloads a package from the server. - * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. - */ -gpii.iod.downloadPackage = function (that, installation) { - fluid.log("IoD: Downloading package " + installation.packageInfo.url); + var packageInfo = Object.assign({}, packages[packageName]); var promise = fluid.promise(); - - installation.tempDir = that.getWorkingPath(installation.packageName); - installation.localPackage = path.join(installation.tempDir, installation.packageInfo.filename); - - fs.copyFile(installation.packageInfo.url, installation.localPackage, function (err) { - if (err) { - promise.reject({ - isError: true, - message: "Unable to download package" - }); - } else { - promise.resolve(installation); - } - }); - - return promise; -}; - -/** - * Checks that a downloaded package is ok. - * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. - */ -gpii.iod.checkPackage = function (that, installation) { - fluid.log("IoD: Checking downloaded package file " + installation.packageInfo.filename); - return installation; -}; - -/** - * Generate the installation instructions. - * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. - */ -gpii.iod.prepareInstall = function (that, installation) { - fluid.log("IoD: Preparing installation for " + installation.packageName); - - installation.installer = that.findInstaller(installation.packageInfo.packageType); - - var promise = fluid.promise(); - if (installation.installer) { - promise = installation.installer.prepareInstall(installation); + if (packageInfo) { + promise.resolve(packageInfo); } else { - promise = fluid.promise(); promise.reject({ isError: true, - error: "Unable to find a package installer for packageType '" + installation.packageInfo.packageType + "'" + error: "no such package: " + packageName }); } return promise; }; -/** - * Installs the package. - * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. - */ -gpii.iod.installPackage = function (that, installation) { - fluid.log("IoD: Installing package " + installation.packageInfo.filename); - return installation.installer.installPackage(installation); -}; - -/** - * Cleans up things that are no longer required. - * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. - */ -gpii.iod.cleanup = function (that, installation) { - fluid.log("IoD: Cleaning installation of " + installation.packageName); - - var result = installation.installer.cleanup(installation); - return fluid.toPromise(result).then(function () { - delete that.installations[installation]; - }); -}; - /** * Starts the package removal routine. * diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 1d8dde8ca..23c37b0b2 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -18,19 +18,162 @@ "use strict"; +var path = require("path"), + fs = require("fs"); + var fluid = require("infusion"); +var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.iod"); fluid.defaults("gpii.iod.packageInstaller", { gradeNames: ["fluid.component"], invokers: { - prepareInstall: "fluid.notImplemented", + startInstaller: { + funcName: "gpii.iod.startInstaller", + args: ["{that}", "{iod}", "{arguments}.0"] + }, + // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns + // a installation, either directly or via a promise. + downloadPackage: { + funcName: "gpii.iod.downloadPackage", + args: ["{that}", "{iod}"] + }, + checkPackage: { + funcName: "gpii.iod.checkPackage", + args: ["{that}", "{iod}"] + }, + prepareInstall: { + funcName: "gpii.iod.prepareInstall", + args: ["{that}", "{iod}"] + }, installPackage: "fluid.notImplemented", - cleanup: "fluid.notImplemented", + cleanup: { + funcName: "gpii.iod.cleanup", + args: ["{that}", "{iod}"] + }, uninstallPackage: "fluid.notImplemented" }, + events: { + // Dummy events for the installation pipe-lines + onInstallPackage: null, + onRemovePackage: null + }, + listeners: { + "onInstallPackage.download": { + func: "{that}.downloadPackage", + priority: "first" + }, + "onInstallPackage.check": { + func: "{that}.checkPackage", + priority: "after:download" + }, + "onInstallPackage.prepareInstall": { + func: "{that}.prepareInstall", + priority: "after:check" + }, + "onInstallPackage.install": { + func: "{that}.installPackage", + priority: "after:prepareInstall" + }, + "onInstallPackage.cleanup": { + func: "{that}.cleanup", + priority: "after:install" + }, + + "onRemovePackage.uninstallPackage": { + func: "{that}.uninstallPackage", + priority: "first" + } + }, - packageTypes: null + // Types of package this installer supports + packageTypes: null, + + members: { + // Package information from the server . + packageInfo: null, + // Where this installation will put it's stuff. + tempDir: null, + // Path of the downloaded package. + localPackage: null + } }); +/** + * Starts the installation pipeline. + * + * @param that {Component} The gpii.iod.installer instance. + * @param iod {object} The gpii.iod instance. + * @param packageInfo {object} The package info. + * @return {Promise} Resolves when complete. + */ +gpii.iod.startInstaller = function (that, iod, packageInfo) { + that.packageInfo = packageInfo; + that.tempDir = iod.getWorkingPath(that.packageInfo.name); + + var promise = fluid.promise.fireTransformEvent(that.events.onInstallPackage); + + return promise; +}; + +/** + * Downloads a package from the server. + * + * @param that {Component} The gpii.iod.installer instance. + * @param iod {object} The gpii.iod instance. + * @return {Promise} Resolves when complete. + */ +gpii.iod.downloadPackage = function (that) { + fluid.log("IoD: Downloading package " + that.packageInfo.url); + + var promise = fluid.promise(); + + that.localPackage = path.join(that.tempDir, that.packageInfo.filename); + + fs.copyFile(that.packageInfo.url, that.localPackage, function (err) { + if (err) { + promise.reject({ + isError: true, + message: "Unable to download package" + }); + } else { + promise.resolve(); + } + }); + + return promise; +}; + +/** + * Checks that a downloaded package is ok. + * + * @param that {Component} The gpii.iod.installer instance. + * @param iod {object} The gpii.iod instance. + * @return {Promise} Resolves when complete. + */ +gpii.iod.checkPackage = function (that) { + fluid.log("IoD: Checking downloaded package file " + that.packageInfo.filename); +}; + +/** + * Generate the installation instructions. + * + * @param that {Component} The gpii.iod.installer instance. + * @param iod {object} The gpii.iod instance. + * @return {Promise} Resolves when complete. + */ +gpii.iod.prepareInstall = function (that) { + fluid.log("IoD: Preparing installation for " + that.packageInfo.name); +}; + +/** + * Cleans up things that are no longer required. + * + * @param that {Component} The gpii.iod.installer instance. + * @param iod {object} The gpii.iod instance. + * @return {Promise} Resolves when complete. + */ +gpii.iod.cleanup = function (that) { + fluid.log("IoD: Cleaning installation of " + that.packageInfo.name); +}; From 281b8742f17ebe882642b84a2755bf44a0d7bad1 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 3 May 2018 09:45:35 +0100 Subject: [PATCH 03/56] GPII-2971: Data source for package info --- .../installOnDemand/src/installOnDemand.js | 52 +++++++++---------- .../installOnDemand/src/packageInstaller.js | 37 ++++++++++--- testData/installOnDemand/wget.json | 6 +++ 3 files changed, 61 insertions(+), 34 deletions(-) create mode 100644 testData/installOnDemand/wget.json diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index af087652a..ab3044d98 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -41,6 +41,18 @@ fluid.defaults("gpii.iod", { } } }, + components: { + "packageDataSource": { + type: "kettle.dataSource.file", + options: { + gradeNames: "kettle.dataSource.file.moduleTerms", + path: "%gpii-universal/testData/installOnDemand/%packageName.json", + termMap: { + packageName: "%packageName" + } + } + } + }, invokers: { requirePackage: { funcName: "gpii.iod.requirePackage", @@ -54,8 +66,8 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.startRemoval", args: ["{that}", "{arguments}.0"] }, - findInstaller: { - funcName: "gpii.iod.findInstaller", + getInstaller: { + funcName: "gpii.iod.getInstaller", args: ["{that}", "{arguments}.0"] }, getWorkingPath: { @@ -99,17 +111,19 @@ gpii.iod.getWorkingPath = function (packageName) { * * @param that {Component} The gpii.iod instance. * @param packageType {string} The package type identifier. - * @return {Component} A gpii.iod.installer component that handles the requested type of package. + * @return {Component} A new instance of the gpii.iod.installer component that handles the requested type of package. */ -gpii.iod.findInstaller = function (that, packageType) { +gpii.iod.getInstaller = function (that, packageType) { var packageInstallers = fluid.queryIoCSelector(that, "gpii.iod.packageInstaller"); - return fluid.find(packageInstallers, function (installer) { + var installerComponent = fluid.find(packageInstallers, function (installer) { var packageTypes = fluid.makeArray(installer.options.packageTypes); return packageTypes.indexOf(packageType) >= 0 ? installer : undefined; }); + + return installerComponent && fluid.invokeGlobalFunction(installerComponent.typeName); }; /** @@ -131,7 +145,7 @@ gpii.iod.requirePackage = function (that, packageName) { var promise = fluid.promise(); that.getPackageInfo(packageName).then(function (packageInfo) { - var installer = that.findInstaller(packageInfo.packageType); + var installer = that.getInstaller(packageInfo.packageType); if (installer) { var result = installer.startInstaller(packageInfo); fluid.promise.follow(result, promise); @@ -168,27 +182,11 @@ gpii.iod.requirePackage = function (that, packageName) { gpii.iod.getPackageInfo = function (that, packageName) { fluid.log("IoD: Getting package info for " + packageName); - var packages = { - "wget": { - name: "wget", - url: "e:\\Wget.1.19.4.nupkg", - //url: "https://chocolatey.org/api/v2/package/Wget/1.19.4", - filename: "Wget.1.19.4.nupkg", - packageType: "chocolatey" - } - }; - - var packageInfo = Object.assign({}, packages[packageName]); - var promise = fluid.promise(); - if (packageInfo) { + + that.packageDataSource.get({packageName: packageName}).then(function (packageInfo) { promise.resolve(packageInfo); - } else { - promise.reject({ - isError: true, - error: "no such package: " + packageName - }); - } + }, promise.reject); return promise; }; @@ -206,7 +204,7 @@ gpii.iod.startRemoval = function (that, installation) { } if (!installation.installer) { - installation.installer = that.findInstaller(installation.packageInfo.packageType); + installation.installer = that.getInstaller(installation.packageInfo.packageType); } fluid.log("IoD: Removing installation of " + installation.packageName); @@ -217,3 +215,5 @@ gpii.iod.startRemoval = function (that, installation) { gpii.iod.uninstallPackage = function (that, installation) { installation.installer.uninstallPackage(installation); }; + + diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 23c37b0b2..0cbd81db0 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -19,7 +19,8 @@ "use strict"; var path = require("path"), - fs = require("fs"); + fs = require("fs"), + https = require("https"); var fluid = require("infusion"); var gpii = fluid.registerNamespace("gpii"); @@ -131,16 +132,34 @@ gpii.iod.downloadPackage = function (that) { that.localPackage = path.join(that.tempDir, that.packageInfo.filename); - fs.copyFile(that.packageInfo.url, that.localPackage, function (err) { - if (err) { + if (that.packageInfo.url.startsWith("https://")) { + // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). + var output = fs.createWriteStream(that.localPackage); + output.on("finish", function () { + promise.resolve(); + }); + var req = https.get(that.packageInfo.url, function (response) { + response.pipe(output); + }); + req.on("error", function (err) { promise.reject({ isError: true, - message: "Unable to download package" + message: "Unable to download package:" + err.message, + error: err }); - } else { - promise.resolve(); - } - }); + }); + } else { + fs.copyFile(that.packageInfo.url, that.localPackage, function (err) { + if (err) { + promise.reject({ + isError: true, + message: "Unable to copy package" + }); + } else { + promise.resolve(); + } + }); + } return promise; }; @@ -154,6 +173,8 @@ gpii.iod.downloadPackage = function (that) { */ gpii.iod.checkPackage = function (that) { fluid.log("IoD: Checking downloaded package file " + that.packageInfo.filename); + // TODO: It shouldn't be checked here - another process may over-write it before the high privilege executes it. + // Instead, take ownership then check the integrity in the same context as it's being ran. }; /** diff --git a/testData/installOnDemand/wget.json b/testData/installOnDemand/wget.json new file mode 100644 index 000000000..884355a5b --- /dev/null +++ b/testData/installOnDemand/wget.json @@ -0,0 +1,6 @@ +{ + "name": "wget", + "url": "https://chocolatey.org/api/v2/package/Wget/1.19.4", + "filename": "Wget.1.19.4.nupkg", + "packageType": "chocolatey" +} From 1bd0a52755a0830e90b452a0f45fc67a8c3cb37d Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 4 May 2018 11:59:31 +0100 Subject: [PATCH 04/56] GPII-2971: tests for installOnDemand --- .../installOnDemand/src/installOnDemand.js | 139 ++++-- .../installOnDemand/src/packageInstaller.js | 13 +- .../test/installOnDemandTests.js | 472 +++++++++++++++++- .../test/testPackages/failInstall.json | 6 + .../test/testPackages/languages.json | 26 + .../test/testPackages/package1.json | 6 + .../test/testPackages/package2.json | 6 + .../test/testPackages/unknownType.json | 6 + 8 files changed, 638 insertions(+), 36 deletions(-) create mode 100644 gpii/node_modules/installOnDemand/test/testPackages/failInstall.json create mode 100644 gpii/node_modules/installOnDemand/test/testPackages/languages.json create mode 100644 gpii/node_modules/installOnDemand/test/testPackages/package1.json create mode 100644 gpii/node_modules/installOnDemand/test/testPackages/package2.json create mode 100644 gpii/node_modules/installOnDemand/test/testPackages/unknownType.json diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index ab3044d98..96506471c 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -43,10 +43,9 @@ fluid.defaults("gpii.iod", { }, components: { "packageDataSource": { - type: "kettle.dataSource.file", + type: "kettle.dataSource.URL", options: { - gradeNames: "kettle.dataSource.file.moduleTerms", - path: "%gpii-universal/testData/installOnDemand/%packageName.json", + url: "%gpii-universal/testData/installOnDemand/%packageName.json", termMap: { packageName: "%packageName" } @@ -83,27 +82,44 @@ fluid.defaults("gpii.iod", { /** * Create a directory where packages are temporarily stored. + * * @param packageName {String} Name of the package for which the directory is being created. + * @return {Object} Contains the full path (fullPath), and the first path that was created (createdPath), for cleanup */ gpii.iod.getWorkingPath = function (packageName) { + var createdPath = null; + + var parts = [ os.tmpdir(), "gpii-iod", - packageName && packageName.replace(/[^a-z0-9]/, "_"), + packageName && packageName.replace(/[^-a-z0-9]/, "_"), Math.random().toString(36) ]; - return parts.reduce(function (parent, child) { + // Create a new directory + var createDirectory = function (parent, child) { var dir = path.join(parent, child); try { fs.mkdirSync(dir); + if (!createdPath) { + createdPath = dir; + } } catch (e) { if (e.code !== "EEXIST") { throw e; } } return dir; - }, ""); + }; + + // Create the parents of the path. (mkdirp isn't used because the first non-existing path is required to be known) + var fullPath = parts.reduce(createDirectory, ""); + + return { + fullPath: fullPath, + createdPath: createdPath + }; }; /** @@ -130,24 +146,38 @@ gpii.iod.getInstaller = function (that, packageType) { * Starts the process of installing a package. * * @param that {Component} The gpii.iod instance. - * @param packageName {string} The package name. + * @param packageRequest {string|Object} Package name, or object containing packageName, language, version. + * @param packageRequest.packageName {string} Name of the package. + * @param packageRequest.version {string} Name of the package. + * @param packageRequest.language {string|string[]} Language. * @return {Promise} Resolves when the installation is complete. */ -gpii.iod.requirePackage = function (that, packageName) { - fluid.log("IoD: Requiring " + packageName); +gpii.iod.requirePackage = function (that, packageRequest) { + if (typeof(packageRequest) === "string") { + packageRequest = { + packageName: packageRequest + }; + } + + fluid.log("IoD: Requiring " + packageRequest.packageName); var installation = { - id: "aaa", - packageName: packageName + id: fluid.allocateGuid(), + packageName: packageRequest.packageName, + packageRequest: packageRequest }; that.installations[installation.id] = installation; var promise = fluid.promise(); - that.getPackageInfo(packageName).then(function (packageInfo) { - var installer = that.getInstaller(packageInfo.packageType); - if (installer) { - var result = installer.startInstaller(packageInfo); + // Get the package info. + that.getPackageInfo(packageRequest).then(function (packageInfo) { + // Create the installer instance. + installation.packageInfo = packageInfo; + installation.installer = that.getInstaller(packageInfo.packageType); + if (installation.installer) { + // Start the installer. + var result = installation.installer.startInstaller(installation); fluid.promise.follow(result, promise); } else { promise.reject({ @@ -155,20 +185,13 @@ gpii.iod.requirePackage = function (that, packageName) { error: "Unable to find an installer for package type " + packageInfo.packageTypes }); } - }); + }, promise.reject); return promise.then(function () { fluid.log("IoD: Installation complete"); }, function (err) { fluid.log("IoD: Installation failed:", err.error || err); - var installation = err.installation; - if (!installation) { - installation = that.installations[packageName]; - } - if (installation) { - installation.failed = true; - that.cleanup(installation); - } + installation.failed = true; }); }; @@ -176,21 +199,79 @@ gpii.iod.requirePackage = function (that, packageName) { * Retrieve the package metadata. * * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. + * @param packageRequest {Object} Containing packageName, language, version. + * @param packageRequest.packageName {string} Name of the package. + * @param packageRequest.version {string} [optional] Version. + * @param packageRequest.language {string} [optional] Language code with optional country code (en, en-US, es-ES). * @return {Promise} Resolves to an object containing package information. */ -gpii.iod.getPackageInfo = function (that, packageName) { - fluid.log("IoD: Getting package info for " + packageName); +gpii.iod.getPackageInfo = function (that, packageRequest) { + fluid.log("IoD: Getting package info for " + packageRequest.packageName); var promise = fluid.promise(); - that.packageDataSource.get({packageName: packageName}).then(function (packageInfo) { + that.packageDataSource.get({ + packageName: packageRequest.packageName, + language: packageRequest.language, + version: packageRequest.version + }).then(function (packageInfo) { + if (packageRequest.language && packageInfo.languages) { + // Merge the language-specific info. + var lang = gpii.iod.matchLanguage(Object.keys(packageInfo.languages), packageRequest.language); + if (lang) { + Object.assign(packageInfo, packageInfo.languages[lang]); + packageInfo.language = lang; + } + } + promise.resolve(packageInfo); - }, promise.reject); + }, function (err) { + promise.reject({ + isError: true, + message: "Unknown package " + packageRequest.packageName, + error: err + }); + }); return promise; }; +/** + * Finds the best language from a list of available languages, using the following priority: + * - Exact match with country code + * - Exact match without country code + * - First language, ignoring country code. + * + * @param languages {string[]} The list of available languages, with optional country code (en, en-US, es-ES) + * @param language {string} The preferred language. + * @return {string} The closest matching item from languages. + */ +gpii.iod.matchLanguage = function (languages, language) { + languages = fluid.makeArray(languages); + + // Exact match. + var index = languages.indexOf(language); + var match = index >= 0 && languages[index]; + + if (!match) { + var langCode = language.substr(0, 2); + // Language without country. + if (language.length > 2) { + index = languages.indexOf(language); + match = index >= 0 && languages[index]; + } + + if (!match) { + // Ignore the country. + match = languages.find(function (lang) { + return lang.substr(0, 2) === langCode; + }); + } + } + + return match; +}; + /** * Starts the package removal routine. * diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 0cbd81db0..78c02267e 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -97,7 +97,9 @@ fluid.defaults("gpii.iod.packageInstaller", { // Where this installation will put it's stuff. tempDir: null, // Path of the downloaded package. - localPackage: null + localPackage: null, + // Paths to remove on cleanup. + cleanupPaths: [] } }); @@ -106,12 +108,17 @@ fluid.defaults("gpii.iod.packageInstaller", { * * @param that {Component} The gpii.iod.installer instance. * @param iod {object} The gpii.iod instance. - * @param packageInfo {object} The package info. + * @param packageInfo {Object} The package info. * @return {Promise} Resolves when complete. */ gpii.iod.startInstaller = function (that, iod, packageInfo) { that.packageInfo = packageInfo; - that.tempDir = iod.getWorkingPath(that.packageInfo.name); + + + var tempDir = iod.getWorkingPath(that.packageInfo.name); + that.tempDir = tempDir.fullPath; + that.cleanupPaths.push(tempDir.createdPath); + var promise = fluid.promise.fireTransformEvent(that.events.onInstallPackage); diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index 3ec69ce89..cca23ed22 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -18,8 +18,13 @@ "use strict"; -var fluid = require("gpii-universal"); +var os = require("os"), + fs = require("fs"), + path = require("path"); +var fluid = require("gpii-universal"); +var kettle = fluid.require("kettle"); +kettle.loadTestingSupport(); var jqUnit = fluid.require("node-jqunit"); var gpii = fluid.registerNamespace("gpii"); @@ -29,11 +34,470 @@ require("../index.js"); jqUnit.module("gpii.tests.iod"); +gpii.tests.iod.getInstallerTests = fluid.freezeRecursive([ + { + packageType: "testPackageType1", + expect: "gpii.tests.iod.testInstaller1" + }, + { + packageType: "testPackageType2a", + expect: "gpii.tests.iod.testInstaller2" + }, + { + packageType: "testPackageType2b", + expect: "gpii.tests.iod.testInstaller2" + }, + { + // Fails at installation, not during initialisation. + packageType: "testFailPackageType", + expect: "gpii.tests.iod.testInstallerFail" + }, + { + packageType: "testPackageType-not-exist", + expect: undefined + } +]); + +gpii.tests.iod.getPackageInfoTests = fluid.freezeRecursive([ + { + id: "No matching package", + request: { + packageName: "package-not-exists" + }, + expect: "reject" + }, + { + id: "Single language package", + request: { + packageName: "package1" + }, + expect: require("./testPackages/package1.json") + }, + { + id: "Single language package, with language specified", + request: { + packageName: "package1", + language: "fr-FR" + }, + expect: require("./testPackages/package1.json") + }, + { + id: "Multi-language package, language not specified", + request: { + packageName: "languages" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, unknown language specified", + request: { + packageName: "languages", + language: "xx-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, unknown language, no country specified", + request: { + packageName: "languages", + language: "xx" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, full language specified", + request: { + packageName: "languages", + language: "es-ES" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es-es", + "language": "es-ES" + } + }, + { + id: "Multi-language package, full language specified 2", + request: { + packageName: "languages", + language: "es-MX" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es-mx", + "language": "es-MX" + } + }, + { + id: "Multi-language package, no country specified", + request: { + packageName: "languages", + language: "es" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es", + "language": "es" + } + }, + { + id: "Multi-language package, unknown country specified", + request: { + packageName: "languages", + language: "es-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es", + "language": "es" + } + }, + { + id: "Multi-language package, no country specified, no non-country package", + request: { + packageName: "languages", + language: "zh" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.zh-cn", + "language": "zh-CN" + } + }, + { + id: "Multi-language package, unknown country specified, no non-country package", + request: { + packageName: "languages", + language: "zh-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.zh-cn", + "language": "zh-CN" + } + } +]); + +gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ + { + packageRequest: "no-such-package", + expect: "reject" + }, + { + packageRequest: "unknownType", + expect: "reject" + }, + { + packageRequest: "package1", + expect: { + installer: "gpii.tests.iod.testInstaller1", + packageName: "package1" + } + }, + { + packageRequest: { + packageName: "package1" + }, + expect: { + installer: "gpii.tests.iod.testInstaller1", + packageName: "package1" + } + }, + { + packageRequest: { + packageName: "package2" + }, + expect: { + installer: "gpii.tests.iod.testInstaller2", + packageName: "package2" + } + }, + { + packageRequest: { + packageName: "languages", + language: "es-ES" + }, + expect: { + installer: "gpii.tests.iod.testInstaller1", + packageName: "languages" + } + }, + { + packageRequest: { + packageName: "languages", + language: "nl-NL" + }, + expect: { + installer: "gpii.tests.iod.testInstaller2", + packageName: "languages" + } + }, + { + packageRequest: "failInstall", + expect: "reject" + } +]); + +fluid.defaults("gpii.tests.iod", { + gradeNames: [ "gpii.iod" ], + components: { + "testInstaller1": { + type: "gpii.tests.iod.testInstaller1" + }, + "testInstaller2": { + type: "gpii.tests.iod.testInstaller2" + }, + "testInstallerFail": { + type: "gpii.tests.iod.testInstallerFail" + }, + "packageDataSource": { + type: "kettle.dataSource.file", + options: { + path: __dirname + "/testPackages/%packageName.json" + } + } + } +}); +fluid.defaults("gpii.tests.iod.testInstaller1", { + gradeNames: ["fluid.component", "gpii.iod.packageInstaller"], + + invokers: { + installPackage: "fluid.identity", + uninstallPackage: "fluid.identity", + startInstaller: { + funcName: "gpii.tests.iod.testInstaller1.startInstaller", + args: ["{that}", "{iod}", "{arguments}.0"] + } + }, + + packageTypes: "testPackageType1" +}); + +fluid.defaults("gpii.tests.iod.testInstaller2", { + gradeNames: [ "gpii.tests.iod.testInstaller1"], + packageTypes: ["testPackageType2a", "testPackageType2b"] +}); + +fluid.defaults("gpii.tests.iod.testInstallerFail", { + gradeNames: ["gpii.tests.iod.testInstaller1"], + testReject: true, + packageTypes: "testFailPackageType" +}); + +/** + * Test function for packageInstaller.startInstaller. + * @param that {Component} The gpii.tests.iod.testInstaller1 instance. + * @param iod {Component} The gpii.test.iod instance. + * @param packageInfo {object} The package to install. + * @return {Promise} A resolved promise. + */ +gpii.tests.iod.testInstaller1.startInstaller = function (that, iod, packageInfo) { + if (iod.startInstallerCalled) { + jqUnit.fail("startInstaller called twice"); + } + iod.startInstallerCalled = { + installer: that.typeName, + packageName: packageInfo.packageName + }; + + var promise = fluid.promise(); + if (that.options.testReject) { + promise.reject({ + isError: true, + error: "Test failure" + }); + } else { + promise.resolve(); + } + + return promise; +}; + + +jqUnit.test("test getWorkingPath", function () { + + var safeToRemove = false; + var packageName = "test" + Math.random().toString(36).substring(2); + var result = gpii.iod.getWorkingPath(packageName); + + jqUnit.assertTrue("getWorkingPath must return something", !!result); + jqUnit.assertEquals("fullPath must be a string", "string", typeof result.fullPath); + jqUnit.assertEquals("createdPath must be a string", "string", typeof result.createdPath); + + try { + jqUnit.assertNotEquals("fullPath must contain the package name", result.fullPath.indexOf(packageName)); + + var isParent = result.fullPath.startsWith(result.createdPath + path.sep); + jqUnit.assertTrue("The first created directory must be a parent of the full path", isParent); + + safeToRemove = isParent; + + var isTempDirParent = result.fullPath.startsWith(os.tmpdir()); + jqUnit.assertTrue("The path must be a subdirectory of the system's temporary directory", isTempDirParent); + + safeToRemove = safeToRemove && isTempDirParent; + + + // These two aren't supposed to be guaranteed, however using a random package name should have ensured this. + jqUnit.assertNotEquals("The first created directory must not be the full path", + result.createdPath, result.fullPath); + jqUnit.assertNotEquals("fullPath must contain the package name", result.fullPath.indexOf(packageName)); + + try { + var stats = fs.lstatSync(result.fullPath); + jqUnit.assertTrue("fullPath must be a directory", stats.isDirectory()); + } catch (e) { + fluid.log("Error checking the existence of result.fullPath"); + jqUnit.fail(e); + } + + var fullPathContents = fs.readdirSync(result.fullPath); + jqUnit.assertEquals("fullPath must be an empty directory", 0, fullPathContents.length); + + var createdPathContents = fs.readdirSync(result.createdPath); + jqUnit.assertEquals("createdPath must only contain a single file", 1, createdPathContents.length); + + } finally { + // Remove directories. If the test failed, the paths could point to anything. So, rimraf is not used which will + // ensure only the directories are removed, and any other content remain. + if (safeToRemove) { + var parts = result.fullPath.split(path.sep); + var tmpDir = os.tmpdir(); + while (parts.length > 0) { + var dir = parts.join(path.sep); + parts.pop(); + if ((dir === result.createdPath) || (dir === tmpDir)) { + break; + } else { + fs.rmdirSync(dir); + } + } + } + } +}); + +// Test getInstaller returns the correct installer +jqUnit.test("test getInstaller", function () { + + var tests = gpii.tests.iod.getInstallerTests; + + var iod = gpii.tests.iod(); + + fluid.each(tests, function (test) { + + var installer = iod.getInstaller(test.packageType); + + if (test.expect) { + jqUnit.assertEquals("getInstaller should return the correct installer for packageType=" + test.packageType, + test.expect, installer && installer.typeName); + } else { + jqUnit.assertFalse("getInstaller should return nothing for packageType=" + test.packageType, !!installer); + } + + if (installer) { + installer.destroy(); + } + }); +}); + +// Test getPackageInfo returns correct information +jqUnit.asyncTest("test getPackageInfo", function () { + + var tests = gpii.tests.iod.getPackageInfoTests; + jqUnit.expect(tests.length * 2); + + var iod = gpii.iod({ + components: { + "packageDataSource": { + type: "kettle.dataSource.file", + options: { + path: __dirname + "/testPackages/%packageName.json" + } + } + } + }); + + var testIndex = -1; + var nextTest = function () { + if (++testIndex >= tests.length) { + jqUnit.start(); + return; + } + + var test = tests[testIndex]; + var suffix = " - test:" + test.id; + + fluid.log("getPackage: " + test.request.packageName + ", " + test.request.language); + + var p = iod.getPackageInfo(test.request); + + jqUnit.assertTrue("getPackageInfo must return a promise" + suffix, fluid.isPromise(p)); + + p.then(function (packageInfo) { + delete packageInfo.languages; + jqUnit.assertDeepEq("packageInfo must match expected" + suffix, test.expect, packageInfo); + nextTest(); + }, function () { + jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); + nextTest(); + }); + + }; + + nextTest(); +}); + +// Test requirePackage correctly starts the installer. +jqUnit.asyncTest("test requirePackage", function () { + var tests = gpii.tests.iod.startInstallerTests; + jqUnit.expect(tests.length * 3); + var iod = gpii.tests.iod(); + + var testIndex = -1; + var nextTest = function () { + if (++testIndex >= tests.length) { + jqUnit.start(); + return; + } + + var test = tests[testIndex]; + var suffix = " - test:" + test.id; + + iod.startInstallerCalled = null; + var p = iod.requirePackage(test.packageRequest); -jqUnit.asyncTest("install tests", function () { + jqUnit.assertTrue("requirePackage must return a promise" + suffix, fluid.isPromise(p)); - var iod = gpii.iod(); + p.then(function () { + jqUnit.assertTrue("startInstaller must have been called" + suffix, !!iod.startInstallerCalled); + jqUnit.assertDeepEq("startInstaller must have been called correctly" + suffix, + test.expect, iod.startInstallerCalled); + nextTest(); + }, function () { + jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); + jqUnit.assert("balance the assert count"); + nextTest(); + }); - iod.requirePackage("wget"); + }; + nextTest(); }); diff --git a/gpii/node_modules/installOnDemand/test/testPackages/failInstall.json b/gpii/node_modules/installOnDemand/test/testPackages/failInstall.json new file mode 100644 index 000000000..2416326e9 --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/testPackages/failInstall.json @@ -0,0 +1,6 @@ +{ + "name": "failInstall", + "filename": "example.filename", + "url": "test://example", + "packageType": "testFailPackageType" +} diff --git a/gpii/node_modules/installOnDemand/test/testPackages/languages.json b/gpii/node_modules/installOnDemand/test/testPackages/languages.json new file mode 100644 index 000000000..cac67f445 --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/testPackages/languages.json @@ -0,0 +1,26 @@ +{ + "name": "languages", + "filename": "example.filename", + "packageType": "testPackageType1", + + "languages": { + "es": { + "filename": "file.es" + }, + "es-ES": { + "filename": "file.es-es" + }, + "es-MX": { + "filename": "file.es-mx" + }, + "zh-CN": { + "filename": "file.zh-cn" + }, + "zh-SG": { + "filename": "file.zh-sg" + }, + "nl-NL": { + "packageType": "testPackageType2a" + } + } +} diff --git a/gpii/node_modules/installOnDemand/test/testPackages/package1.json b/gpii/node_modules/installOnDemand/test/testPackages/package1.json new file mode 100644 index 000000000..42da7d502 --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/testPackages/package1.json @@ -0,0 +1,6 @@ +{ + "name": "package1", + "filename": "example.filename", + "url": "test://example", + "packageType": "testPackageType1" +} diff --git a/gpii/node_modules/installOnDemand/test/testPackages/package2.json b/gpii/node_modules/installOnDemand/test/testPackages/package2.json new file mode 100644 index 000000000..2507a5291 --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/testPackages/package2.json @@ -0,0 +1,6 @@ +{ + "name": "package1", + "filename": "example.filename", + "url": "test://example", + "packageType": "testPackageType2a" +} diff --git a/gpii/node_modules/installOnDemand/test/testPackages/unknownType.json b/gpii/node_modules/installOnDemand/test/testPackages/unknownType.json new file mode 100644 index 000000000..f032725ed --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/testPackages/unknownType.json @@ -0,0 +1,6 @@ +{ + "name": "unknownType", + "filename": "example.filename", + "url": "test://example", + "packageType": "unknown-package-type" +} From e84463ee0431190bd402eb70b6384babefaf2215 Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 4 May 2018 19:51:38 +0100 Subject: [PATCH 05/56] GPII-2971: packageInstaller tests --- .../installOnDemand/src/packageInstaller.js | 106 ++++++-- .../test/installOnDemandTests.js | 12 +- .../test/packageInstallerTests.js | 244 ++++++++++++++++++ 3 files changed, 328 insertions(+), 34 deletions(-) create mode 100644 gpii/node_modules/installOnDemand/test/packageInstallerTests.js diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 78c02267e..6c214bf45 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -20,7 +20,7 @@ var path = require("path"), fs = require("fs"), - https = require("https"); + request = require("request"); var fluid = require("infusion"); var gpii = fluid.registerNamespace("gpii"); @@ -36,6 +36,10 @@ fluid.defaults("gpii.iod.packageInstaller", { }, // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns // a installation, either directly or via a promise. + initialise: { + funcName: "gpii.iod.initialise", + args: ["{that}", "{iod}"] + }, downloadPackage: { funcName: "gpii.iod.downloadPackage", args: ["{that}", "{iod}"] @@ -61,9 +65,13 @@ fluid.defaults("gpii.iod.packageInstaller", { onRemovePackage: null }, listeners: { + "onInstallPackage.initialise": { + func: "{that}.initialise", + priority: "first" + }, "onInstallPackage.download": { func: "{that}.downloadPackage", - priority: "first" + priority: "after:initialise" }, "onInstallPackage.check": { func: "{that}.checkPackage", @@ -111,18 +119,23 @@ fluid.defaults("gpii.iod.packageInstaller", { * @param packageInfo {Object} The package info. * @return {Promise} Resolves when complete. */ -gpii.iod.startInstaller = function (that, iod, packageInfo) { - that.packageInfo = packageInfo; - +gpii.iod.startInstaller = function (that, iod, installation) { + that.installation = installation; + that.packageInfo = that.installation.packageInfo; + return fluid.promise.fireTransformEvent(that.events.onInstallPackage); +}; +/** + * Initialises the installation. + * + * @param that {Component} The gpii.iod.installer instance. + * @param iod {object} The gpii.iod instance. + * @return {Promise} Resolves when complete. + */ +gpii.iod.initialise = function (that, iod) { var tempDir = iod.getWorkingPath(that.packageInfo.name); that.tempDir = tempDir.fullPath; that.cleanupPaths.push(tempDir.createdPath); - - - var promise = fluid.promise.fireTransformEvent(that.events.onInstallPackage); - - return promise; }; /** @@ -141,20 +154,8 @@ gpii.iod.downloadPackage = function (that) { if (that.packageInfo.url.startsWith("https://")) { // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). - var output = fs.createWriteStream(that.localPackage); - output.on("finish", function () { - promise.resolve(); - }); - var req = https.get(that.packageInfo.url, function (response) { - response.pipe(output); - }); - req.on("error", function (err) { - promise.reject({ - isError: true, - message: "Unable to download package:" + err.message, - error: err - }); - }); + var downloadPromise = gpii.iod.httpsDownload(that.packageInfo.url, that.localPackage); + fluid.promise.follow(downloadPromise, promise); } else { fs.copyFile(that.packageInfo.url, that.localPackage, function (err) { if (err) { @@ -171,6 +172,63 @@ gpii.iod.downloadPackage = function (that) { return promise; }; +/** + * Downloads a file, trying extra hard to use only https. + * + * @param url {string} The remote uri. + * @param localPath {string} Destination path. + * @return {Promise} Resolves when done. + */ +gpii.iod.httpsDownload = function (url, localPath) { + var promise = fluid.promise(); + var output = fs.createWriteStream(localPath); + output.on("finish", function () { + promise.resolve(); + }); + + if (url.startsWith("https:")) { + var req = request.get({ + url: url, + strictSSL: true, + // Force https (and fail) if http is attempted. + httpModules: {"http:": require("https")}, + // Don't permit redirecting to non-https. + followRedirect: function (response) { + var allow = response.caseless.get("location").startsWith("https:"); + if (!allow) { + fluid.log("IoD: Denying non-https redirect"); + } + return allow; + } + }); + + req.on("error", function (err) { + promise.reject({ + isError: true, + message: "Unable to download package: " + err.message, + error: err + }); + }); + + req.on("response", function (response) { + if ((response.statusCode >= 300) && (response.statusCode < 400)) { + req.emit("error", { + message: "Redirect failed" + }); + } + }); + + req.pipe(output); + } else { + promise.reject({ + isError: true, + message: "IoD only supports HTTPS" + }); + } + + return promise; +}; + /** * Checks that a downloaded package is ok. * diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index cca23ed22..4d2e7db7f 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -276,6 +276,7 @@ fluid.defaults("gpii.tests.iod", { } } }); + fluid.defaults("gpii.tests.iod.testInstaller1", { gradeNames: ["fluid.component", "gpii.iod.packageInstaller"], @@ -424,16 +425,7 @@ jqUnit.asyncTest("test getPackageInfo", function () { var tests = gpii.tests.iod.getPackageInfoTests; jqUnit.expect(tests.length * 2); - var iod = gpii.iod({ - components: { - "packageDataSource": { - type: "kettle.dataSource.file", - options: { - path: __dirname + "/testPackages/%packageName.json" - } - } - } - }); + var iod = gpii.tests.iod(); var testIndex = -1; var nextTest = function () { diff --git a/gpii/node_modules/installOnDemand/test/packageInstallerTests.js b/gpii/node_modules/installOnDemand/test/packageInstallerTests.js new file mode 100644 index 000000000..d59147d32 --- /dev/null +++ b/gpii/node_modules/installOnDemand/test/packageInstallerTests.js @@ -0,0 +1,244 @@ +/* + * IoD Tests. + * + * Copyright 2018 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var os = require("os"), + fs = require("fs"), + path = require("path"), + crypto = require("crypto"); + +var fluid = require("gpii-universal"); +var kettle = fluid.require("kettle"); +kettle.loadTestingSupport(); +var jqUnit = fluid.require("node-jqunit"); +var gpii = fluid.registerNamespace("gpii"); + +fluid.registerNamespace("gpii.tests.iod.installer"); + +require("../index.js"); + +gpii.tests.iod.teardowns = []; + +jqUnit.module("gpii.tests.iod.installer", { + teardown: function () { + while (gpii.tests.iod.teardowns.length) { + gpii.tests.iod.teardowns.pop()(); + } + } +}); + +fluid.defaults("gpii.tests.iod", { + gradeNames: [ "gpii.iod" ], + components: { + "testInstaller": { + type: "gpii.tests.iod.installer" + }, + "packageDataSource": { + type: "kettle.dataSource.file", + options: { + path: __dirname + "/testPackages/%packageName.json" + } + } + } +}); + +fluid.defaults("gpii.tests.iod.installer", { + gradeNames: ["fluid.component", "gpii.iod.packageInstaller"], + + invokers: { + initialise: "gpii.tests.iod.installer.stage({that}, initialise)", + downloadPackage: "gpii.tests.iod.installer.stage({that}, downloadPackage)", + checkPackage: "gpii.tests.iod.installer.stage({that}, checkPackage)", + prepareInstall: "gpii.tests.iod.installer.stage({that}, prepareInstall)", + installPackage: "gpii.tests.iod.installer.stage({that}, installPackage)", + cleanup: "gpii.tests.iod.installer.stage({that}, cleanup)", + uninstallPackage: "gpii.tests.iod.installer.stage({that}, uninstallPackage)" + }, + + packageTypes: "testPackageType1" +}); + +gpii.tests.iod.installer.stage = function (that, stage) { + that.stages.push(stage); +}; + +// Test startInstaller starts the installation pipe-line. +jqUnit.test("test getInstaller", function () { + + var iod = gpii.tests.iod(); + var installer = iod.getInstaller("testPackageType1"); + + installer.stages = []; + + installer.startInstaller({}).then(function () { + var expect = [ + "initialise", + "downloadPackage", + "checkPackage", + "prepareInstall", + "installPackage", + "cleanup" + ]; + + jqUnit.assertDeepEq("All stages of the installation should be called in order.", expect, installer.stages); + + }, jqUnit.fail); +}); + + +jqUnit.asyncTest("test https download", function () { + + gpii.tests.iod.installer.downloadTests = fluid.freezeRecursive([ + { + url: "https://raw.githubusercontent.com/GPII/universal/108be0f5f0377eaec9100c1926647e7550efc2ea/gpii.js", + expect: "8cb82683c931e15995b2573fda41c41eaacab59e" + }, + { + url: "https://gpii-test.invalid", + expect: "reject" + }, + // Certificate problems + { + url: "https://badssl.com", + expect: "resolve" + }, + { + url: "https://expired.badssl.com/", + expect: "reject" + }, + { + url: "https://wrong.host.badssl.com/", + expect: "reject" + }, + { + url: "https://self-signed.badssl.com/", + expect: "reject" + }, + { + url: "https://untrusted-root.badssl.com/", + expect: "reject" + }, + // Prohibited ciphers + { + url: "https://rc4-md5.badssl.com/", + expect: "reject" + }, + { + url: "https://rc4.badssl.com/", + expect: "reject" + }, + { + url: "https://3des.badssl.com/", + expect: "reject" + }, + { + url: "https://null.badssl.com/", + expect: "reject" + }, + // HTTP + { + // This redirects to http + url: "https://http.badssl.com/", + expect: "reject" + }, + { + url: "http://http.badssl.com/", + expect: "reject" + }, + { + // Unopened port (hopefully) + url: "https://127.0.0.1:51749", + expect: "reject" + } + ]); + + var filePrefix = path.join(os.tmpdir(), "gpii-test-download" + Math.random().toString(36) + "-"); + + var files = []; + // Remove all temporary files. + gpii.tests.iod.teardowns.push(function () { + fluid.each(files, function (file) { + try { + fs.unlinkSync(file); + } catch (e) { + // ignore. + } + }); + }); + + + var tests = gpii.tests.iod.installer.downloadTests; + jqUnit.expect(tests.length * 3); + + var testIndex = -1; + var nextTest = function () { + if (++testIndex >= tests.length) { + jqUnit.start(); + return; + } + + var test = tests[testIndex]; + var suffix = " - test " + testIndex + "(" + test.url + ")"; + + var outFile = filePrefix + testIndex; + files.push(outFile); + + var p = gpii.iod.httpsDownload(test.url, outFile); + + jqUnit.assertTrue("httpsDownload must return a promise" + suffix, fluid.isPromise(p)); + + p.then(function () { + jqUnit.assertNotEquals("httpsDownload must only succeed if expected" + suffix, test.expect, "reject"); + + if (test.expect === "resolve") { + jqUnit.assert("resolved"); + nextTest(); + } else if (test.expect !== "reject") { + var input = fs.createReadStream(outFile); + var hash = crypto.createHash("sha1"); + input.on("readable", function () { + var buffer = input.read(); + if (buffer) { + hash.update(buffer); + } else { + var digest = hash.digest("hex"); + jqUnit.assertEquals("Hash of download must be correct", test.expect, digest); + nextTest(); + } + }); + + input.on("error", function (err) { + fluid.log(err); + jqUnit.fail(err); + }); + } + }, function (err) { + jqUnit.assertEquals("httpsDownload must only reject if expected" + suffix, test.expect, "reject"); + jqUnit.assert("Balancing the expected assert count"); + if (test.expects !== "reject") { + fluid.log(err); + } + nextTest(); + }); + + }; + + nextTest(); +}); + From e9b2ba27f2e03ed510338626ce7ba5ecacc8ee98 Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 8 May 2018 10:18:47 +0100 Subject: [PATCH 06/56] GPII-2972: Client + IoD connectivity. --- .../flowManager/src/FlowManager.js | 6 + .../configs/gpii.iod.config.base.json | 3 + .../configs/gpii.iod.config.development.json | 7 ++ .../configs/gpii.iod.config.local.base.json | 24 ++++ .../configs/gpii.iod.config.remote.base.json | 21 ++++ gpii/node_modules/installOnDemand/index.js | 4 + .../installOnDemand/src/installOnDemand.js | 117 ++++++++++++++---- package.json | 1 + testData/installOnDemand/local.json | 6 + 9 files changed, 163 insertions(+), 26 deletions(-) create mode 100644 gpii/node_modules/installOnDemand/configs/gpii.iod.config.base.json create mode 100644 gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json create mode 100644 gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json create mode 100644 gpii/node_modules/installOnDemand/configs/gpii.iod.config.remote.base.json create mode 100644 testData/installOnDemand/local.json diff --git a/gpii/node_modules/flowManager/src/FlowManager.js b/gpii/node_modules/flowManager/src/FlowManager.js index e5631dd64..f2bc1c484 100644 --- a/gpii/node_modules/flowManager/src/FlowManager.js +++ b/gpii/node_modules/flowManager/src/FlowManager.js @@ -107,6 +107,12 @@ fluid.defaults("gpii.flowManager.local", { options: { gradeNames: ["gpii.userListeners"] } + }, + installOnDemand: { + type: "gpii.iod", + options: { + gradeNames: ["gpii.iod"] + } } }, requestHandlers: { diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.base.json b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.base.json new file mode 100644 index 000000000..57530923c --- /dev/null +++ b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.base.json @@ -0,0 +1,3 @@ +{ + "type": "gpii.iod.config.base" +} diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json new file mode 100644 index 000000000..fc4346036 --- /dev/null +++ b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json @@ -0,0 +1,7 @@ +{ + "type": "gpii.iod.config.development", + "mergeConfigs": [ + "./gpii.iod.config.local.base.json", + "./gpii.iod.config.remote.base.json" + ] +} diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json new file mode 100644 index 000000000..f20ae8da5 --- /dev/null +++ b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json @@ -0,0 +1,24 @@ +{ + "type": "gpii.iod.config.local.base", + "options": { + "distributeOptions": { + "packageData.local": { + "record": { + "gradeNames": "kettle.dataSource.file", + "path": "%gpii-universal/testData/installOnDemand/%packageName.json", + "termMap": { + "packageName": "%packageName" + } + }, + "target": "{that iod packageDataFallback}.options" + }, + "packageData.moduleTerms": { + "record": "kettle.dataSource.file.moduleTerms", + "target": "{that iod packageDataFallback}.options.gradeNames" + } + } + }, + "mergeConfigs": [ + "./gpii.iod.config.base.json" + ] +} diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.remote.base.json b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.remote.base.json new file mode 100644 index 000000000..1753357b8 --- /dev/null +++ b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.remote.base.json @@ -0,0 +1,21 @@ +{ + "type": "gpii.iod.config.remote.base", + "options": { + "distributeOptions": { + "packageData.remote": { + "record": { + "gradeNames": "kettle.dataSource.URL", + "url": "%endpoint/packages/%packageName", + "termMap": { + "packageName": "%packageName", + "endpoint": "noencode:%endpoint" + } + }, + "target": "{that iod packageData}.options" + } + } + }, + "mergeConfigs": [ + "./gpii.iod.config.base.json" + ] +} diff --git a/gpii/node_modules/installOnDemand/index.js b/gpii/node_modules/installOnDemand/index.js index 73345c021..f002bacb3 100644 --- a/gpii/node_modules/installOnDemand/index.js +++ b/gpii/node_modules/installOnDemand/index.js @@ -18,5 +18,9 @@ "use strict"; +var fluid = require("infusion"); + +fluid.module.register("installOnDemand", __dirname, require); + require("./src/installOnDemand.js"); require("./src/packageInstaller.js"); diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 96506471c..2297049de 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -42,17 +42,26 @@ fluid.defaults("gpii.iod", { } }, components: { - "packageDataSource": { - type: "kettle.dataSource.URL", - options: { - url: "%gpii-universal/testData/installOnDemand/%packageName.json", - termMap: { - packageName: "%packageName" - } - } + "packageData": { + createOnEvent: "onServiceFound", + type: "gpii.iod.packageDataSource" + }, + "packageDataFallback": { + type: "gpii.iod.packageDataSource" } }, + events: { + onServiceFound: null, + onServiceLost: null + }, + listeners: { + "onCreate.discoverServer": "{that}.discoverServer" + }, invokers: { + discoverServer: { + funcName: "gpii.iod.discoverServer", + args: ["{that}"] + }, requirePackage: { funcName: "gpii.iod.requirePackage", args: ["{that}", "{arguments}.0"] @@ -76,10 +85,16 @@ fluid.defaults("gpii.iod", { }, members: { - installations: {} + installations: {}, + endpoint: "http://gpii-iod:8087" } }); +fluid.defaults("gpii.iod.packageDataSource", { + gradeNames: ["fluid.component"], + readOnlyGrade: "gpii.iod.packageDataSource" +}); + /** * Create a directory where packages are temporarily stored. * @@ -210,28 +225,38 @@ gpii.iod.getPackageInfo = function (that, packageRequest) { var promise = fluid.promise(); - that.packageDataSource.get({ - packageName: packageRequest.packageName, - language: packageRequest.language, - version: packageRequest.version - }).then(function (packageInfo) { - if (packageRequest.language && packageInfo.languages) { - // Merge the language-specific info. - var lang = gpii.iod.matchLanguage(Object.keys(packageInfo.languages), packageRequest.language); - if (lang) { - Object.assign(packageInfo, packageInfo.languages[lang]); - packageInfo.language = lang; + var dataSource = that.packageData || (that.packageDataFallback.options.path && that.packageDataFallback); + + if (dataSource) { + dataSource.get({ + packageName: packageRequest.packageName, + language: packageRequest.language, + version: packageRequest.version, + server: that.remoteServer + }).then(function (packageInfo) { + if (packageRequest.language && packageInfo.languages) { + // Merge the language-specific info. + var lang = gpii.iod.matchLanguage(Object.keys(packageInfo.languages), packageRequest.language); + if (lang) { + Object.assign(packageInfo, packageInfo.languages[lang]); + packageInfo.language = lang; + } } - } - promise.resolve(packageInfo); - }, function (err) { + promise.resolve(packageInfo); + }, function (err) { + promise.reject({ + isError: true, + message: "Unknown package " + packageRequest.packageName, + error: err + }); + }); + } else { promise.reject({ isError: true, - message: "Unknown package " + packageRequest.packageName, - error: err + message: "No package data source for IoD" }); - }); + } return promise; }; @@ -293,8 +318,48 @@ gpii.iod.startRemoval = function (that, installation) { return promise; }; +/** + * Uninstall a package. + * + * @param that {Component} The gpii.iod instance. + * @param installation {object} The installation state. + */ gpii.iod.uninstallPackage = function (that, installation) { installation.installer.uninstallPackage(installation); }; +/** + * Discovers the IoD server. + * + * @param that + */ +gpii.iod.discoverServer = function (that) { + + var addr = process.env.GPII_IOD_ENDPOINT; + + if (!addr) { + var bonjour = that.bonjourInstance || (that.bonjourInstance = require("bonjour")()); + if (bonjour) { + var b = bonjour.find({type: "gpii-iod"}); + b.on("up", function (service) { + that.endpoint = "https://" + service.host + ":" + service.port; + that.endpointService = service.fqdn; + fluid.log("IoD: Found service '" + that.endpointService + "': " + that.endpoint); + that.events.onServiceFound.fire(); + }); + + b.on("down", function (service) { + if (that.endpoint && that.endpointService === service.fqdn) { + fluid.log("IoD: Lost service '" + that.endpointService + "': " + that.endpoint); + that.endpoint = null; + that.endpointService = null; + that.events.onServiceLost.fire(); + that.packageData.destroy(); + } + }); + } + } + + that.endpoint = addr; +}; diff --git a/package.json b/package.json index b7f172d36..baf4f8e39 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "homepage": "http://gpii.net/", "dependencies": { "body-parser": "1.18.2", + "bonjour": "3.5", "connect-ensure-login": "0.1.1", "express": "4.16.2", "express-handlebars": "3.0.0", diff --git a/testData/installOnDemand/local.json b/testData/installOnDemand/local.json new file mode 100644 index 000000000..0156da2fe --- /dev/null +++ b/testData/installOnDemand/local.json @@ -0,0 +1,6 @@ +{ + "name": "dummy-local-package", + "url": "https://gpii.invalid", + "filename": "dummy-local-package", + "packageType": "dummy" +} From 46943c17357d6a1bdcfccdda8515bbe1bbd96459 Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 8 May 2018 13:15:50 +0100 Subject: [PATCH 07/56] GPII-2972: Improved IoD server detection. --- .../gpii.config.development.base.local.json | 3 +- gpii/node_modules/installOnDemand/README.md | 34 ++++-- .../configs/gpii.iod.config.development.json | 10 ++ .../configs/gpii.iod.config.local.base.json | 4 +- .../installOnDemand/src/installOnDemand.js | 102 ++++++++++++++---- 5 files changed, 125 insertions(+), 28 deletions(-) diff --git a/gpii/configs/gpii.config.development.base.local.json b/gpii/configs/gpii.config.development.base.local.json index 4e25e7f4b..86df8f408 100644 --- a/gpii/configs/gpii.config.development.base.local.json +++ b/gpii/configs/gpii.config.development.base.local.json @@ -12,6 +12,7 @@ "%flowManager/configs/gpii.flowManager.config.development.json", "%preferencesServer/configs/gpii.preferencesServer.config.development.json", "%canopyMatchMaker/configs/gpii.canopyMatchMaker.config.base.json", - "%rawPreferencesServer/configs/gpii.rawPreferencesServer.config.development.json" + "%rawPreferencesServer/configs/gpii.rawPreferencesServer.config.development.json", + "%installOnDemand/configs/gpii.iod.config.development.json" ] } diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/installOnDemand/README.md index 6860bfb3a..42a1251f7 100644 --- a/gpii/node_modules/installOnDemand/README.md +++ b/gpii/node_modules/installOnDemand/README.md @@ -2,12 +2,15 @@ Provides the ability to install software on demand. -## `gpii.iod` component - The stages of installation: -### getInfo -Retrieves the package information (eg download location, installation instructions) +### start + +* Gets the package info from the server. +* `requirePackage` finds a suitable `gpii.iod.packageInstaller` for the package. + +### initialise +Creates a temporary directory, starts the following pipeline. ### download Downloads the package. @@ -18,8 +21,6 @@ Checks the downloaded package. ### prepareInstall Generates the installation commands. -Finds the `packageInstaller` component that handles this type of package. - ### install Installs the package. @@ -27,6 +28,25 @@ Installs the package. Cleans the files. -## `gpii.iod.packageInstaller` +## Parts + +### `gpii.iod` + +The install on demand component. + +### `gpii.iod.packageInstaller` Base component of the package installers, which perform the work that's specific to the type of package being installed. + +Implementations will probably be found in the OS-specific repository. + +### `gpii.iod.packageDataSource` + +The package data source. + +## IoD Server + +Using the development config, GPII will provide package data from [testData/installOnDemand](testData/installOnDemand). + +But, if it detects a running instance of the server [stegru/gpii-iod](https://github.com/stegru/gpii-iod) somewhere on +the network then that will be used instead. diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json index fc4346036..35ec9612a 100644 --- a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json +++ b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json @@ -1,5 +1,15 @@ { "type": "gpii.iod.config.development", + "options": { + "distributeOptions": { + "iod.local": { + "record": { + "defaultEndpoint": "http://localhost:8087" + }, + "target": "{that iod}.options" + } + } + }, "mergeConfigs": [ "./gpii.iod.config.local.base.json", "./gpii.iod.config.remote.base.json" diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json index f20ae8da5..5cf731203 100644 --- a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json +++ b/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json @@ -2,7 +2,7 @@ "type": "gpii.iod.config.local.base", "options": { "distributeOptions": { - "packageData.local": { + "packageDataFallback.file": { "record": { "gradeNames": "kettle.dataSource.file", "path": "%gpii-universal/testData/installOnDemand/%packageName.json", @@ -12,7 +12,7 @@ }, "target": "{that iod packageDataFallback}.options" }, - "packageData.moduleTerms": { + "packageDataFallback.moduleTerms": { "record": "kettle.dataSource.file.moduleTerms", "target": "{that iod packageDataFallback}.options.gradeNames" } diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 2297049de..73ec24383 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -22,7 +22,8 @@ var fluid = require("infusion"); var path = require("path"), os = require("os"), - fs = require("fs"); + fs = require("fs"), + request = require("request"); var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.iod"); @@ -55,7 +56,9 @@ fluid.defaults("gpii.iod", { onServiceLost: null }, listeners: { - "onCreate.discoverServer": "{that}.discoverServer" + "onCreate.discoverServer": "{that}.discoverServer", + "onServiceFound": "{that}.serviceFound", + "onServiceLost": "{that}.serviceLost" }, invokers: { discoverServer: { @@ -81,12 +84,19 @@ fluid.defaults("gpii.iod", { getWorkingPath: { funcName: "gpii.iod.getWorkingPath", args: ["{arguments}.0"] + }, + serviceFound: { + funcName: "gpii.iod.serviceFound", + args: ["{that}", "{arguments}.0"] + }, + serviceLost: { + funcName: "gpii.iod.serviceLost", + args: ["{that}", "{arguments}.0"] } }, members: { - installations: {}, - endpoint: "http://gpii-iod:8087" + installations: {} } }); @@ -331,35 +341,91 @@ gpii.iod.uninstallPackage = function (that, installation) { /** * Discovers the IoD server. * - * @param that + * @param that {Component} The gpii.iod instance. */ gpii.iod.discoverServer = function (that) { var addr = process.env.GPII_IOD_ENDPOINT; - if (!addr) { + if (addr) { + gpii.iod.checkService(addr).then(that.events.onServiceFound.fire); + } else { var bonjour = that.bonjourInstance || (that.bonjourInstance = require("bonjour")()); if (bonjour) { - var b = bonjour.find({type: "gpii-iod"}); - b.on("up", function (service) { - that.endpoint = "https://" + service.host + ":" + service.port; - that.endpointService = service.fqdn; - fluid.log("IoD: Found service '" + that.endpointService + "': " + that.endpoint); - that.events.onServiceFound.fire(); + var browser = bonjour.find({type: "gpii-iod"}); + browser.on("up", function (service) { + fluid.log("IoD: Service up: " + service.fqdn); + if (that.endpoint && that.packageData) { + that.events.onServiceLost.fire(that.endpoint); + that.packageData.destroy(); + } + var endpoint = service.txt.url || ("https://" + service.host + ":" + service.port); + gpii.iod.checkService(endpoint).then(that.events.onServiceFound.fire); }); - b.on("down", function (service) { + browser.on("down", function (service) { if (that.endpoint && that.endpointService === service.fqdn) { - fluid.log("IoD: Lost service '" + that.endpointService + "': " + that.endpoint); - that.endpoint = null; - that.endpointService = null; - that.events.onServiceLost.fire(); - that.packageData.destroy(); + fluid.log("IoD: Service down: " + service.fqdn); + var oldEndpoint = service.txt.url || ("https://" + service.host + ":" + service.port); + if (oldEndpoint === that.endpoint) { + that.events.onServiceLost.fire(that.endpoint); + that.packageData.destroy(); + } } }); + // After a timeout use the default endpoint (if configured) + if (that.options.defaultEndpoint) { + setTimeout(function () { + if (!that.endpoint) { + fluid.log("IoD: No endpoint detected, trying " + that.options.defaultEndpoint); + gpii.iod.checkService(that.options.defaultEndpoint).then(that.events.onServiceFound.fire); + } + }, 5000); + } } } that.endpoint = addr; }; + +/** + * Check if an endpoint is listening for connections. + * + * @param endpoint + * @return {Promise} + */ +gpii.iod.checkService = function (endpoint) { + var promise = fluid.promise(); + request(endpoint, function (error, response) { + if (response) { + promise.resolve(endpoint); + } else { + fluid.log("IoD: Unable to connect to endpoint " + endpoint); + promise.reject(error); + } + }); + return promise; +}; + +/** + * Invoked when the service endpoint is down. + * + * @param that {Component} The gpii.iod instance. + * @param endPoint {string} The endpoint address. + */ +gpii.iod.serviceLost = function (that, endPoint) { + fluid.log("IoD: Endpoint lost: " + endPoint); + that.endpoint = null; +}; + +/** + * Invoked when a service endpoint is up. + * + * @param that {Component} The gpii.iod instance. + * @param endPoint {string} The endpoint address. + */ +gpii.iod.serviceFound = function (that, endPoint) { + fluid.log("IoD: Endpoint found: " + endPoint); + that.endpoint = endPoint; +}; From 4839108b9fcc84b0fd3f6125572989a12e08569d Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 8 May 2018 20:22:24 +0100 Subject: [PATCH 08/56] GPII-2972: Proper jsdoc. --- .../installOnDemand/src/installOnDemand.js | 36 +++++++++---------- .../installOnDemand/src/packageInstaller.js | 30 ++++++++-------- .../test/installOnDemandTests.js | 6 ++-- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 73ec24383..a785c5e67 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -108,7 +108,7 @@ fluid.defaults("gpii.iod.packageDataSource", { /** * Create a directory where packages are temporarily stored. * - * @param packageName {String} Name of the package for which the directory is being created. + * @param {String} packageName Name of the package for which the directory is being created. * @return {Object} Contains the full path (fullPath), and the first path that was created (createdPath), for cleanup */ gpii.iod.getWorkingPath = function (packageName) { @@ -150,8 +150,8 @@ gpii.iod.getWorkingPath = function (packageName) { /** * Finds a package installer component that handles the given type of package. * - * @param that {Component} The gpii.iod instance. - * @param packageType {string} The package type identifier. + * @param {Component} that The gpii.iod instance. + * @param {string} packageType The package type identifier. * @return {Component} A new instance of the gpii.iod.installer component that handles the requested type of package. */ gpii.iod.getInstaller = function (that, packageType) { @@ -170,8 +170,8 @@ gpii.iod.getInstaller = function (that, packageType) { /** * Starts the process of installing a package. * - * @param that {Component} The gpii.iod instance. - * @param packageRequest {string|Object} Package name, or object containing packageName, language, version. + * @param {Component} that The gpii.iod instance. + * @param {string|Object} packageRequest Package name, or object containing packageName, language, version. * @param packageRequest.packageName {string} Name of the package. * @param packageRequest.version {string} Name of the package. * @param packageRequest.language {string|string[]} Language. @@ -223,8 +223,8 @@ gpii.iod.requirePackage = function (that, packageRequest) { /** * Retrieve the package metadata. * - * @param that {Component} The gpii.iod instance. - * @param packageRequest {Object} Containing packageName, language, version. + * @param {Component} that The gpii.iod instance. + * @param {Object} packageRequest Containing packageName, language, version. * @param packageRequest.packageName {string} Name of the package. * @param packageRequest.version {string} [optional] Version. * @param packageRequest.language {string} [optional] Language code with optional country code (en, en-US, es-ES). @@ -277,8 +277,8 @@ gpii.iod.getPackageInfo = function (that, packageRequest) { * - Exact match without country code * - First language, ignoring country code. * - * @param languages {string[]} The list of available languages, with optional country code (en, en-US, es-ES) - * @param language {string} The preferred language. + * @param {string[]} languages The list of available languages, with optional country code (en, en-US, es-ES) + * @param {string} language The preferred language. * @return {string} The closest matching item from languages. */ gpii.iod.matchLanguage = function (languages, language) { @@ -310,8 +310,8 @@ gpii.iod.matchLanguage = function (languages, language) { /** * Starts the package removal routine. * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. + * @param {Component} that The gpii.iod instance. + * @param {object} installation The installation state. * @return {Promise} Resolves to an object containing package information and installation state. */ gpii.iod.startRemoval = function (that, installation) { @@ -331,8 +331,8 @@ gpii.iod.startRemoval = function (that, installation) { /** * Uninstall a package. * - * @param that {Component} The gpii.iod instance. - * @param installation {object} The installation state. + * @param {Component} that The gpii.iod instance. + * @param {object} installation The installation state. */ gpii.iod.uninstallPackage = function (that, installation) { installation.installer.uninstallPackage(installation); @@ -341,7 +341,7 @@ gpii.iod.uninstallPackage = function (that, installation) { /** * Discovers the IoD server. * - * @param that {Component} The gpii.iod instance. + * @param {Component} that The gpii.iod instance. */ gpii.iod.discoverServer = function (that) { @@ -411,8 +411,8 @@ gpii.iod.checkService = function (endpoint) { /** * Invoked when the service endpoint is down. * - * @param that {Component} The gpii.iod instance. - * @param endPoint {string} The endpoint address. + * @param {Component} that The gpii.iod instance. + * @param {string} endPoint The endpoint address. */ gpii.iod.serviceLost = function (that, endPoint) { fluid.log("IoD: Endpoint lost: " + endPoint); @@ -422,8 +422,8 @@ gpii.iod.serviceLost = function (that, endPoint) { /** * Invoked when a service endpoint is up. * - * @param that {Component} The gpii.iod instance. - * @param endPoint {string} The endpoint address. + * @param {Component} that The gpii.iod instance. + * @param {string} endPoint The endpoint address. */ gpii.iod.serviceFound = function (that, endPoint) { fluid.log("IoD: Endpoint found: " + endPoint); diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 6c214bf45..2861d4a69 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -114,9 +114,9 @@ fluid.defaults("gpii.iod.packageInstaller", { /** * Starts the installation pipeline. * - * @param that {Component} The gpii.iod.installer instance. - * @param iod {object} The gpii.iod instance. - * @param packageInfo {Object} The package info. + * @param {Component} that The gpii.iod.installer instance. + * @param {object} iod The gpii.iod instance. + * @param {Object} packageInfo The package info. * @return {Promise} Resolves when complete. */ gpii.iod.startInstaller = function (that, iod, installation) { @@ -128,8 +128,8 @@ gpii.iod.startInstaller = function (that, iod, installation) { /** * Initialises the installation. * - * @param that {Component} The gpii.iod.installer instance. - * @param iod {object} The gpii.iod instance. + * @param {Component} that The gpii.iod.installer instance. + * @param {object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.initialise = function (that, iod) { @@ -141,8 +141,8 @@ gpii.iod.initialise = function (that, iod) { /** * Downloads a package from the server. * - * @param that {Component} The gpii.iod.installer instance. - * @param iod {object} The gpii.iod instance. + * @param {Component} that The gpii.iod.installer instance. + * @param {object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.downloadPackage = function (that) { @@ -175,8 +175,8 @@ gpii.iod.downloadPackage = function (that) { /** * Downloads a file, trying extra hard to use only https. * - * @param url {string} The remote uri. - * @param localPath {string} Destination path. + * @param {string} url The remote uri. + * @param {string} localPath Destination path. * @return {Promise} Resolves when done. */ gpii.iod.httpsDownload = function (url, localPath) { @@ -232,8 +232,8 @@ gpii.iod.httpsDownload = function (url, localPath) { /** * Checks that a downloaded package is ok. * - * @param that {Component} The gpii.iod.installer instance. - * @param iod {object} The gpii.iod instance. + * @param {Component} that The gpii.iod.installer instance. + * @param {object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.checkPackage = function (that) { @@ -245,8 +245,8 @@ gpii.iod.checkPackage = function (that) { /** * Generate the installation instructions. * - * @param that {Component} The gpii.iod.installer instance. - * @param iod {object} The gpii.iod instance. + * @param {Component} that The gpii.iod.installer instance. + * @param {object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.prepareInstall = function (that) { @@ -256,8 +256,8 @@ gpii.iod.prepareInstall = function (that) { /** * Cleans up things that are no longer required. * - * @param that {Component} The gpii.iod.installer instance. - * @param iod {object} The gpii.iod instance. + * @param {Component} that The gpii.iod.installer instance. + * @param {object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.cleanup = function (that) { diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index 4d2e7db7f..7cb818027 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -305,9 +305,9 @@ fluid.defaults("gpii.tests.iod.testInstallerFail", { /** * Test function for packageInstaller.startInstaller. - * @param that {Component} The gpii.tests.iod.testInstaller1 instance. - * @param iod {Component} The gpii.test.iod instance. - * @param packageInfo {object} The package to install. + * @param {Component} that The gpii.tests.iod.testInstaller1 instance. + * @param {Component} iod The gpii.test.iod instance. + * @param {object} packageInfo The package to install. * @return {Promise} A resolved promise. */ gpii.tests.iod.testInstaller1.startInstaller = function (that, iod, packageInfo) { From 75e5e5bd0f01441656090192ee481c73614f7822 Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 8 May 2018 21:16:54 +0100 Subject: [PATCH 09/56] GPII-2971: Documented packageInfo. --- gpii/node_modules/installOnDemand/README.md | 25 ++++++++++++++++++- .../installOnDemand/src/installOnDemand.js | 22 +++++++++++++++- .../installOnDemand/src/packageInstaller.js | 12 ++++----- 3 files changed, 51 insertions(+), 8 deletions(-) diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/installOnDemand/README.md index 42a1251f7..dcc097eec 100644 --- a/gpii/node_modules/installOnDemand/README.md +++ b/gpii/node_modules/installOnDemand/README.md @@ -37,13 +37,36 @@ The install on demand component. ### `gpii.iod.packageInstaller` Base component of the package installers, which perform the work that's specific to the type of package being installed. - Implementations will probably be found in the OS-specific repository. +This performs the initialise->cleanup pipeline. + ### `gpii.iod.packageDataSource` The package data source. +## Packages + +Packages consist of a `packageInfo` json file, and the package file. Support can be provided for different types of +packages, however chocolatey already does a good job at wrapping installers. + +```javascript +/** + * @typedef {Object} packageInfo + * @property {string} name - The package name. + * @property {string} url - The package location (optional, to override the IoD server with an external location). + * @property {string} filename - The package filename. + * @property {string} packageType - Type of installer to use. + * @property {string} hash - The signed hash of the package. + */ +packageInfo = { + name: "some-package", + url: "https://iod-server.example.com/some-package", + filename: "some-package.1.0.0.nupkg", + packageType: "chocolatey" +} +``` + ## IoD Server Using the development config, GPII will provide package data from [testData/installOnDemand](testData/installOnDemand). diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index a785c5e67..76bea37a5 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -30,6 +30,27 @@ fluid.registerNamespace("gpii.iod"); require("./packageInstaller.js"); +/** + * Information about a package. + * @typedef {Object} packageInfo + * @property {string} name - The package name. + * @property {string} url - The package location. + * @property {string} filename - The package filename. + * @property {string} packageType - Type of installer to use. + * + * Installation state. + * @typedef {Object} installation + * @property {id} - Unique identifier. + * @property {packageInfo} packageInfo - Package data. + * @property {packageInfo} packageName - packageInfo.name + * @property {Component} installer - The gpii.iod.installer instance. + * @property {boolean} failed - true if the installation had failed. + * @property {string} tmpDir - Temporary working directory. + * @property {string} localPackage - Path to the downloaded package file. + * @property {string[]} cleanupPaths - The directories to remove during cleanup. + * + */ + fluid.defaults("gpii.iod", { gradeNames: ["fluid.component"], contextAwareness: { @@ -173,7 +194,6 @@ gpii.iod.getInstaller = function (that, packageType) { * @param {Component} that The gpii.iod instance. * @param {string|Object} packageRequest Package name, or object containing packageName, language, version. * @param packageRequest.packageName {string} Name of the package. - * @param packageRequest.version {string} Name of the package. * @param packageRequest.language {string|string[]} Language. * @return {Promise} Resolves when the installation is complete. */ diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 2861d4a69..b7721fdcc 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -129,13 +129,13 @@ gpii.iod.startInstaller = function (that, iod, installation) { * Initialises the installation. * * @param {Component} that The gpii.iod.installer instance. - * @param {object} iod The gpii.iod instance. + * @param {Component} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.initialise = function (that, iod) { var tempDir = iod.getWorkingPath(that.packageInfo.name); - that.tempDir = tempDir.fullPath; - that.cleanupPaths.push(tempDir.createdPath); + that.installation.tempDir = tempDir.fullPath; + that.installation.cleanupPaths.push(tempDir.createdPath); }; /** @@ -150,14 +150,14 @@ gpii.iod.downloadPackage = function (that) { var promise = fluid.promise(); - that.localPackage = path.join(that.tempDir, that.packageInfo.filename); + that.installation.localPackage = path.join(that.installation.tempDir, that.packageInfo.filename); if (that.packageInfo.url.startsWith("https://")) { // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). - var downloadPromise = gpii.iod.httpsDownload(that.packageInfo.url, that.localPackage); + var downloadPromise = gpii.iod.httpsDownload(that.packageInfo.url, that.installation.localPackage); fluid.promise.follow(downloadPromise, promise); } else { - fs.copyFile(that.packageInfo.url, that.localPackage, function (err) { + fs.copyFile(that.packageInfo.url, that.installation.localPackage, function (err) { if (err) { promise.reject({ isError: true, From db6ca288f3806bc7e6ce1a8d37d618b76706adc2 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 10 May 2018 15:04:21 +0100 Subject: [PATCH 10/56] GPII-2971: Made the package installer creation nicer. --- .../installOnDemand/src/installOnDemand.js | 36 +++++++++++------ .../installOnDemand/src/packageInstaller.js | 39 ++++++++++++------- 2 files changed, 49 insertions(+), 26 deletions(-) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 76bea37a5..137a3acb2 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -40,7 +40,7 @@ require("./packageInstaller.js"); * * Installation state. * @typedef {Object} installation - * @property {id} - Unique identifier. + * @property {id} - Installation ID * @property {packageInfo} packageInfo - Package data. * @property {packageInfo} packageName - packageInfo.name * @property {Component} installer - The gpii.iod.installer instance. @@ -52,13 +52,13 @@ require("./packageInstaller.js"); */ fluid.defaults("gpii.iod", { - gradeNames: ["fluid.component"], + gradeNames: ["fluid.component", "gpii.windows.iod"], contextAwareness: { platform: { checks: { windows: { contextValue: "{gpii.contexts.windows}", - gradeNames: ["gpii.windows.iod"] + gradeNames: "gpii.windows.iod" } } } @@ -72,9 +72,19 @@ fluid.defaults("gpii.iod", { type: "gpii.iod.packageDataSource" } }, + dynamicComponents: { + installers: { + createOnEvent: "onInstallerStart", + type: "{arguments}.0", + options: { + installationID: "{arguments}.1" + } + } + }, events: { - onServiceFound: null, - onServiceLost: null + onServiceFound: null, // [ endpoint address ] + onServiceLost: null, // [ endpoint address ] + onInstallerStart: null // [ packageInstaller grade name, installation ID ] }, listeners: { "onCreate.discoverServer": "{that}.discoverServer", @@ -173,7 +183,7 @@ gpii.iod.getWorkingPath = function (packageName) { * * @param {Component} that The gpii.iod instance. * @param {string} packageType The package type identifier. - * @return {Component} A new instance of the gpii.iod.installer component that handles the requested type of package. + * @return {string} The grade name of the package installer. */ gpii.iod.getInstaller = function (that, packageType) { var packageInstallers = fluid.queryIoCSelector(that, "gpii.iod.packageInstaller"); @@ -185,7 +195,7 @@ gpii.iod.getInstaller = function (that, packageType) { : undefined; }); - return installerComponent && fluid.invokeGlobalFunction(installerComponent.typeName); + return installerComponent && installerComponent.typeName; }; /** @@ -209,7 +219,8 @@ gpii.iod.requirePackage = function (that, packageRequest) { var installation = { id: fluid.allocateGuid(), packageName: packageRequest.packageName, - packageRequest: packageRequest + packageRequest: packageRequest, + cleanupPaths: [] }; that.installations[installation.id] = installation; @@ -219,15 +230,16 @@ gpii.iod.requirePackage = function (that, packageRequest) { that.getPackageInfo(packageRequest).then(function (packageInfo) { // Create the installer instance. installation.packageInfo = packageInfo; - installation.installer = that.getInstaller(packageInfo.packageType); - if (installation.installer) { + var installerGrade = that.getInstaller(packageInfo.packageType); + if (installerGrade) { // Start the installer. - var result = installation.installer.startInstaller(installation); + that.events.onInstallerStart.fire(installerGrade, installation.id); + var result = installation.installer.startInstaller(); fluid.promise.follow(result, promise); } else { promise.reject({ isError: true, - error: "Unable to find an installer for package type " + packageInfo.packageTypes + error: "Unable to find an installer for package type " + packageInfo.packageType }); } }, promise.reject); diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index b7721fdcc..5fdc61ef3 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -30,9 +30,13 @@ fluid.defaults("gpii.iod.packageInstaller", { gradeNames: ["fluid.component"], invokers: { + created: { + funcName: "gpii.iod.installerCreated", + args: ["{that}", "{iod}"] + }, startInstaller: { funcName: "gpii.iod.startInstaller", - args: ["{that}", "{iod}", "{arguments}.0"] + args: ["{that}", "{iod}"] }, // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns // a installation, either directly or via a promise. @@ -65,6 +69,7 @@ fluid.defaults("gpii.iod.packageInstaller", { onRemovePackage: null }, listeners: { + "onCreate": "{that}.created", "onInstallPackage.initialise": { func: "{that}.initialise", priority: "first" @@ -100,28 +105,34 @@ fluid.defaults("gpii.iod.packageInstaller", { packageTypes: null, members: { - // Package information from the server . - packageInfo: null, - // Where this installation will put it's stuff. - tempDir: null, - // Path of the downloaded package. - localPackage: null, - // Paths to remove on cleanup. - cleanupPaths: [] + // Package information from the server. + packageInfo: null } }); + +/** + * Installer component created. + * + * @param {Component} that The gpii.iod.installer instance. + * @param {Component} iod The gpii.iod instance. + */ +gpii.iod.installerCreated = function (that, iod) { + that.installation = iod.installations[that.options.installationID]; + if (that.installation) { + that.installation.installer = that; + that.packageInfo = that.installation.packageInfo; + } +}; + /** * Starts the installation pipeline. * * @param {Component} that The gpii.iod.installer instance. - * @param {object} iod The gpii.iod instance. - * @param {Object} packageInfo The package info. + * @param {Component} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ -gpii.iod.startInstaller = function (that, iod, installation) { - that.installation = installation; - that.packageInfo = that.installation.packageInfo; +gpii.iod.startInstaller = function (that) { return fluid.promise.fireTransformEvent(that.events.onInstallPackage); }; From 34fb819d9fd04b250b9cae923d81c313f6e76ce6 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 10 May 2018 16:01:35 +0100 Subject: [PATCH 11/56] GPII-2971: Documented multi-language packages --- gpii/node_modules/installOnDemand/README.md | 28 +++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/installOnDemand/README.md index dcc097eec..e142cb839 100644 --- a/gpii/node_modules/installOnDemand/README.md +++ b/gpii/node_modules/installOnDemand/README.md @@ -67,6 +67,34 @@ packageInfo = { } ``` +### Multi-lingual packages + +Multiple languages can be supported in a single package, like so: + +```json +{ + "name": "another-package", + "url": "https://example.com/default-language", + "languages": { + "en-GB": { + "url": "https://example.com/real-english" + }, + "es": { + "url": "https://example.com/general-spanish" + }, + "es-ES": { + "url": "https://example.com/spainish-spanish" + }, + "es-MX": { + "url": "https://example.com/mexican-spanish" + } + } +} +``` + +When a package is requested, a language may be specified. The package can contain a `languages` field which contains +the language-specific fields for each supported language, which over-write the fields in the root. + ## IoD Server Using the development config, GPII will provide package data from [testData/installOnDemand](testData/installOnDemand). From e58bc395cacfae7d7b79246057621bded140237c Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 11 May 2018 16:14:00 +0100 Subject: [PATCH 12/56] GPII-2971: Can install/uninstall packages at key-in/out (very hacky) --- .../installOnDemand/src/installOnDemand.js | 126 ++++++++++++++++-- testData/installOnDemand/demo-package.json | 7 + testData/solutions/win32.json5 | 26 +++- 3 files changed, 144 insertions(+), 15 deletions(-) create mode 100644 testData/installOnDemand/demo-package.json diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 137a3acb2..0a8578c86 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -42,7 +42,7 @@ require("./packageInstaller.js"); * @typedef {Object} installation * @property {id} - Installation ID * @property {packageInfo} packageInfo - Package data. - * @property {packageInfo} packageName - packageInfo.name + * @property {string} packageName - packageInfo.name * @property {Component} installer - The gpii.iod.installer instance. * @property {boolean} failed - true if the installation had failed. * @property {string} tmpDir - Temporary working directory. @@ -74,7 +74,7 @@ fluid.defaults("gpii.iod", { }, dynamicComponents: { installers: { - createOnEvent: "onInstallerStart", + createOnEvent: "onInstallerLoad", type: "{arguments}.0", options: { installationID: "{arguments}.1" @@ -84,7 +84,7 @@ fluid.defaults("gpii.iod", { events: { onServiceFound: null, // [ endpoint address ] onServiceLost: null, // [ endpoint address ] - onInstallerStart: null // [ packageInstaller grade name, installation ID ] + onInstallerLoad: null // [ packageInstaller grade name, installation ID ] }, listeners: { "onCreate.discoverServer": "{that}.discoverServer", @@ -100,6 +100,14 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.requirePackage", args: ["{that}", "{arguments}.0"] }, + initialiseInstallation: { + funcName: "gpii.iod.initialiseInstallation", + args: ["{that}", "{arguments}.0"] + }, + uninstallPackage: { + funcName: "gpii.iod.uninstallPackage", + args: ["{that}", "{arguments}.0"] + }, getPackageInfo: { funcName: "gpii.iod.getPackageInfo", args: ["{that}", "{arguments}.0"] @@ -216,6 +224,30 @@ gpii.iod.requirePackage = function (that, packageRequest) { fluid.log("IoD: Requiring " + packageRequest.packageName); + var promise = fluid.promise(); + + that.initialiseInstallation(packageRequest).then(function (installation) { + var result = installation.installer.startInstaller(); + fluid.promise.follow(result, promise); + }); + + + return promise.then(function () { + fluid.log("IoD: Installation of " + packageRequest.packageName + " complete"); + }, function (err) { + fluid.log("IoD: Installation of " + packageRequest.packageName + " failed:", err.error || err); + }); +}; + +gpii.iod.initialiseInstallation = function (that, packageRequest) { + if (typeof(packageRequest) === "string") { + packageRequest = { + packageName: packageRequest + }; + } + + fluid.log("IoD: Initialising installation for " + packageRequest.packageName); + var installation = { id: fluid.allocateGuid(), packageName: packageRequest.packageName, @@ -232,10 +264,9 @@ gpii.iod.requirePackage = function (that, packageRequest) { installation.packageInfo = packageInfo; var installerGrade = that.getInstaller(packageInfo.packageType); if (installerGrade) { - // Start the installer. - that.events.onInstallerStart.fire(installerGrade, installation.id); - var result = installation.installer.startInstaller(); - fluid.promise.follow(result, promise); + // Load the installer. + that.events.onInstallerLoad.fire(installerGrade, installation.id); + promise.resolve(installation); } else { promise.reject({ isError: true, @@ -244,12 +275,7 @@ gpii.iod.requirePackage = function (that, packageRequest) { } }, promise.reject); - return promise.then(function () { - fluid.log("IoD: Installation complete"); - }, function (err) { - fluid.log("IoD: Installation failed:", err.error || err); - installation.failed = true; - }); + return promise; }; /** @@ -367,7 +393,37 @@ gpii.iod.startRemoval = function (that, installation) { * @param {object} installation The installation state. */ gpii.iod.uninstallPackage = function (that, installation) { - installation.installer.uninstallPackage(installation); + + var packageName; + if (typeof(installation) === "string") { + packageName = installation; + installation = fluid.find(that.installations, function (inst) { + return (inst.packageName === packageName) ? inst : undefined; + }); + } else { + packageName = installation.packageName; + } + + var initPromise; + if (installation) { + initPromise = fluid.toPromise(installation); + } else { + initPromise = that.initialiseInstallation(installation); + } + + var promiseTogo = fluid.promise(); + initPromise.then(function (installation) { + var result = installation.installer.uninstallPackage(); + fluid.promise.follow(result, promiseTogo); + }); + + + return promiseTogo.then(function () { + fluid.log("IoD: Uninstallation of " + packageName + " complete"); + }, function (err) { + fluid.log("IoD: Uninstallation of " + packageName + " failed:", err.error || err); + }); + }; /** @@ -461,3 +517,45 @@ gpii.iod.serviceFound = function (that, endPoint) { fluid.log("IoD: Endpoint found: " + endPoint); that.endpoint = endPoint; }; + + +/** + * Used while abusing launch handlers in order to provide a way to invoke IoD from the solutions registry. + * @param method {string} Method of gpii.iod to invoke + * @param args {array} Arguments for the method. + * @return {Promise} Resolves with the return value. + */ +gpii.iod.invoke = function (method, args) { + var iod = fluid.queryIoCSelector(fluid.rootComponent, "gpii.iod")[0]; + var promise = fluid.promise(); + + // For some reason the datasource.get call inside requirePackage doesn't resolve while it's inside this stack. + // A better developer would have discovered why, but you have to make do with what you've got. + process.nextTick(function () { + var result = iod[method].apply(iod, args); + fluid.promise.follow(fluid.toPromise(result), promise); + }); + + return promise; +}; + +/** + * A bad way of checking if a package is installed. + */ +gpii.iod.isInstalled = function (packageName) { + return fs.existsSync(path.join(process.env.ProgramData, "chocolatey\\lib", packageName)); +}; + +fluid.defaults("gpii.iod.invoke", { + gradeNames: "fluid.function", + argumentMap: { + method: 0, + args: 1 + } +}); +fluid.defaults("gpii.iod.isInstalled", { + gradeNames: "fluid.function", + argumentMap: { + packageName: 0 + } +}); diff --git a/testData/installOnDemand/demo-package.json b/testData/installOnDemand/demo-package.json new file mode 100644 index 000000000..9e7dc24bf --- /dev/null +++ b/testData/installOnDemand/demo-package.json @@ -0,0 +1,7 @@ +{ + "name": "demo-package", + "description": "Installs a small demo application", + "url": "https://github.com/stegru/gpii-iod/raw/GPII-2972/testData/packages/demo-package.1.0.0.nupkg", + "filename": "demo-package.1.0.0.nupkg", + "packageType": "chocolatey" +} diff --git a/testData/solutions/win32.json5 b/testData/solutions/win32.json5 index 02b010906..3234ab73e 100644 --- a/testData/solutions/win32.json5 +++ b/testData/solutions/win32.json5 @@ -1647,7 +1647,7 @@ "tr": "tr", "vi": "vi", "zh": "zh-cmn", - "zh-yue": "zh-yue", + "zh-yue": "zh-yue" } } ] @@ -2068,6 +2068,30 @@ } } }, + "launchHandlers": { + "launcher": { + "type": "gpii.launchHandlers.flexibleHandler", + "options": { + "getState": { + "type": "gpii.iod.isInstalled", + "packageName": "demo-package" + }, + "setTrue": { + "type": "gpii.iod.invoke", + "method": "requirePackage", + "args": ["demo-package"] + }, + "setFalse": { + "type": "gpii.iod.invoke", + "method": "uninstallPackage", + "args": ["demo-package"] + } + + + + } + } + }, "isInstalled": [ { "type": "gpii.deviceReporter.alwaysInstalled" From 5646ed35467941cf22d27283bb33cd3f7a89888e Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 14 May 2018 11:39:02 +0100 Subject: [PATCH 13/56] GPII-2971: Fixed incorrect context awareness --- gpii/node_modules/installOnDemand/src/installOnDemand.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 0a8578c86..c65b56c4a 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -52,7 +52,7 @@ require("./packageInstaller.js"); */ fluid.defaults("gpii.iod", { - gradeNames: ["fluid.component", "gpii.windows.iod"], + gradeNames: ["fluid.component", "fluid.contextAware"], contextAwareness: { platform: { checks: { @@ -133,6 +133,9 @@ fluid.defaults("gpii.iod", { args: ["{that}", "{arguments}.0"] } }, + model: { + installations: {} + }, members: { installations: {} From e69b63f7a18f9d31ba5f36aba1a3bbf6b8c8a025 Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 14 May 2018 15:19:39 +0100 Subject: [PATCH 14/56] GPII-2971: Persistent storage of installation info. --- .../installOnDemand/src/installOnDemand.js | 67 ++++++++-- .../test/installOnDemandTests.js | 117 ++++++++++++++++-- 2 files changed, 164 insertions(+), 20 deletions(-) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index c65b56c4a..8edfb94e4 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -23,7 +23,8 @@ var fluid = require("infusion"); var path = require("path"), os = require("os"), fs = require("fs"), - request = require("request"); + request = require("request"), + glob = require("glob"); var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.iod"); @@ -88,6 +89,7 @@ fluid.defaults("gpii.iod", { }, listeners: { "onCreate.discoverServer": "{that}.discoverServer", + "onCreate.readInstallations": "{that}.readInstallations", "onServiceFound": "{that}.serviceFound", "onServiceLost": "{that}.serviceLost" }, @@ -131,11 +133,16 @@ fluid.defaults("gpii.iod", { serviceLost: { funcName: "gpii.iod.serviceLost", args: ["{that}", "{arguments}.0"] + }, + readInstallations: { + funcName: "gpii.iod.readInstallations", + args: ["{that}", "{gpii.journal}.settingsDir.gpiiSettingsDir" ] + }, + writeInstallation: { + funcName: "gpii.iod.writeInstallation", + args: ["{that}", "{gpii.journal}.settingsDir.gpiiSettingsDir", "{arguments}.0"] } }, - model: { - installations: {} - }, members: { installations: {} @@ -147,6 +154,50 @@ fluid.defaults("gpii.iod.packageDataSource", { readOnlyGrade: "gpii.iod.packageDataSource" }); +/** + * Reads the stored installations from a previous instance. + * + * @param {Component} that The gpii.iod instance. + * @param {string} directory The directory containing the stored installation info. + * @return {Promise} Resolves when complete. + */ +gpii.iod.readInstallations = function (that, directory) { + var promise = fluid.promise(); + glob(path.join(directory, "iod-installation.*.json"), function (err, files) { + if (err) { + promise.reject({ + isError: true, + message: "Unable to read previous installations", + error: err + }); + } else { + fluid.each(files, function (file) { + var content = fs.readFileSync(file); + var installation = JSON.parse(content); + if (installation && installation.id) { + that.installations[installation.id] = installation; + } + }); + process.nextTick(promise.resolve); + } + }); + return promise; +}; + +/** + * Writes information about an installation so it can be uninstalled at a later time. + * + * @param {Component} that The gpii.iod instance. + * @param {string} directory The directory containing the stored installation info. + * @return {string} The file that the installation data was written to. + */ +gpii.iod.writeInstallation = function (that, directory, installation) { + var content = JSON.stringify(installation); + var filename = path.join(directory, "iod-installation." + installation.id + ".json"); + fs.writeFileSync(filename, content); + return filename; +}; + /** * Create a directory where packages are temporarily stored. * @@ -156,7 +207,6 @@ fluid.defaults("gpii.iod.packageDataSource", { gpii.iod.getWorkingPath = function (packageName) { var createdPath = null; - var parts = [ os.tmpdir(), "gpii-iod", @@ -230,9 +280,12 @@ gpii.iod.requirePackage = function (that, packageRequest) { var promise = fluid.promise(); that.initialiseInstallation(packageRequest).then(function (installation) { - var result = installation.installer.startInstaller(); + var result = installation.installer.startInstaller().then(function () { + // Store the installation info so it can still get removed if gpii restarts. + that.writeInstallation(installation); + }); fluid.promise.follow(result, promise); - }); + }, promise.reject); return promise.then(function () { diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index 7cb818027..7dcf1764f 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -20,7 +20,8 @@ var os = require("os"), fs = require("fs"), - path = require("path"); + path = require("path"), + rimraf = require("rimraf"); var fluid = require("gpii-universal"); var kettle = fluid.require("kettle"); @@ -32,7 +33,16 @@ fluid.registerNamespace("gpii.tests.iod"); require("../index.js"); -jqUnit.module("gpii.tests.iod"); +var teardowns = []; + +jqUnit.module("gpii.tests.iod", { + teardown: function () { + while (teardowns.length) { + teardowns.pop()(); + } + } +}); + gpii.tests.iod.getInstallerTests = fluid.freezeRecursive([ { @@ -258,6 +268,10 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ fluid.defaults("gpii.tests.iod", { gradeNames: [ "gpii.iod" ], + listeners: { + "onCreate.discoverServer": null, + "onCreate.readInstallations": null + }, components: { "testInstaller1": { type: "gpii.tests.iod.testInstaller1" @@ -268,10 +282,15 @@ fluid.defaults("gpii.tests.iod", { "testInstallerFail": { type: "gpii.tests.iod.testInstallerFail" }, - "packageDataSource": { + "packageDataFallback": { + createOnEvent: null, type: "kettle.dataSource.file", options: { - path: __dirname + "/testPackages/%packageName.json" + gradeNames: [ "kettle.dataSource.file.moduleTerms"], + path: __dirname + "/testPackages/%packageName.json", + termMap: { + "packageName": "%packageName" + } } } } @@ -307,16 +326,15 @@ fluid.defaults("gpii.tests.iod.testInstallerFail", { * Test function for packageInstaller.startInstaller. * @param {Component} that The gpii.tests.iod.testInstaller1 instance. * @param {Component} iod The gpii.test.iod instance. - * @param {object} packageInfo The package to install. * @return {Promise} A resolved promise. */ -gpii.tests.iod.testInstaller1.startInstaller = function (that, iod, packageInfo) { +gpii.tests.iod.testInstaller1.startInstaller = function (that, iod) { if (iod.startInstallerCalled) { jqUnit.fail("startInstaller called twice"); } iod.startInstallerCalled = { installer: that.typeName, - packageName: packageInfo.packageName + packageName: that.installation && that.installation.packageName }; var promise = fluid.promise(); @@ -408,14 +426,10 @@ jqUnit.test("test getInstaller", function () { if (test.expect) { jqUnit.assertEquals("getInstaller should return the correct installer for packageType=" + test.packageType, - test.expect, installer && installer.typeName); + test.expect, installer); } else { jqUnit.assertFalse("getInstaller should return nothing for packageType=" + test.packageType, !!installer); } - - if (installer) { - installer.destroy(); - } }); }); @@ -447,7 +461,10 @@ jqUnit.asyncTest("test getPackageInfo", function () { delete packageInfo.languages; jqUnit.assertDeepEq("packageInfo must match expected" + suffix, test.expect, packageInfo); nextTest(); - }, function () { + }, function (e) { + if (test.expect !== "reject") { + fluid.log(e); + } jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); nextTest(); }); @@ -493,3 +510,77 @@ jqUnit.asyncTest("test requirePackage", function () { nextTest(); }); + +jqUnit.asyncTest("test installation storage", function () { + + var dir = path.join(os.tmpdir(), "gpii-test" + Math.random()); + + fs.mkdirSync(dir); + teardowns.push(function () { + rimraf.sync(dir); + }); + + var iod = gpii.tests.iod(); + + var testData = { + existingInst: { + id: "existing-installation" + }, + newInst: { + id: "new-installation" + }, + updatedInst: { + id: "new-installation", + updated: "yes" + } + }; + + iod.installations = {}; + iod.installations[testData.existingInst.id] = testData.existingInst; + + var origInstallations = Object.assign({}, iod.installations); + + // No files exist - check the data is the same. + var p = gpii.iod.readInstallations(iod, dir); + + jqUnit.assertTrue("readInstallations should return a promise", fluid.isPromise(p)); + + p.then(function () { + jqUnit.assertDeepEq("readInstallation on an empty directory shouldn't change anything", + origInstallations, iod.installations); + + // Write the installation data. + var file = gpii.iod.writeInstallation(iod, dir, testData.newInst); + + jqUnit.assertEquals("writeInstallation should write to the correct file", + path.join(dir, "iod-installation." + testData.newInst.id + ".json"), file); + + // Check it was written correctly. + var writtenContent = fs.readFileSync(file); + var writtenObject = JSON.parse(writtenContent); + + jqUnit.assertDeepEq("writeInstallation should write the correct data", testData.newInst, writtenObject); + + // Check it gets loaded. + gpii.iod.readInstallations(iod, dir).then(function () { + jqUnit.assertDeepEq("readInstallations should read the correct data", + testData.newInst, iod.installations[testData.newInst.id]); + + // Overwrite the existing file. + var file = gpii.iod.writeInstallation(iod, dir, testData.updatedInst); + + // Check it was written correctly. + var writtenContent = fs.readFileSync(file); + var writtenObject = JSON.parse(writtenContent); + jqUnit.assertDeepEq("writeInstallation should overwrite with the correct data", + testData.updatedInst, writtenObject); + + // Check the updated file gets loaded. + gpii.iod.readInstallations(iod, dir).then(function () { + jqUnit.assertDeepEq("readInstallations should update with the correct data", + testData.updatedInst, iod.installations[testData.newInst.id]); + jqUnit.start(); + }); + }); + }); +}); From ea898f973051d039c81aa51e4f14048a21094922 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 17 May 2018 09:53:09 +0100 Subject: [PATCH 15/56] GPII-2971: Uninstall when inactive, or after restart if failed. --- .../installOnDemand/src/installOnDemand.js | 138 +++++++--- .../test/installOnDemandTests.js | 241 +++++++++++++++--- testData/solutions/win32.json5 | 2 +- tests/all-tests.js | 3 +- 4 files changed, 324 insertions(+), 60 deletions(-) diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 8edfb94e4..49ba7cfff 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -53,7 +53,7 @@ require("./packageInstaller.js"); */ fluid.defaults("gpii.iod", { - gradeNames: ["fluid.component", "fluid.contextAware"], + gradeNames: ["fluid.component", "fluid.contextAware", "fluid.modelComponent"], contextAwareness: { platform: { checks: { @@ -91,7 +91,8 @@ fluid.defaults("gpii.iod", { "onCreate.discoverServer": "{that}.discoverServer", "onCreate.readInstallations": "{that}.readInstallations", "onServiceFound": "{that}.serviceFound", - "onServiceLost": "{that}.serviceLost" + "onServiceLost": "{that}.serviceLost", + "{lifecycleManager}.events.onSessionStop": "{that}.uninstallPackages" }, invokers: { discoverServer: { @@ -106,18 +107,10 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.initialiseInstallation", args: ["{that}", "{arguments}.0"] }, - uninstallPackage: { - funcName: "gpii.iod.uninstallPackage", - args: ["{that}", "{arguments}.0"] - }, getPackageInfo: { funcName: "gpii.iod.getPackageInfo", args: ["{that}", "{arguments}.0"] }, - startRemoval: { - funcName: "gpii.iod.startRemoval", - args: ["{that}", "{arguments}.0"] - }, getInstaller: { funcName: "gpii.iod.getInstaller", args: ["{that}", "{arguments}.0"] @@ -141,9 +134,25 @@ fluid.defaults("gpii.iod", { writeInstallation: { funcName: "gpii.iod.writeInstallation", args: ["{that}", "{gpii.journal}.settingsDir.gpiiSettingsDir", "{arguments}.0"] + }, + unrequirePackage: { + funcName: "gpii.iod.unrequirePackage", + args: ["{that}", "{arguments}.0"] + }, + uninstallPackages: { + funcName: "gpii.iod.uninstallPackages", + args: ["{that}", "{arguments}.0"] + }, + uninstallPackage: { + funcName: "gpii.iod.uninstallPackage", + args: ["{that}", "{arguments}.0"] } }, + model: { + logonChange: "{lifecycleManager}.model.logonChange" + }, + members: { installations: {} } @@ -155,14 +164,19 @@ fluid.defaults("gpii.iod.packageDataSource", { }); /** - * Reads the stored installations from a previous instance. + * Reads the stored installations from a previous instance. Installations will be kept when an uninstall fails, or if + * GPII was closed without keying out. + * All installed packages from the previous instance will be set for removal. * * @param {Component} that The gpii.iod instance. * @param {string} directory The directory containing the stored installation info. + * @param {bool} justRead true to only read the packages, and not mark them for removal. * @return {Promise} Resolves when complete. */ gpii.iod.readInstallations = function (that, directory) { var promise = fluid.promise(); + var needRemove = false; + glob(path.join(directory, "iod-installation.*.json"), function (err, files) { if (err) { promise.reject({ @@ -175,26 +189,46 @@ gpii.iod.readInstallations = function (that, directory) { var content = fs.readFileSync(file); var installation = JSON.parse(content); if (installation && installation.id) { + installation.remove = true; + installation.uninstalling = false; + needRemove = needRemove || installation.remove; that.installations[installation.id] = installation; } }); process.nextTick(promise.resolve); } }); + + promise.then(function () { + if (needRemove) { + // Loaded some uninstalled installation. + that.uninstallPackages(0); + } + }); + return promise; }; /** * Writes information about an installation so it can be uninstalled at a later time. + * If the installation has been removed, then the file will be deleted. * * @param {Component} that The gpii.iod instance. * @param {string} directory The directory containing the stored installation info. * @return {string} The file that the installation data was written to. */ gpii.iod.writeInstallation = function (that, directory, installation) { - var content = JSON.stringify(installation); var filename = path.join(directory, "iod-installation." + installation.id + ".json"); - fs.writeFileSync(filename, content); + + if (installation.removed) { + fs.unlinkSync(filename); + } else { + // Don't write the installer component. + var out = Object.assign({}, installation); + delete out.installer; + var content = JSON.stringify(out); + fs.writeFileSync(filename, content); + } return filename; }; @@ -279,8 +313,19 @@ gpii.iod.requirePackage = function (that, packageRequest) { var promise = fluid.promise(); + var installation = fluid.find(that.installations, function (inst) { + return inst.packageName === packageRequest.packageName ? inst : undefined; + }); + + if (installation) { + // Package is already installed by IoD. + installation.remove = false; + that.writeInstallation(installation); + } + that.initialiseInstallation(packageRequest).then(function (installation) { var result = installation.installer.startInstaller().then(function () { + installation.installed = true; // Store the installation info so it can still get removed if gpii restarts. that.writeInstallation(installation); }); @@ -422,34 +467,62 @@ gpii.iod.matchLanguage = function (languages, language) { }; /** - * Starts the package removal routine. + * No longer require a package. This will cause the package to be uninstalled in a short-time if there is no active + * session. * * @param {Component} that The gpii.iod instance. - * @param {object} installation The installation state. - * @return {Promise} Resolves to an object containing package information and installation state. + * @param {string} packageName The name of the package to no longer require. */ -gpii.iod.startRemoval = function (that, installation) { - if (typeof(installation) === "string") { - installation = that.installations[installation]; - } +gpii.iod.unrequirePackage = function (that, packageName) { + + var installation = fluid.find(that.installations, function (inst) { + return inst.packageName === packageName ? inst : undefined; + }); - if (!installation.installer) { - installation.installer = that.getInstaller(installation.packageInfo.packageType); + if (installation) { + installation.remove = true; + that.writeInstallation(installation); } +}; - fluid.log("IoD: Removing installation of " + installation.packageName); - var promise = fluid.promise.fireTransformEvent(that.events.onRequirePackage, installation); - return promise; +/** + * Called by onSessionStop to uninstall the packages that are no longer required. The removal will be performed after + * a short time if there is no active session, to avoid giving the computer too much to do while it's in use. + * + * @param {Component} that The gpii.iod instance. + * @param {number} wait Number of seconds to wait until uninstalling (default: 30). + */ +gpii.iod.uninstallPackages = function (that, wait) { + + var uninstall = function () { + var inSession = that.model.logonChange.inProgress && that.model.logonChange.type !== "login"; + if (!inSession) { + var installation = fluid.find(that.installations, function (inst) { + return inst.remove && !inst.removed ? inst : undefined; + }); + + if (installation && !installation.uninstalling) { + installation.uninstalling = true; + that.uninstallPackage(installation).then(uninstall, uninstall); + } + } + }; + + if (wait === 0) { + uninstall(); + } else { + setTimeout(uninstall, (wait || 30) * 1000); + } }; /** * Uninstall a package. * * @param {Component} that The gpii.iod instance. - * @param {object} installation The installation state. + * @param {object|string} The installation state, or installation ID. + * @return {Promise} Resolves when the package is removed. */ gpii.iod.uninstallPackage = function (that, installation) { - var packageName; if (typeof(installation) === "string") { packageName = installation; @@ -461,7 +534,7 @@ gpii.iod.uninstallPackage = function (that, installation) { } var initPromise; - if (installation) { + if (installation && installation.installer) { initPromise = fluid.toPromise(installation); } else { initPromise = that.initialiseInstallation(installation); @@ -476,10 +549,17 @@ gpii.iod.uninstallPackage = function (that, installation) { return promiseTogo.then(function () { fluid.log("IoD: Uninstallation of " + packageName + " complete"); + installation.remove = false; + installation.removed = true; + that.writeInstallation(installation); + delete that.installations[installation.id]; + }, function (err) { fluid.log("IoD: Uninstallation of " + packageName + " failed:", err.error || err); + // Remove it from the list so it's uninstalled again, but the file is kept so it tries again upon restart. + that.writeInstallation(installation); + delete that.installations[installation.id]; }); - }; /** diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index 7dcf1764f..96827a424 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -23,9 +23,10 @@ var os = require("os"), path = require("path"), rimraf = require("rimraf"); -var fluid = require("gpii-universal"); +var fluid = require("infusion"); var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); + var jqUnit = fluid.require("node-jqunit"); var gpii = fluid.registerNamespace("gpii"); @@ -267,10 +268,12 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ ]); fluid.defaults("gpii.tests.iod", { - gradeNames: [ "gpii.iod" ], + gradeNames: [ "gpii.iod", "gpii.lifecycleManager" ], + listeners: { "onCreate.discoverServer": null, - "onCreate.readInstallations": null + "onCreate.readInstallations": null, + "{lifecycleManager}.events.onSessionStop": null }, components: { "testInstaller1": { @@ -293,6 +296,16 @@ fluid.defaults("gpii.tests.iod", { } } } + }, + invokers: { + readInstallations: "fluid.identity", + writeInstallation: "fluid.identity" + }, + model: { + loginChange: null + }, + members: { + funcCalled: {} } }); @@ -301,10 +314,13 @@ fluid.defaults("gpii.tests.iod.testInstaller1", { invokers: { installPackage: "fluid.identity", - uninstallPackage: "fluid.identity", + uninstallPackage: { + funcName: "gpii.tests.iod.testInstaller1.testFunctionCalled", + args: ["{that}", "{iod}", "uninstallPackage"] + }, startInstaller: { - funcName: "gpii.tests.iod.testInstaller1.startInstaller", - args: ["{that}", "{iod}", "{arguments}.0"] + funcName: "gpii.tests.iod.testInstaller1.testFunctionCalled", + args: ["{that}", "{iod}", "startInstaller"] } }, @@ -318,27 +334,28 @@ fluid.defaults("gpii.tests.iod.testInstaller2", { fluid.defaults("gpii.tests.iod.testInstallerFail", { gradeNames: ["gpii.tests.iod.testInstaller1"], - testReject: true, + testReject: "startInstaller", packageTypes: "testFailPackageType" }); /** - * Test function for packageInstaller.startInstaller. + * Test function for packageInstaller, to check if a certain function has been called. * @param {Component} that The gpii.tests.iod.testInstaller1 instance. * @param {Component} iod The gpii.test.iod instance. + * @param {string} funcName Name of the function that id being tests. * @return {Promise} A resolved promise. */ -gpii.tests.iod.testInstaller1.startInstaller = function (that, iod) { - if (iod.startInstallerCalled) { - jqUnit.fail("startInstaller called twice"); +gpii.tests.iod.testInstaller1.testFunctionCalled = function (that, iod, funcName) { + if (iod.funcCalled[funcName]) { + jqUnit.fail(funcName + " called twice"); } - iod.startInstallerCalled = { + iod.funcCalled[funcName] = { installer: that.typeName, packageName: that.installation && that.installation.packageName }; var promise = fluid.promise(); - if (that.options.testReject) { + if (that.options.testReject === funcName || iod.options.testReject === funcName) { promise.reject({ isError: true, error: "Test failure" @@ -490,15 +507,15 @@ jqUnit.asyncTest("test requirePackage", function () { var test = tests[testIndex]; var suffix = " - test:" + test.id; - iod.startInstallerCalled = null; + iod.funcCalled.startInstaller = null; var p = iod.requirePackage(test.packageRequest); jqUnit.assertTrue("requirePackage must return a promise" + suffix, fluid.isPromise(p)); p.then(function () { - jqUnit.assertTrue("startInstaller must have been called" + suffix, !!iod.startInstallerCalled); + jqUnit.assertTrue("startInstaller must have been called" + suffix, !!iod.funcCalled.startInstaller); jqUnit.assertDeepEq("startInstaller must have been called correctly" + suffix, - test.expect, iod.startInstallerCalled); + test.expect, iod.funcCalled.startInstaller); nextTest(); }, function () { jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); @@ -520,23 +537,57 @@ jqUnit.asyncTest("test installation storage", function () { rimraf.sync(dir); }); - var iod = gpii.tests.iod(); + var iod = gpii.tests.iod({ + invokers: { + uninstallPackages: { + funcName: "gpii.tests.iod.testInstaller1.testFunctionCalled", + args: ["{that}", "{iod}", "uninstallPackages"] + } + } + }); var testData = { existingInst: { - id: "existing-installation" + input: { + id: "existing-installation" + }, + expect: { + id: "existing-installation" + } }, newInst: { - id: "new-installation" + input: { + id: "new-installation" + }, + expectFile: { + id: "new-installation" + }, + expectRead: { + id: "new-installation", + remove: true, + uninstalling: false + } }, updatedInst: { - id: "new-installation", - updated: "yes" + input: { + id: "new-installation", + test1: "something" + }, + expectFile: { + id: "new-installation", + test1: "something" + }, + expectRead: { + id: "new-installation", + test1: "something", + remove: true, + uninstalling: false + } } }; iod.installations = {}; - iod.installations[testData.existingInst.id] = testData.existingInst; + iod.installations[testData.existingInst.input.id] = testData.existingInst.input; var origInstallations = Object.assign({}, iod.installations); @@ -549,38 +600,170 @@ jqUnit.asyncTest("test installation storage", function () { jqUnit.assertDeepEq("readInstallation on an empty directory shouldn't change anything", origInstallations, iod.installations); + jqUnit.assertFalse("uninstallPackages should not have been called", iod.funcCalled.uninstallPackages); + // Write the installation data. - var file = gpii.iod.writeInstallation(iod, dir, testData.newInst); + var file = gpii.iod.writeInstallation(iod, dir, testData.newInst.input); jqUnit.assertEquals("writeInstallation should write to the correct file", - path.join(dir, "iod-installation." + testData.newInst.id + ".json"), file); + path.join(dir, "iod-installation." + testData.newInst.input.id + ".json"), file); // Check it was written correctly. var writtenContent = fs.readFileSync(file); var writtenObject = JSON.parse(writtenContent); - jqUnit.assertDeepEq("writeInstallation should write the correct data", testData.newInst, writtenObject); + jqUnit.assertDeepEq("writeInstallation should write the correct data", + testData.newInst.expectFile, writtenObject); // Check it gets loaded. gpii.iod.readInstallations(iod, dir).then(function () { jqUnit.assertDeepEq("readInstallations should read the correct data", - testData.newInst, iod.installations[testData.newInst.id]); + testData.newInst.expectRead, iod.installations[testData.newInst.input.id]); + + jqUnit.assertTrue("uninstallPackages should have been called", !!iod.funcCalled.uninstallPackages); + iod.funcCalled.uninstallPackages = null; // Overwrite the existing file. - var file = gpii.iod.writeInstallation(iod, dir, testData.updatedInst); + var file = gpii.iod.writeInstallation(iod, dir, testData.updatedInst.input); // Check it was written correctly. var writtenContent = fs.readFileSync(file); var writtenObject = JSON.parse(writtenContent); jqUnit.assertDeepEq("writeInstallation should overwrite with the correct data", - testData.updatedInst, writtenObject); + testData.updatedInst.expectFile, writtenObject); // Check the updated file gets loaded. gpii.iod.readInstallations(iod, dir).then(function () { jqUnit.assertDeepEq("readInstallations should update with the correct data", - testData.updatedInst, iod.installations[testData.newInst.id]); + testData.updatedInst.expectRead, iod.installations[testData.newInst.input.id]); + + jqUnit.assertTrue("uninstallPackages should have been called again", + !!iod.funcCalled.uninstallPackages); + jqUnit.start(); }); }); }); }); + +// Tests package gets uninstalled after unrequirePackage is called. +jqUnit.asyncTest("test uninstallation", function () { + + jqUnit.expect(4); + + var iod = gpii.tests.iod(); + + var packageName = "package1"; + + iod.requirePackage(packageName).then(function () { + var installation = fluid.find(iod.installations, function (inst) { + return inst.packageName === packageName ? inst : undefined; + }); + + jqUnit.assertTrue("Package should have installed", installation && installation.installed); + jqUnit.assertTrue("Package should have been added to the list", !!iod.installations[installation.id]); + + iod.unrequirePackage(packageName); + + jqUnit.assertTrue("Package should have been set to be removed", installation.remove); + + iod.uninstallPackages(0); + + var retries = 10; + var waitForRemoval = function () { + // There's no promise or event, so just poll. + if (iod.installations[installation.id]) { + if (--retries > 0) { + setTimeout(waitForRemoval, 100); + } else { + fluid.fail("Package was not removed"); + } + } else { + jqUnit.assertTrue("packageInstaller.uninstallPackage should have been called", + !!iod.funcCalled.uninstallPackage); + jqUnit.start(); + } + }; + + process.nextTick(waitForRemoval); + + }, jqUnit.fail); + +}); + + +// Tests package whose uninstallation fails gets uninstalled after a restart. +jqUnit.asyncTest("test uninstallation after restart", function () { + + var dir = path.join(os.tmpdir(), "gpii-test" + Math.random()); + fs.mkdirSync(dir); + teardowns.push(function () { + rimraf.sync(dir); + }); + + var iodOptions = { + invokers: { + readInstallations: { + funcName: "gpii.iod.readInstallations", + args: ["{that}", dir ] + }, + writeInstallation: { + funcName: "gpii.iod.writeInstallation", + args: ["{that}", dir, "{arguments}.0"] + } + }, + testReject: "uninstallPackage" + }; + + var iod = gpii.tests.iod(iodOptions); + + var packageName = "package1"; + + // Wait for uninstallPackage to be called (should be called instantly) + var waitForUninstall = function () { + var promise = fluid.promise(); + var retries = 50; + var retry = function () { + if (iod.funcCalled.uninstallPackage) { + promise.resolve(); + } else { + if (--retries > 0) { + setTimeout(retry, 100); + } else { + promise.reject("Package was not removed"); + } + } + }; + retry(); + + return promise; + }; + + // Install the package, and fail uninstall. + iod.requirePackage(packageName).then(function () { + var installation = fluid.find(iod.installations, function (inst) { + return inst.packageName === packageName ? inst : undefined; + }); + + jqUnit.assertTrue("Package should have installed", installation && installation.installed); + jqUnit.assertTrue("Package should have been added to the list", !!iod.installations[installation.id]); + + iod.unrequirePackage(packageName); + + jqUnit.assertTrue("Package should have been set to be removed", installation.remove); + + iod.uninstallPackages(0); + + waitForUninstall().then(function () { + // Fake a restart by creating a new instance of iod. + iod.destroy(); + iodOptions.testReject = null; + iod = gpii.tests.iod(iodOptions); + iod.readInstallations(); + + waitForUninstall().then(jqUnit.start, jqUnit.fail); + + + }, jqUnit.fail); + }, jqUnit.fail); +}); diff --git a/testData/solutions/win32.json5 b/testData/solutions/win32.json5 index 3234ab73e..d0c671d93 100644 --- a/testData/solutions/win32.json5 +++ b/testData/solutions/win32.json5 @@ -2083,7 +2083,7 @@ }, "setFalse": { "type": "gpii.iod.invoke", - "method": "uninstallPackage", + "method": "unrequirePackage", "args": ["demo-package"] } diff --git a/tests/all-tests.js b/tests/all-tests.js index bc564c920..a315fdf89 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -90,7 +90,8 @@ var testIncludes = [ "../gpii/node_modules/contextManager/test/ContextManagerTests.js", "../gpii/node_modules/singleInstance/test/SingleInstanceTests.js", "../gpii/node_modules/eventLog/test/EventLogTests.js", - "../gpii/node_modules/userListeners/test/all-tests.js" + "../gpii/node_modules/userListeners/test/all-tests.js", + "../gpii/node_modules/installOnDemand/test/installOnDemandTests.js" ]; fluid.each(testIncludes, function (path) { From 577428bc6170b25573b1a8f54be5c8e17ee9e692 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 17 May 2018 12:10:07 +0100 Subject: [PATCH 16/56] GPII-2971: Documentation --- gpii/node_modules/installOnDemand/README.md | 48 ++++++++++++------- .../test/installOnDemandTests.js | 2 +- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/installOnDemand/README.md index e142cb839..8af3f552c 100644 --- a/gpii/node_modules/installOnDemand/README.md +++ b/gpii/node_modules/installOnDemand/README.md @@ -2,30 +2,37 @@ Provides the ability to install software on demand. -The stages of installation: +[Technical Design](https://tinyurl.com/y7xhcghu) (google doc) + +## Operation -### start +### GPII Start -* Gets the package info from the server. -* `requirePackage` finds a suitable `gpii.iod.packageInstaller` for the package. +* IoD server is detected using mDNS (or configured server, or local datasource) +* Installation information about packages that *GPII* has installed but not removed, is loaded (if any). + * These packages are uninstalled. -### initialise -Creates a temporary directory, starts the following pipeline. +### Key-in -### download -Downloads the package. +Current implementation is a hack which uses a launch handler to invoke `gpii.iod.requirePackage`. -### check -Checks the downloaded package. +The stages of installation: -### prepareInstall -Generates the installation commands. +* Get the package info from the server. +* `requirePackage` finds a suitable `gpii.iod.packageInstaller` for the type package. +* Package file is downloaded from the URI in the package info. +* The installer component installs the package: + * chocolatey: Windows service is instructed to run `choco install`. +* Installation info is stored to disk to survive reboot. +* Package file is removed. -### install -Installs the package. +### Key-out -### cleanup -Cleans the files. +* If the key-out is due to another user logging in, then wait for the next key-out. +* The installer component uninstalls the package. + * chocolatey: Windows service is instructed to run `choco uninstall`. +* If successful, installation info file is removed. +* If failed, the package is uninstalled when GPII starts again. ## Parts @@ -48,7 +55,7 @@ The package data source. ## Packages Packages consist of a `packageInfo` json file, and the package file. Support can be provided for different types of -packages, however chocolatey already does a good job at wrapping installers. +packages, however chocolatey already does a good job at wrapping other installer types. ```javascript /** @@ -83,7 +90,7 @@ Multiple languages can be supported in a single package, like so: "url": "https://example.com/general-spanish" }, "es-ES": { - "url": "https://example.com/spainish-spanish" + "url": "https://example.com/spain-spanish" }, "es-MX": { "url": "https://example.com/mexican-spanish" @@ -95,6 +102,11 @@ Multiple languages can be supported in a single package, like so: When a package is requested, a language may be specified. The package can contain a `languages` field which contains the language-specific fields for each supported language, which over-write the fields in the root. +In the example above, all languages shall use the root values unless the requested language is British English, in that +case the `en-GB` block is used, or any type of Spanish. Spanish from Spain or Mexico would use the block that specific +to those countries (`es-ES` or `es-MX`), any other Spanish dialect will use the generic `es` block. + + ## IoD Server Using the development config, GPII will provide package data from [testData/installOnDemand](testData/installOnDemand). diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index 96827a424..da438e0da 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -367,7 +367,7 @@ gpii.tests.iod.testInstaller1.testFunctionCalled = function (that, iod, funcName return promise; }; - +///* jqUnit.test("test getWorkingPath", function () { var safeToRemove = false; From a7f58cc59f88530dc03e29736ab62c7a9d37ca97 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 17 May 2018 12:27:53 +0100 Subject: [PATCH 17/56] GPII-2971: Documentation fix --- gpii/node_modules/installOnDemand/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/installOnDemand/README.md index 8af3f552c..318ec4caf 100644 --- a/gpii/node_modules/installOnDemand/README.md +++ b/gpii/node_modules/installOnDemand/README.md @@ -109,7 +109,7 @@ to those countries (`es-ES` or `es-MX`), any other Spanish dialect will use the ## IoD Server -Using the development config, GPII will provide package data from [testData/installOnDemand](testData/installOnDemand). +Using the development config, GPII will provide package data from [testData/installOnDemand](../../../testData/installOnDemand). But, if it detects a running instance of the server [stegru/gpii-iod](https://github.com/stegru/gpii-iod) somewhere on the network then that will be used instead. From ff6615a1b089e905340ac0092e5d7ef200f9b9d1 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 17 May 2018 12:36:27 +0100 Subject: [PATCH 18/56] GPII-2971: Missing comma --- tests/all-tests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/all-tests.js b/tests/all-tests.js index d32f0cebe..06a54494c 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -92,7 +92,7 @@ var testIncludes = [ "../gpii/node_modules/settingsHandlers/test/WebSocketsSettingsHandlerTests.js", "../gpii/node_modules/settingsHandlers/test/settingsHandlerUtilitiesTests.js", "../gpii/node_modules/singleInstance/test/SingleInstanceTests.js", - "../gpii/node_modules/userListeners/test/all-tests.js" + "../gpii/node_modules/userListeners/test/all-tests.js", "../gpii/node_modules/installOnDemand/test/installOnDemandTests.js" ]; From 0e4685ee935c28fa865555b14d0063804ab0d819 Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 22 May 2018 14:46:14 +0100 Subject: [PATCH 19/56] GPII-2971: Removed chocolatey installation script --- gpii/node_modules/installOnDemand/scripts/setup.ps1 | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 gpii/node_modules/installOnDemand/scripts/setup.ps1 diff --git a/gpii/node_modules/installOnDemand/scripts/setup.ps1 b/gpii/node_modules/installOnDemand/scripts/setup.ps1 deleted file mode 100644 index badaf0500..000000000 --- a/gpii/node_modules/installOnDemand/scripts/setup.ps1 +++ /dev/null @@ -1,3 +0,0 @@ - -Set-ExecutionPolicy Bypass -Scope Process -Force -iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1')) From 3620aa2c00657488062501543bf539d620639fad Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 6 Jun 2018 13:14:34 +0100 Subject: [PATCH 20/56] GPII-2971: IoD settings handler --- .../installOnDemand/src/installOnDemand.js | 36 +------ .../installOnDemand/src/iodSettingsHandler.js | 93 +++++++++++++++++++ .../deviceReporter/installedSolutions.json | 4 + testData/solutions/win32.json5 | 40 +++++++- 4 files changed, 137 insertions(+), 36 deletions(-) create mode 100644 gpii/node_modules/installOnDemand/src/iodSettingsHandler.js diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 49ba7cfff..3c96c6c70 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -30,6 +30,7 @@ var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.iod"); require("./packageInstaller.js"); +require("./iodSettingsHandler.js"); /** * Information about a package. @@ -654,44 +655,9 @@ gpii.iod.serviceFound = function (that, endPoint) { that.endpoint = endPoint; }; - -/** - * Used while abusing launch handlers in order to provide a way to invoke IoD from the solutions registry. - * @param method {string} Method of gpii.iod to invoke - * @param args {array} Arguments for the method. - * @return {Promise} Resolves with the return value. - */ -gpii.iod.invoke = function (method, args) { - var iod = fluid.queryIoCSelector(fluid.rootComponent, "gpii.iod")[0]; - var promise = fluid.promise(); - - // For some reason the datasource.get call inside requirePackage doesn't resolve while it's inside this stack. - // A better developer would have discovered why, but you have to make do with what you've got. - process.nextTick(function () { - var result = iod[method].apply(iod, args); - fluid.promise.follow(fluid.toPromise(result), promise); - }); - - return promise; -}; - /** * A bad way of checking if a package is installed. */ gpii.iod.isInstalled = function (packageName) { return fs.existsSync(path.join(process.env.ProgramData, "chocolatey\\lib", packageName)); }; - -fluid.defaults("gpii.iod.invoke", { - gradeNames: "fluid.function", - argumentMap: { - method: 0, - args: 1 - } -}); -fluid.defaults("gpii.iod.isInstalled", { - gradeNames: "fluid.function", - argumentMap: { - packageName: 0 - } -}); diff --git a/gpii/node_modules/installOnDemand/src/iodSettingsHandler.js b/gpii/node_modules/installOnDemand/src/iodSettingsHandler.js new file mode 100644 index 000000000..582de7d4f --- /dev/null +++ b/gpii/node_modules/installOnDemand/src/iodSettingsHandler.js @@ -0,0 +1,93 @@ +/* + * Install on Demand. + * + * Copyright 2018 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); + + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.iod.settingsHandler"); + +require("./packageInstaller.js"); + +gpii.iod.settingsHandler.getImpl = function () { +}; +gpii.iod.settingsHandler.setImpl = function (payload) { + + var packages = fluid.transform(payload.options, function (value, key) { + return Object.assign({}, value, payload.settings[key]); + }); + + var iod = fluid.queryIoCSelector(fluid.rootComponent, "gpii.iod")[0]; + var promise = fluid.promise(); + var results = {}; + var packageKeys = Object.keys(packages); + + var nextPackage = function () { + if (packageKeys.length === 0) { + promise.resolve(results); + } else { + var key = packageKeys.shift(); + var packageRequest = packages[key]; + + var isInstalled = iod.isInstalled(packageRequest.packageName); + + var p; + results[key] = { + oldValue: { + uninstall: true, + installed: isInstalled + }, + newValue: { + } + }; + + if (packageRequest.uninstall) { + iod.unrequirePackage(packageRequest.packageName); + } else if (!isInstalled) { + p = iod.requirePackage(packageRequest); + p.then(function () { + results[key].newValue.installed = true; + }, function (error) { + fluid.log(error); + results[key].newValue.installed = false; + }); + } + + fluid.toPromise(p).then(nextPackage, nextPackage); + } + }; + + // For some reason the datasource.get call inside requirePackage doesn't resolve while it's inside this stack. + // A better developer would have discovered why, but you have to make do with what you've got. + process.nextTick(nextPackage); + + + return promise; + +}; + +gpii.iod.settingsHandler.get = function (payload) { + return gpii.settingsHandlers.invokeSettingsHandler(gpii.iod.settingsHandler.getImpl, payload); +}; + +gpii.iod.settingsHandler.set = function (payload) { + return gpii.settingsHandlers.invokeSettingsHandler(gpii.iod.settingsHandler.setImpl, payload); + +}; diff --git a/testData/deviceReporter/installedSolutions.json b/testData/deviceReporter/installedSolutions.json index 96c08de5d..1d0ae0328 100644 --- a/testData/deviceReporter/installedSolutions.json +++ b/testData/deviceReporter/installedSolutions.json @@ -125,6 +125,10 @@ { "id": "net.gpii.uioPlus" + }, + + { + "id": "net.gpii.test.iod" } ] diff --git a/testData/solutions/win32.json5 b/testData/solutions/win32.json5 index 6b46e7272..4c90dd725 100644 --- a/testData/solutions/win32.json5 +++ b/testData/solutions/win32.json5 @@ -1,4 +1,42 @@ { + "net.gpii.test.iod": { + "name": "Install on demand demo package", + "contexts": { + "OS": [ + { + "id": "win32", + "version": ">=5.0" + } + ] + }, + "settingsHandlers": { + "install": { + "type": "gpii.iod.settingsHandler", + "capabilities": [], + "options": { + "package1": { + "packageName": "demo-package" + } + }, + "capabilitiesTransformations": { + "package1": { + "transform": { + "type": "fluid.transforms.literalValue", + "input": "hello", + "outputPath": "a", + } + } + }, + } + }, + "isInstalled": [ + { + "type": "gpii.deviceReporter.alwaysInstalled" + } + ] + }, + + "com.freedomscientific.jaws": { "name": "JAWS", "contexts": { @@ -3378,5 +3416,5 @@ } ], "isRunning": [] - } + }, } From d6bd66696e0a0250ed46a98648972d6a36b3474f Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 20 Aug 2018 19:26:33 +0100 Subject: [PATCH 21/56] GPII-2971: Linting fixes. --- .../flowManager/src/FlowManager.js | 2 +- .../installOnDemand/src/installOnDemand.js | 50 ++++++++++--------- .../installOnDemand/src/packageInstaller.js | 21 +++++--- .../test/installOnDemandTests.js | 2 +- .../test/packageInstallerTests.js | 1 - testData/solutions/win32.json5 | 4 +- tests/all-tests.js | 2 +- 7 files changed, 45 insertions(+), 37 deletions(-) diff --git a/gpii/node_modules/flowManager/src/FlowManager.js b/gpii/node_modules/flowManager/src/FlowManager.js index 566b474c7..c1659149b 100644 --- a/gpii/node_modules/flowManager/src/FlowManager.js +++ b/gpii/node_modules/flowManager/src/FlowManager.js @@ -106,7 +106,7 @@ fluid.defaults("gpii.flowManager.local", { type: "gpii.userErrors" }, installOnDemand: { - type: "gpii.iod", + type: "gpii.iod" } }, requestHandlers: { diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/installOnDemand/src/installOnDemand.js index 3c96c6c70..78a571856 100644 --- a/gpii/node_modules/installOnDemand/src/installOnDemand.js +++ b/gpii/node_modules/installOnDemand/src/installOnDemand.js @@ -34,14 +34,14 @@ require("./iodSettingsHandler.js"); /** * Information about a package. - * @typedef {Object} packageInfo + * @typedef {Object} PackageInfo * @property {string} name - The package name. * @property {string} url - The package location. * @property {string} filename - The package filename. * @property {string} packageType - Type of installer to use. * * Installation state. - * @typedef {Object} installation + * @typedef {Object} Installation * @property {id} - Installation ID * @property {packageInfo} packageInfo - Package data. * @property {string} packageName - packageInfo.name @@ -170,8 +170,7 @@ fluid.defaults("gpii.iod.packageDataSource", { * All installed packages from the previous instance will be set for removal. * * @param {Component} that The gpii.iod instance. - * @param {string} directory The directory containing the stored installation info. - * @param {bool} justRead true to only read the packages, and not mark them for removal. + * @param {String} directory The directory containing the stored installation info. * @return {Promise} Resolves when complete. */ gpii.iod.readInstallations = function (that, directory) { @@ -215,8 +214,9 @@ gpii.iod.readInstallations = function (that, directory) { * If the installation has been removed, then the file will be deleted. * * @param {Component} that The gpii.iod instance. - * @param {string} directory The directory containing the stored installation info. - * @return {string} The file that the installation data was written to. + * @param {String} directory The directory containing the stored installation info. + * @param {Installation} installation The installation state. + * @return {String} The file that the installation data was written to. */ gpii.iod.writeInstallation = function (that, directory, installation) { var filename = path.join(directory, "iod-installation." + installation.id + ".json"); @@ -278,8 +278,8 @@ gpii.iod.getWorkingPath = function (packageName) { * Finds a package installer component that handles the given type of package. * * @param {Component} that The gpii.iod instance. - * @param {string} packageType The package type identifier. - * @return {string} The grade name of the package installer. + * @param {String} packageType The package type identifier. + * @return {String} The grade name of the package installer. */ gpii.iod.getInstaller = function (that, packageType) { var packageInstallers = fluid.queryIoCSelector(that, "gpii.iod.packageInstaller"); @@ -298,9 +298,9 @@ gpii.iod.getInstaller = function (that, packageType) { * Starts the process of installing a package. * * @param {Component} that The gpii.iod instance. - * @param {string|Object} packageRequest Package name, or object containing packageName, language, version. - * @param packageRequest.packageName {string} Name of the package. - * @param packageRequest.language {string|string[]} Language. + * @param {String|Object} packageRequest Package name, or object containing packageName, language, version. + * @param {String} packageRequest.packageName Name of the package. + * @param {String|String[]} packageRequest.language Language. * @return {Promise} Resolves when the installation is complete. */ gpii.iod.requirePackage = function (that, packageRequest) { @@ -385,9 +385,9 @@ gpii.iod.initialiseInstallation = function (that, packageRequest) { * * @param {Component} that The gpii.iod instance. * @param {Object} packageRequest Containing packageName, language, version. - * @param packageRequest.packageName {string} Name of the package. - * @param packageRequest.version {string} [optional] Version. - * @param packageRequest.language {string} [optional] Language code with optional country code (en, en-US, es-ES). + * @param {String} packageRequest.packageName Name of the package. + * @param {String} packageRequest.version [optional] Version. + * @param {String} packageRequest.language [optional] Language code with optional country code (en, en-US, es-ES). * @return {Promise} Resolves to an object containing package information. */ gpii.iod.getPackageInfo = function (that, packageRequest) { @@ -437,9 +437,9 @@ gpii.iod.getPackageInfo = function (that, packageRequest) { * - Exact match without country code * - First language, ignoring country code. * - * @param {string[]} languages The list of available languages, with optional country code (en, en-US, es-ES) - * @param {string} language The preferred language. - * @return {string} The closest matching item from languages. + * @param {Array} languages The list of available languages, with optional country code (en, en-US, es-ES) + * @param {String} language The preferred language. + * @return {String} The closest matching item from languages. */ gpii.iod.matchLanguage = function (languages, language) { languages = fluid.makeArray(languages); @@ -472,7 +472,7 @@ gpii.iod.matchLanguage = function (languages, language) { * session. * * @param {Component} that The gpii.iod instance. - * @param {string} packageName The name of the package to no longer require. + * @param {String} packageName The name of the package to no longer require. */ gpii.iod.unrequirePackage = function (that, packageName) { @@ -491,7 +491,7 @@ gpii.iod.unrequirePackage = function (that, packageName) { * a short time if there is no active session, to avoid giving the computer too much to do while it's in use. * * @param {Component} that The gpii.iod instance. - * @param {number} wait Number of seconds to wait until uninstalling (default: 30). + * @param {Number} wait Number of seconds to wait until uninstalling (default: 30). */ gpii.iod.uninstallPackages = function (that, wait) { @@ -520,7 +520,7 @@ gpii.iod.uninstallPackages = function (that, wait) { * Uninstall a package. * * @param {Component} that The gpii.iod instance. - * @param {object|string} The installation state, or installation ID. + * @param {Installation|String} installation The installation state, or installation ID. * @return {Promise} Resolves when the package is removed. */ gpii.iod.uninstallPackage = function (that, installation) { @@ -617,8 +617,8 @@ gpii.iod.discoverServer = function (that) { /** * Check if an endpoint is listening for connections. * - * @param endpoint - * @return {Promise} + * @param {String} endpoint The service end point URI + * @return {Promise} Resolves */ gpii.iod.checkService = function (endpoint) { var promise = fluid.promise(); @@ -637,7 +637,7 @@ gpii.iod.checkService = function (endpoint) { * Invoked when the service endpoint is down. * * @param {Component} that The gpii.iod instance. - * @param {string} endPoint The endpoint address. + * @param {String} endPoint The endpoint address. */ gpii.iod.serviceLost = function (that, endPoint) { fluid.log("IoD: Endpoint lost: " + endPoint); @@ -648,7 +648,7 @@ gpii.iod.serviceLost = function (that, endPoint) { * Invoked when a service endpoint is up. * * @param {Component} that The gpii.iod instance. - * @param {string} endPoint The endpoint address. + * @param {String} endPoint The endpoint address. */ gpii.iod.serviceFound = function (that, endPoint) { fluid.log("IoD: Endpoint found: " + endPoint); @@ -657,6 +657,8 @@ gpii.iod.serviceFound = function (that, endPoint) { /** * A bad way of checking if a package is installed. + * @param {String} packageName Package name + * @return {Boolean} true if the package is installed. */ gpii.iod.isInstalled = function (packageName) { return fs.existsSync(path.join(process.env.ProgramData, "chocolatey\\lib", packageName)); diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/installOnDemand/src/packageInstaller.js index 5fdc61ef3..7e9f37d33 100644 --- a/gpii/node_modules/installOnDemand/src/packageInstaller.js +++ b/gpii/node_modules/installOnDemand/src/packageInstaller.js @@ -141,7 +141,6 @@ gpii.iod.startInstaller = function (that) { * * @param {Component} that The gpii.iod.installer instance. * @param {Component} iod The gpii.iod instance. - * @return {Promise} Resolves when complete. */ gpii.iod.initialise = function (that, iod) { var tempDir = iod.getWorkingPath(that.packageInfo.name); @@ -153,7 +152,7 @@ gpii.iod.initialise = function (that, iod) { * Downloads a package from the server. * * @param {Component} that The gpii.iod.installer instance. - * @param {object} iod The gpii.iod instance. + * @param {Object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.downloadPackage = function (that) { @@ -186,8 +185,8 @@ gpii.iod.downloadPackage = function (that) { /** * Downloads a file, trying extra hard to use only https. * - * @param {string} url The remote uri. - * @param {string} localPath Destination path. + * @param {String} url The remote uri. + * @param {String} localPath Destination path. * @return {Promise} Resolves when done. */ gpii.iod.httpsDownload = function (url, localPath) { @@ -244,33 +243,41 @@ gpii.iod.httpsDownload = function (url, localPath) { * Checks that a downloaded package is ok. * * @param {Component} that The gpii.iod.installer instance. - * @param {object} iod The gpii.iod instance. + * @param {Object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.checkPackage = function (that) { + var promise = fluid.promise(); fluid.log("IoD: Checking downloaded package file " + that.packageInfo.filename); // TODO: It shouldn't be checked here - another process may over-write it before the high privilege executes it. // Instead, take ownership then check the integrity in the same context as it's being ran. + promise.resolve(); + return promise; }; /** * Generate the installation instructions. * * @param {Component} that The gpii.iod.installer instance. - * @param {object} iod The gpii.iod instance. + * @param {Object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.prepareInstall = function (that) { + var promise = fluid.promise(); fluid.log("IoD: Preparing installation for " + that.packageInfo.name); + promise.resolve(); + return promise; }; /** * Cleans up things that are no longer required. * * @param {Component} that The gpii.iod.installer instance. - * @param {object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ gpii.iod.cleanup = function (that) { + var promise = fluid.promise(); fluid.log("IoD: Cleaning installation of " + that.packageInfo.name); + promise.resolve(); + return promise; }; diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js index da438e0da..7c383185d 100644 --- a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js +++ b/gpii/node_modules/installOnDemand/test/installOnDemandTests.js @@ -342,7 +342,7 @@ fluid.defaults("gpii.tests.iod.testInstallerFail", { * Test function for packageInstaller, to check if a certain function has been called. * @param {Component} that The gpii.tests.iod.testInstaller1 instance. * @param {Component} iod The gpii.test.iod instance. - * @param {string} funcName Name of the function that id being tests. + * @param {String} funcName Name of the function that id being tests. * @return {Promise} A resolved promise. */ gpii.tests.iod.testInstaller1.testFunctionCalled = function (that, iod, funcName) { diff --git a/gpii/node_modules/installOnDemand/test/packageInstallerTests.js b/gpii/node_modules/installOnDemand/test/packageInstallerTests.js index d59147d32..e3d97fc40 100644 --- a/gpii/node_modules/installOnDemand/test/packageInstallerTests.js +++ b/gpii/node_modules/installOnDemand/test/packageInstallerTests.js @@ -241,4 +241,3 @@ jqUnit.asyncTest("test https download", function () { nextTest(); }); - diff --git a/testData/solutions/win32.json5 b/testData/solutions/win32.json5 index eeb2c6229..7e3290a92 100644 --- a/testData/solutions/win32.json5 +++ b/testData/solutions/win32.json5 @@ -23,10 +23,10 @@ "transform": { "type": "fluid.transforms.literalValue", "input": "hello", - "outputPath": "a", + "outputPath": "a" } } - }, + } } }, "isInstalled": [ diff --git a/tests/all-tests.js b/tests/all-tests.js index 7dfbb8f68..b3f85bc5f 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -71,7 +71,7 @@ var testIncludes = [ "../gpii/node_modules/settingsHandlers/test/settingsHandlerUtilitiesTests.js", "../gpii/node_modules/singleInstance/test/SingleInstanceTests.js", "../gpii/node_modules/userListeners/test/all-tests.js", - "../gpii/node_modules/gpii-ini-file/test/iniFileTests.js" + "../gpii/node_modules/gpii-ini-file/test/iniFileTests.js", "../gpii/node_modules/installOnDemand/test/installOnDemandTests.js" ]; From d810e098be75dbc5ebb53c1ceefef8a578d0ab5d Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 22 Aug 2018 16:52:55 +0100 Subject: [PATCH 22/56] GPII-2971: JSON5 --- .../installOnDemand/{demo-package.json => demo-package.json5} | 0 testData/installOnDemand/{local.json => local.json5} | 0 testData/installOnDemand/{wget.json => wget.json5} | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename testData/installOnDemand/{demo-package.json => demo-package.json5} (100%) rename testData/installOnDemand/{local.json => local.json5} (100%) rename testData/installOnDemand/{wget.json => wget.json5} (100%) diff --git a/testData/installOnDemand/demo-package.json b/testData/installOnDemand/demo-package.json5 similarity index 100% rename from testData/installOnDemand/demo-package.json rename to testData/installOnDemand/demo-package.json5 diff --git a/testData/installOnDemand/local.json b/testData/installOnDemand/local.json5 similarity index 100% rename from testData/installOnDemand/local.json rename to testData/installOnDemand/local.json5 diff --git a/testData/installOnDemand/wget.json b/testData/installOnDemand/wget.json5 similarity index 100% rename from testData/installOnDemand/wget.json rename to testData/installOnDemand/wget.json5 From de3398d8287f4fb21899e86eaec481e1a512f7ed Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 11 Sep 2018 12:39:28 +0100 Subject: [PATCH 23/56] GPII-2971: Renamed IoD module name to gpii-iod --- gpii/configs/gpii.config.development.base.local.json5 | 2 +- gpii/node_modules/{installOnDemand => gpii-iod}/README.md | 0 .../configs/gpii.iod.config.base.json | 0 .../configs/gpii.iod.config.development.json | 0 .../configs/gpii.iod.config.local.base.json | 0 .../configs/gpii.iod.config.remote.base.json | 0 gpii/node_modules/{installOnDemand => gpii-iod}/index.js | 2 +- gpii/node_modules/{installOnDemand => gpii-iod}/package.json | 2 +- .../{installOnDemand => gpii-iod}/src/installOnDemand.js | 0 .../{installOnDemand => gpii-iod}/src/iodSettingsHandler.js | 0 .../{installOnDemand => gpii-iod}/src/packageInstaller.js | 0 .../{installOnDemand => gpii-iod}/test/installOnDemandTests.js | 0 .../{installOnDemand => gpii-iod}/test/packageInstallerTests.js | 0 .../test/testPackages/failInstall.json | 0 .../test/testPackages/languages.json | 0 .../test/testPackages/package1.json | 0 .../test/testPackages/package2.json | 0 .../test/testPackages/unknownType.json | 0 index.js | 2 +- tests/all-tests.js | 2 +- 20 files changed, 5 insertions(+), 5 deletions(-) rename gpii/node_modules/{installOnDemand => gpii-iod}/README.md (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/configs/gpii.iod.config.base.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/configs/gpii.iod.config.development.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/configs/gpii.iod.config.local.base.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/configs/gpii.iod.config.remote.base.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/index.js (92%) rename gpii/node_modules/{installOnDemand => gpii-iod}/package.json (91%) rename gpii/node_modules/{installOnDemand => gpii-iod}/src/installOnDemand.js (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/src/iodSettingsHandler.js (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/src/packageInstaller.js (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/installOnDemandTests.js (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/packageInstallerTests.js (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/testPackages/failInstall.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/testPackages/languages.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/testPackages/package1.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/testPackages/package2.json (100%) rename gpii/node_modules/{installOnDemand => gpii-iod}/test/testPackages/unknownType.json (100%) diff --git a/gpii/configs/gpii.config.development.base.local.json5 b/gpii/configs/gpii.config.development.base.local.json5 index aaf123763..5989bd9d8 100644 --- a/gpii/configs/gpii.config.development.base.local.json5 +++ b/gpii/configs/gpii.config.development.base.local.json5 @@ -29,6 +29,6 @@ "%flowManager/configs/gpii.flowManager.config.local.base.json5", "%preferencesServer/configs/gpii.preferencesServer.config.base.json5", "%canopyMatchMaker/configs/gpii.canopyMatchMaker.config.base.json5", - "%installOnDemand/configs/gpii.iod.config.development.json" + "%gpii-iod/configs/gpii.iod.config.development.json" ] } diff --git a/gpii/node_modules/installOnDemand/README.md b/gpii/node_modules/gpii-iod/README.md similarity index 100% rename from gpii/node_modules/installOnDemand/README.md rename to gpii/node_modules/gpii-iod/README.md diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.base.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.base.json similarity index 100% rename from gpii/node_modules/installOnDemand/configs/gpii.iod.config.base.json rename to gpii/node_modules/gpii-iod/configs/gpii.iod.config.base.json diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json similarity index 100% rename from gpii/node_modules/installOnDemand/configs/gpii.iod.config.development.json rename to gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.local.base.json similarity index 100% rename from gpii/node_modules/installOnDemand/configs/gpii.iod.config.local.base.json rename to gpii/node_modules/gpii-iod/configs/gpii.iod.config.local.base.json diff --git a/gpii/node_modules/installOnDemand/configs/gpii.iod.config.remote.base.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.remote.base.json similarity index 100% rename from gpii/node_modules/installOnDemand/configs/gpii.iod.config.remote.base.json rename to gpii/node_modules/gpii-iod/configs/gpii.iod.config.remote.base.json diff --git a/gpii/node_modules/installOnDemand/index.js b/gpii/node_modules/gpii-iod/index.js similarity index 92% rename from gpii/node_modules/installOnDemand/index.js rename to gpii/node_modules/gpii-iod/index.js index f002bacb3..86b7160be 100644 --- a/gpii/node_modules/installOnDemand/index.js +++ b/gpii/node_modules/gpii-iod/index.js @@ -20,7 +20,7 @@ var fluid = require("infusion"); -fluid.module.register("installOnDemand", __dirname, require); +fluid.module.register("gpii-iod", __dirname, require); require("./src/installOnDemand.js"); require("./src/packageInstaller.js"); diff --git a/gpii/node_modules/installOnDemand/package.json b/gpii/node_modules/gpii-iod/package.json similarity index 91% rename from gpii/node_modules/installOnDemand/package.json rename to gpii/node_modules/gpii-iod/package.json index 5781321d0..1bc606603 100644 --- a/gpii/node_modules/installOnDemand/package.json +++ b/gpii/node_modules/gpii-iod/package.json @@ -1,5 +1,5 @@ { - "name": "installOnDemand", + "name": "gpii-iod", "description": "Install on Demand", "version": "0.3.0", "author": "GPII", diff --git a/gpii/node_modules/installOnDemand/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js similarity index 100% rename from gpii/node_modules/installOnDemand/src/installOnDemand.js rename to gpii/node_modules/gpii-iod/src/installOnDemand.js diff --git a/gpii/node_modules/installOnDemand/src/iodSettingsHandler.js b/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js similarity index 100% rename from gpii/node_modules/installOnDemand/src/iodSettingsHandler.js rename to gpii/node_modules/gpii-iod/src/iodSettingsHandler.js diff --git a/gpii/node_modules/installOnDemand/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js similarity index 100% rename from gpii/node_modules/installOnDemand/src/packageInstaller.js rename to gpii/node_modules/gpii-iod/src/packageInstaller.js diff --git a/gpii/node_modules/installOnDemand/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js similarity index 100% rename from gpii/node_modules/installOnDemand/test/installOnDemandTests.js rename to gpii/node_modules/gpii-iod/test/installOnDemandTests.js diff --git a/gpii/node_modules/installOnDemand/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js similarity index 100% rename from gpii/node_modules/installOnDemand/test/packageInstallerTests.js rename to gpii/node_modules/gpii-iod/test/packageInstallerTests.js diff --git a/gpii/node_modules/installOnDemand/test/testPackages/failInstall.json b/gpii/node_modules/gpii-iod/test/testPackages/failInstall.json similarity index 100% rename from gpii/node_modules/installOnDemand/test/testPackages/failInstall.json rename to gpii/node_modules/gpii-iod/test/testPackages/failInstall.json diff --git a/gpii/node_modules/installOnDemand/test/testPackages/languages.json b/gpii/node_modules/gpii-iod/test/testPackages/languages.json similarity index 100% rename from gpii/node_modules/installOnDemand/test/testPackages/languages.json rename to gpii/node_modules/gpii-iod/test/testPackages/languages.json diff --git a/gpii/node_modules/installOnDemand/test/testPackages/package1.json b/gpii/node_modules/gpii-iod/test/testPackages/package1.json similarity index 100% rename from gpii/node_modules/installOnDemand/test/testPackages/package1.json rename to gpii/node_modules/gpii-iod/test/testPackages/package1.json diff --git a/gpii/node_modules/installOnDemand/test/testPackages/package2.json b/gpii/node_modules/gpii-iod/test/testPackages/package2.json similarity index 100% rename from gpii/node_modules/installOnDemand/test/testPackages/package2.json rename to gpii/node_modules/gpii-iod/test/testPackages/package2.json diff --git a/gpii/node_modules/installOnDemand/test/testPackages/unknownType.json b/gpii/node_modules/gpii-iod/test/testPackages/unknownType.json similarity index 100% rename from gpii/node_modules/installOnDemand/test/testPackages/unknownType.json rename to gpii/node_modules/gpii-iod/test/testPackages/unknownType.json diff --git a/index.js b/index.js index b86d724d0..02feba5c1 100644 --- a/index.js +++ b/index.js @@ -40,7 +40,7 @@ require("./gpii/node_modules/processReporter"); require("./gpii/node_modules/gpii-db-operation"); require("./gpii/node_modules/userListeners"); require("./gpii/node_modules/gpii-ini-file"); -require("./gpii/node_modules/installOnDemand"); +require("./gpii/node_modules/gpii-iod"); gpii.loadTestingSupport = function () { fluid.contextAware.makeChecks({ diff --git a/tests/all-tests.js b/tests/all-tests.js index 05b48648f..27f0f6334 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -73,7 +73,7 @@ var testIncludes = [ "../gpii/node_modules/singleInstance/test/SingleInstanceTests.js", "../gpii/node_modules/userListeners/test/all-tests.js", "../gpii/node_modules/gpii-ini-file/test/iniFileTests.js", - "../gpii/node_modules/installOnDemand/test/installOnDemandTests.js" + "../gpii/node_modules/gpii-iod/test/installOnDemandTests.js" ]; fluid.each(testIncludes, function (path) { From c985951c5edf5b22f210b29ea67f7464aec4cac7 Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 11 Sep 2018 13:16:34 +0100 Subject: [PATCH 24/56] GPII-2971: Disable IoD server auto detection by default --- gpii/node_modules/gpii-iod/src/installOnDemand.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 78a571856..0bc94a22a 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -572,9 +572,7 @@ gpii.iod.discoverServer = function (that) { var addr = process.env.GPII_IOD_ENDPOINT; - if (addr) { - gpii.iod.checkService(addr).then(that.events.onServiceFound.fire); - } else { + if (addr === "auto") { var bonjour = that.bonjourInstance || (that.bonjourInstance = require("bonjour")()); if (bonjour) { var browser = bonjour.find({type: "gpii-iod"}); @@ -609,6 +607,8 @@ gpii.iod.discoverServer = function (that) { }, 5000); } } + } else if (addr) { + gpii.iod.checkService(addr).then(that.events.onServiceFound.fire); } that.endpoint = addr; From 5d5e7d5628342b5be7f8bda06dce4d052da418e3 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 13 Sep 2018 20:18:33 +0100 Subject: [PATCH 25/56] GPII-2971: Start detecting if a package is already installed --- .../gpii-iod/src/iodSettingsHandler.js | 78 +++++++++---------- testData/solutions/win32.json5 | 3 + 2 files changed, 42 insertions(+), 39 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js b/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js index 582de7d4f..7ffc306e1 100644 --- a/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js +++ b/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js @@ -35,52 +35,52 @@ gpii.iod.settingsHandler.setImpl = function (payload) { }); var iod = fluid.queryIoCSelector(fluid.rootComponent, "gpii.iod")[0]; - var promise = fluid.promise(); + var results = {}; - var packageKeys = Object.keys(packages); - - var nextPackage = function () { - if (packageKeys.length === 0) { - promise.resolve(results); - } else { - var key = packageKeys.shift(); - var packageRequest = packages[key]; - - var isInstalled = iod.isInstalled(packageRequest.packageName); - - var p; - results[key] = { - oldValue: { - uninstall: true, - installed: isInstalled - }, - newValue: { - } - }; - - if (packageRequest.uninstall) { - iod.unrequirePackage(packageRequest.packageName); - } else if (!isInstalled) { - p = iod.requirePackage(packageRequest); - p.then(function () { - results[key].newValue.installed = true; - }, function (error) { - fluid.log(error); - results[key].newValue.installed = false; - }); + + var packagePromises = []; + + fluid.each(Object.keys(packages), function (packageKey) { + var packageRequest = packages[packageKey]; + + // Check if it's already installed + var packageInstalled = fluid.makeArray(packageRequest.isInstalled).every(function (isInstalled) { + return fluid.invokeGradedFunction(isInstalled.type, isInstalled); + }); + + results[packageKey] = { + oldValue: { + installed: packageInstalled + }, + newValue: { } + }; - fluid.toPromise(p).then(nextPackage, nextPackage); - } - }; + var p = fluid.promise(); - // For some reason the datasource.get call inside requirePackage doesn't resolve while it's inside this stack. - // A better developer would have discovered why, but you have to make do with what you've got. - process.nextTick(nextPackage); + // For some reason the datasource.get call inside requirePackage doesn't resolve while it's inside this stack. + // A better developer would have discovered why, but you have to make do with what you've got. + process.nextTick(function () { + if (packageInstalled) { + p.resolve(); + } else { + fluid.promise.follow(iod.requirePackage(packageRequest), p); + } + }); + + p.then(function () { + results[packageKey].newValue.installed = true; + }, function (error) { + fluid.log(error); + results[packageKey].newValue.installed = false; + }); + packagePromises.push(p); - return promise; + }); + + return fluid.promise.sequence(packagePromises); }; gpii.iod.settingsHandler.get = function (payload) { diff --git a/testData/solutions/win32.json5 b/testData/solutions/win32.json5 index 7d3a6cc6b..260e20228 100644 --- a/testData/solutions/win32.json5 +++ b/testData/solutions/win32.json5 @@ -15,6 +15,9 @@ "capabilities": [], "options": { "package1": { + "isInstalled": { + "type": "gpii.deviceReporter.alwaysInstalled" + }, "packageName": "demo-package" } }, From d5f2d628efc5ec6ab099e7be2901c7cbf61433f0 Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 6 Sep 2019 21:43:16 +0100 Subject: [PATCH 26/56] GPII-2971: IoD install/uninstall on key-in/out --- .../flowManager/src/FlowManager.js | 5 +- gpii/node_modules/gpii-iod/README.md | 6 +- .../configs/gpii.iod.config.development.json | 6 +- .../configs/gpii.iod.config.local.base.json | 12 +- .../configs/gpii.iod.config.remote.base.json | 3 +- gpii/node_modules/gpii-iod/index.js | 2 + .../gpii-iod/src/installOnDemand.js | 219 +++++------------ .../gpii-iod/src/iodSettingsHandler.js | 52 ++-- gpii/node_modules/gpii-iod/src/packages.js | 160 ++++++++++++ .../gpii-iod/test/installOnDemandTests.js | 229 ++---------------- .../gpii-iod/test/packageInstallerTests.js | 13 +- .../gpii-iod/test/packagesTests.js | 220 +++++++++++++++++ testData/installOnDemand/demo-package.json5 | 5 +- testData/solutions/win32.json5 | 22 +- 14 files changed, 530 insertions(+), 424 deletions(-) create mode 100644 gpii/node_modules/gpii-iod/src/packages.js create mode 100644 gpii/node_modules/gpii-iod/test/packagesTests.js diff --git a/gpii/node_modules/flowManager/src/FlowManager.js b/gpii/node_modules/flowManager/src/FlowManager.js index 1755a8e1f..7cc7c3655 100644 --- a/gpii/node_modules/flowManager/src/FlowManager.js +++ b/gpii/node_modules/flowManager/src/FlowManager.js @@ -188,7 +188,10 @@ fluid.defaults("gpii.flowManager.local", { type: "gpii.userErrors" }, installOnDemand: { - type: "gpii.iod" + type: "gpii.iod", + options: { + gradeNames: ["gpii.iodLifeCycleManager"] + } } }, requestHandlers: { diff --git a/gpii/node_modules/gpii-iod/README.md b/gpii/node_modules/gpii-iod/README.md index 318ec4caf..fd07222e8 100644 --- a/gpii/node_modules/gpii-iod/README.md +++ b/gpii/node_modules/gpii-iod/README.md @@ -1,6 +1,6 @@ # Install on Demand -Provides the ability to install software on demand. +Provides the ability to install software based on the requirements of a user. [Technical Design](https://tinyurl.com/y7xhcghu) (google doc) @@ -9,6 +9,7 @@ Provides the ability to install software on demand. ### GPII Start * IoD server is detected using mDNS (or configured server, or local datasource) +* Package list is taken from the IoD server. * Installation information about packages that *GPII* has installed but not removed, is loaded (if any). * These packages are uninstalled. @@ -41,6 +42,9 @@ The stages of installation: The install on demand component. +### `gpii.iod.packages` + + ### `gpii.iod.packageInstaller` Base component of the package installers, which perform the work that's specific to the type of package being installed. diff --git a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json index 35ec9612a..197a886e5 100644 --- a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json +++ b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json @@ -2,11 +2,11 @@ "type": "gpii.iod.config.development", "options": { "distributeOptions": { - "iod.local": { + "packageData.dev": { "record": { - "defaultEndpoint": "http://localhost:8087" + "endpoint": "http://localhost:8087" }, - "target": "{that iod}.options" + "target": "{that gpii.iod}.options" } } }, diff --git a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.local.base.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.local.base.json index 5cf731203..108799589 100644 --- a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.local.base.json +++ b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.local.base.json @@ -2,19 +2,15 @@ "type": "gpii.iod.config.local.base", "options": { "distributeOptions": { - "packageDataFallback.file": { + "packageData.local": { "record": { - "gradeNames": "kettle.dataSource.file", - "path": "%gpii-universal/testData/installOnDemand/%packageName.json", + "gradeNames": [ "kettle.dataSource.file.moduleTerms" ], + "path": "%gpii-universal/testData/installOnDemand/%packageName.json5", "termMap": { "packageName": "%packageName" } }, - "target": "{that iod packageDataFallback}.options" - }, - "packageDataFallback.moduleTerms": { - "record": "kettle.dataSource.file.moduleTerms", - "target": "{that iod packageDataFallback}.options.gradeNames" + "target": "{that gpii.iod.packages packageData}.options" } } }, diff --git a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.remote.base.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.remote.base.json index 1753357b8..1ee1d867a 100644 --- a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.remote.base.json +++ b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.remote.base.json @@ -4,14 +4,13 @@ "distributeOptions": { "packageData.remote": { "record": { - "gradeNames": "kettle.dataSource.URL", "url": "%endpoint/packages/%packageName", "termMap": { "packageName": "%packageName", "endpoint": "noencode:%endpoint" } }, - "target": "{that iod packageData}.options" + "target": "{that gpii.iod.packages remotePackageData}.options" } } }, diff --git a/gpii/node_modules/gpii-iod/index.js b/gpii/node_modules/gpii-iod/index.js index 86b7160be..85f14a61d 100644 --- a/gpii/node_modules/gpii-iod/index.js +++ b/gpii/node_modules/gpii-iod/index.js @@ -22,5 +22,7 @@ var fluid = require("infusion"); fluid.module.register("gpii-iod", __dirname, require); +require("./src/iodSettingsHandler.js"); require("./src/installOnDemand.js"); require("./src/packageInstaller.js"); +require("./src/packages.js"); diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 0bc94a22a..03f94f9f2 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -29,17 +29,7 @@ var path = require("path"), var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.iod"); -require("./packageInstaller.js"); -require("./iodSettingsHandler.js"); - /** - * Information about a package. - * @typedef {Object} PackageInfo - * @property {string} name - The package name. - * @property {string} url - The package location. - * @property {string} filename - The package filename. - * @property {string} packageType - Type of installer to use. - * * Installation state. * @typedef {Object} Installation * @property {id} - Installation ID @@ -66,12 +56,13 @@ fluid.defaults("gpii.iod", { } }, components: { - "packageData": { - createOnEvent: "onServiceFound", - type: "gpii.iod.packageDataSource" - }, - "packageDataFallback": { - type: "gpii.iod.packageDataSource" + packages: { + type: "gpii.iod.packages", + options: { + events: { + "onServiceFound": "onServiceFound" + } + } } }, dynamicComponents: { @@ -92,8 +83,7 @@ fluid.defaults("gpii.iod", { "onCreate.discoverServer": "{that}.discoverServer", "onCreate.readInstallations": "{that}.readInstallations", "onServiceFound": "{that}.serviceFound", - "onServiceLost": "{that}.serviceLost", - "{lifecycleManager}.events.onSessionStop": "{that}.uninstallPackages" + "onServiceLost": "{that}.serviceLost" }, invokers: { discoverServer: { @@ -108,10 +98,6 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.initialiseInstallation", args: ["{that}", "{arguments}.0"] }, - getPackageInfo: { - funcName: "gpii.iod.getPackageInfo", - args: ["{that}", "{arguments}.0"] - }, getInstaller: { funcName: "gpii.iod.getInstaller", args: ["{that}", "{arguments}.0"] @@ -140,8 +126,8 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.unrequirePackage", args: ["{that}", "{arguments}.0"] }, - uninstallPackages: { - funcName: "gpii.iod.uninstallPackages", + uninitialiseInstallation: { + funcName: "gpii.iod.uninitialiseInstallation", args: ["{that}", "{arguments}.0"] }, uninstallPackage: { @@ -150,18 +136,23 @@ fluid.defaults("gpii.iod", { } }, - model: { - logonChange: "{lifecycleManager}.model.logonChange" - }, + endpoint: undefined, members: { installations: {} } }); -fluid.defaults("gpii.iod.packageDataSource", { +fluid.defaults("gpii.iodLifeCycleManager", { gradeNames: ["fluid.component"], - readOnlyGrade: "gpii.iod.packageDataSource" + listeners: { + "{lifecycleManager}.events.onSessionStop": "{that}.uninitialiseInstallation" + }, + + model: { + logonChange: "{lifecycleManager}.model.logonChange" + } + }); /** @@ -186,13 +177,18 @@ gpii.iod.readInstallations = function (that, directory) { }); } else { fluid.each(files, function (file) { - var content = fs.readFileSync(file); - var installation = JSON.parse(content); - if (installation && installation.id) { - installation.remove = true; - installation.uninstalling = false; - needRemove = needRemove || installation.remove; - that.installations[installation.id] = installation; + try { + var content = fs.readFileSync(file); + var installation = JSON.parse(content); + if (installation && installation.id) { + fluid.log("IoD: Existing installation file '" + file + "': ", installation); + installation.remove = true; + installation.uninstalling = false; + needRemove = needRemove || installation.remove; + that.installations[installation.id] = installation; + } + } catch (e) { + fluid.log("IoD: Error reading stored installation file '" + file + "': ", e); } }); process.nextTick(promise.resolve); @@ -202,7 +198,7 @@ gpii.iod.readInstallations = function (that, directory) { promise.then(function () { if (needRemove) { // Loaded some uninstalled installation. - that.uninstallPackages(0); + that.uninitialiseInstallation(0); } }); @@ -322,18 +318,18 @@ gpii.iod.requirePackage = function (that, packageRequest) { // Package is already installed by IoD. installation.remove = false; that.writeInstallation(installation); + promise.resolve(false); + } else { + that.initialiseInstallation(packageRequest).then(function (installation) { + installation.installer.startInstaller().then(function () { + installation.installed = true; + // Store the installation info so it can still get removed if gpii restarts. + that.writeInstallation(installation); + promise.resolve(true); + }, promise.reject); + }, promise.reject); } - that.initialiseInstallation(packageRequest).then(function (installation) { - var result = installation.installer.startInstaller().then(function () { - installation.installed = true; - // Store the installation info so it can still get removed if gpii restarts. - that.writeInstallation(installation); - }); - fluid.promise.follow(result, promise); - }, promise.reject); - - return promise.then(function () { fluid.log("IoD: Installation of " + packageRequest.packageName + " complete"); }, function (err) { @@ -361,7 +357,7 @@ gpii.iod.initialiseInstallation = function (that, packageRequest) { var promise = fluid.promise(); // Get the package info. - that.getPackageInfo(packageRequest).then(function (packageInfo) { + that.packages.getPackageInfo(packageRequest).then(function (packageInfo) { // Create the installer instance. installation.packageInfo = packageInfo; var installerGrade = that.getInstaller(packageInfo.packageType); @@ -380,99 +376,13 @@ gpii.iod.initialiseInstallation = function (that, packageRequest) { return promise; }; -/** - * Retrieve the package metadata. - * - * @param {Component} that The gpii.iod instance. - * @param {Object} packageRequest Containing packageName, language, version. - * @param {String} packageRequest.packageName Name of the package. - * @param {String} packageRequest.version [optional] Version. - * @param {String} packageRequest.language [optional] Language code with optional country code (en, en-US, es-ES). - * @return {Promise} Resolves to an object containing package information. - */ -gpii.iod.getPackageInfo = function (that, packageRequest) { - fluid.log("IoD: Getting package info for " + packageRequest.packageName); - - var promise = fluid.promise(); - - var dataSource = that.packageData || (that.packageDataFallback.options.path && that.packageDataFallback); - - if (dataSource) { - dataSource.get({ - packageName: packageRequest.packageName, - language: packageRequest.language, - version: packageRequest.version, - server: that.remoteServer - }).then(function (packageInfo) { - if (packageRequest.language && packageInfo.languages) { - // Merge the language-specific info. - var lang = gpii.iod.matchLanguage(Object.keys(packageInfo.languages), packageRequest.language); - if (lang) { - Object.assign(packageInfo, packageInfo.languages[lang]); - packageInfo.language = lang; - } - } - - promise.resolve(packageInfo); - }, function (err) { - promise.reject({ - isError: true, - message: "Unknown package " + packageRequest.packageName, - error: err - }); - }); - } else { - promise.reject({ - isError: true, - message: "No package data source for IoD" - }); - } - - return promise; -}; - -/** - * Finds the best language from a list of available languages, using the following priority: - * - Exact match with country code - * - Exact match without country code - * - First language, ignoring country code. - * - * @param {Array} languages The list of available languages, with optional country code (en, en-US, es-ES) - * @param {String} language The preferred language. - * @return {String} The closest matching item from languages. - */ -gpii.iod.matchLanguage = function (languages, language) { - languages = fluid.makeArray(languages); - - // Exact match. - var index = languages.indexOf(language); - var match = index >= 0 && languages[index]; - - if (!match) { - var langCode = language.substr(0, 2); - // Language without country. - if (language.length > 2) { - index = languages.indexOf(language); - match = index >= 0 && languages[index]; - } - - if (!match) { - // Ignore the country. - match = languages.find(function (lang) { - return lang.substr(0, 2) === langCode; - }); - } - } - - return match; -}; - /** * No longer require a package. This will cause the package to be uninstalled in a short-time if there is no active * session. * * @param {Component} that The gpii.iod instance. * @param {String} packageName The name of the package to no longer require. + * @return {Promise} Resolves immediately with a boolean indicating if the package was installed, and will be removed. */ gpii.iod.unrequirePackage = function (that, packageName) { @@ -484,6 +394,10 @@ gpii.iod.unrequirePackage = function (that, packageName) { installation.remove = true; that.writeInstallation(installation); } + + var promise = fluid.promise(); + promise.resolve(!!installation); + return promise; }; /** @@ -493,10 +407,10 @@ gpii.iod.unrequirePackage = function (that, packageName) { * @param {Component} that The gpii.iod instance. * @param {Number} wait Number of seconds to wait until uninstalling (default: 30). */ -gpii.iod.uninstallPackages = function (that, wait) { +gpii.iod.uninitialiseInstallation = function (that, wait) { var uninstall = function () { - var inSession = that.model.logonChange.inProgress && that.model.logonChange.type !== "login"; + var inSession = false;// that.model.logonChange.inProgress && that.model.logonChange.type !== "login"; if (!inSession) { var installation = fluid.find(that.installations, function (inst) { return inst.remove && !inst.removed ? inst : undefined; @@ -570,7 +484,7 @@ gpii.iod.uninstallPackage = function (that, installation) { */ gpii.iod.discoverServer = function (that) { - var addr = process.env.GPII_IOD_ENDPOINT; + var addr = process.env.GPII_IOD_ENDPOINT || that.options.endpoint; if (addr === "auto") { var bonjour = that.bonjourInstance || (that.bonjourInstance = require("bonjour")()); @@ -578,9 +492,8 @@ gpii.iod.discoverServer = function (that) { var browser = bonjour.find({type: "gpii-iod"}); browser.on("up", function (service) { fluid.log("IoD: Service up: " + service.fqdn); - if (that.endpoint && that.packageData) { + if (that.endpoint) { that.events.onServiceLost.fire(that.endpoint); - that.packageData.destroy(); } var endpoint = service.txt.url || ("https://" + service.host + ":" + service.port); gpii.iod.checkService(endpoint).then(that.events.onServiceFound.fire); @@ -592,7 +505,6 @@ gpii.iod.discoverServer = function (that) { var oldEndpoint = service.txt.url || ("https://" + service.host + ":" + service.port); if (oldEndpoint === that.endpoint) { that.events.onServiceLost.fire(that.endpoint); - that.packageData.destroy(); } } }); @@ -610,8 +522,6 @@ gpii.iod.discoverServer = function (that) { } else if (addr) { gpii.iod.checkService(addr).then(that.events.onServiceFound.fire); } - - that.endpoint = addr; }; /** @@ -637,10 +547,10 @@ gpii.iod.checkService = function (endpoint) { * Invoked when the service endpoint is down. * * @param {Component} that The gpii.iod instance. - * @param {String} endPoint The endpoint address. + * @param {String} endpoint The endpoint address. */ -gpii.iod.serviceLost = function (that, endPoint) { - fluid.log("IoD: Endpoint lost: " + endPoint); +gpii.iod.serviceLost = function (that, endpoint) { + fluid.log("IoD: Endpoint lost: " + endpoint); that.endpoint = null; }; @@ -648,18 +558,9 @@ gpii.iod.serviceLost = function (that, endPoint) { * Invoked when a service endpoint is up. * * @param {Component} that The gpii.iod instance. - * @param {String} endPoint The endpoint address. - */ -gpii.iod.serviceFound = function (that, endPoint) { - fluid.log("IoD: Endpoint found: " + endPoint); - that.endpoint = endPoint; -}; - -/** - * A bad way of checking if a package is installed. - * @param {String} packageName Package name - * @return {Boolean} true if the package is installed. + * @param {String} endpoint The endpoint address. */ -gpii.iod.isInstalled = function (packageName) { - return fs.existsSync(path.join(process.env.ProgramData, "chocolatey\\lib", packageName)); +gpii.iod.serviceFound = function (that, endpoint) { + fluid.log("IoD: Endpoint found: " + endpoint); + that.endpoint = endpoint; }; diff --git a/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js b/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js index 7ffc306e1..2a6e29018 100644 --- a/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js +++ b/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js @@ -30,9 +30,7 @@ gpii.iod.settingsHandler.getImpl = function () { }; gpii.iod.settingsHandler.setImpl = function (payload) { - var packages = fluid.transform(payload.options, function (value, key) { - return Object.assign({}, value, payload.settings[key]); - }); + var packages = payload.settings.packages ? fluid.makeArray(payload.settings.packages) : payload.settings; var iod = fluid.queryIoCSelector(fluid.rootComponent, "gpii.iod")[0]; @@ -41,46 +39,48 @@ gpii.iod.settingsHandler.setImpl = function (payload) { var packagePromises = []; fluid.each(Object.keys(packages), function (packageKey) { - var packageRequest = packages[packageKey]; + var packageRequest = packages[packageKey] || packageKey; + if (!packageRequest.packageName) { + packageRequest.packageName = packageKey; + } + + if (packageRequest.install === undefined) { + packageRequest.install = true; + } - // Check if it's already installed - var packageInstalled = fluid.makeArray(packageRequest.isInstalled).every(function (isInstalled) { - return fluid.invokeGradedFunction(isInstalled.type, isInstalled); - }); results[packageKey] = { - oldValue: { - installed: packageInstalled - }, - newValue: { - } + oldValue: {}, + newValue: {} }; - var p = fluid.promise(); + var installPromise = fluid.promise(); // For some reason the datasource.get call inside requirePackage doesn't resolve while it's inside this stack. // A better developer would have discovered why, but you have to make do with what you've got. process.nextTick(function () { - if (packageInstalled) { - p.resolve(); - } else { - fluid.promise.follow(iod.requirePackage(packageRequest), p); - } + var p = packageRequest.install + ? iod.requirePackage(packageRequest) + : iod.unrequirePackage(packageRequest.packageName); + fluid.promise.follow(p, installPromise); }); - p.then(function () { - results[packageKey].newValue.installed = true; + installPromise.then(function (installResult) { + results[packageKey].oldValue.install = packageRequest.install ? !installResult : installResult; + results[packageKey].newValue.install = packageRequest.install; }, function (error) { fluid.log(error); - results[packageKey].newValue.installed = false; + results[packageKey].newValue.install = false; }); - packagePromises.push(p); - - + packagePromises.push(installPromise); }); - return fluid.promise.sequence(packagePromises); + var promise = fluid.promise(); + fluid.promise.sequence(packagePromises).then(function () { + promise.resolve(results); + }, promise.reject); + return promise; }; gpii.iod.settingsHandler.get = function (payload) { diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js new file mode 100644 index 000000000..1ba5742d4 --- /dev/null +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -0,0 +1,160 @@ +/* + * Install on Demand packages. + * + * Copyright 2019 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); + +var path = require("path"), + fs = require("fs"); + +require("kettle"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.iod.packages"); + +require("./packageInstaller.js"); +require("./iodSettingsHandler.js"); + +/** + * Information about a package. + * @typedef {Object} PackageInfo + * @property {string} name - The package name. + * @property {string} url - The package location. + * @property {string} filename - The package filename. + * @property {string} packageType - Type of installer to use. + * + */ + +fluid.defaults("gpii.iod.packages", { + gradeNames: ["fluid.component"], + components: { + packageData: { + type: "kettle.dataSource.file" + }, + remotePackageData: { + createOnEvent: "onServiceFound", + type: "kettle.dataSource.URL" + } + }, + invokers: { + getPackageInfo: { + funcName: "gpii.iod.getPackageInfo", + args: ["{that}", "{arguments}.0"] + } + }, + listeners: { + onCreate: "fluid.identity" + } +}); + +/** + * Retrieve the package metadata. + * + * @param {Component} that The gpii.iod.packages instance. + * @param {Object} packageRequest Containing packageName, language, version. + * @param {String} packageRequest.packageName Name of the package. + * @param {String} packageRequest.version [optional] Version. + * @param {String} packageRequest.language [optional] Language code with optional country code (en, en-US, es-ES). + * @return {Promise} Resolves to an object containing package information. + */ +gpii.iod.getPackageInfo = function (that, packageRequest) { + fluid.log("IoD: Getting package info for " + packageRequest.packageName); + + var promise = fluid.promise(); + + var dataSource = that.remotePackageData || that.packageData; + + if (dataSource) { + dataSource.get({ + packageName: packageRequest.packageName, + language: packageRequest.language, + version: packageRequest.version, + server: that.remoteServer + }).then(function (packageInfo) { + if (packageRequest.language && packageInfo.languages) { + // Merge the language-specific info. + var lang = gpii.iod.matchLanguage(Object.keys(packageInfo.languages), packageRequest.language); + if (lang) { + Object.assign(packageInfo, packageInfo.languages[lang]); + packageInfo.language = lang; + } + } + + promise.resolve(packageInfo); + }, function (err) { + promise.reject({ + isError: true, + message: "Unknown package " + packageRequest.packageName, + error: err + }); + }); + } else { + promise.reject({ + isError: true, + message: "No package data source for IoD" + }); + } + + return promise; +}; + +/** + * Finds the best language from a list of available languages, using the following priority: + * - Exact match with country code + * - Exact match without country code + * - First language, ignoring country code. + * + * @param {Array} languages The list of available languages, with optional country code (en, en-US, es-ES) + * @param {String} language The preferred language. + * @return {String} The closest matching item from languages. + */ +gpii.iod.matchLanguage = function (languages, language) { + languages = fluid.makeArray(languages); + + // Exact match. + var index = languages.indexOf(language); + var match = index >= 0 && languages[index]; + + if (!match) { + var langCode = language.substr(0, 2); + // Language without country. + if (language.length > 2) { + index = languages.indexOf(language); + match = index >= 0 && languages[index]; + } + + if (!match) { + // Ignore the country. + match = languages.find(function (lang) { + return lang.substr(0, 2) === langCode; + }); + } + } + + return match; +}; + +/** + * A bad way of checking if a package is installed. + * @param {String} packageName Package name + * @return {Boolean} true if the package is installed. + */ +gpii.iod.isInstalled = function (packageName) { + return fs.existsSync(path.join(process.env.ProgramData, "chocolatey\\lib", packageName)); +}; diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 7c383185d..65e9a1416 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -69,144 +69,6 @@ gpii.tests.iod.getInstallerTests = fluid.freezeRecursive([ } ]); -gpii.tests.iod.getPackageInfoTests = fluid.freezeRecursive([ - { - id: "No matching package", - request: { - packageName: "package-not-exists" - }, - expect: "reject" - }, - { - id: "Single language package", - request: { - packageName: "package1" - }, - expect: require("./testPackages/package1.json") - }, - { - id: "Single language package, with language specified", - request: { - packageName: "package1", - language: "fr-FR" - }, - expect: require("./testPackages/package1.json") - }, - { - id: "Multi-language package, language not specified", - request: { - packageName: "languages" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "example.filename" - } - }, - { - id: "Multi-language package, unknown language specified", - request: { - packageName: "languages", - language: "xx-YY" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "example.filename" - } - }, - { - id: "Multi-language package, unknown language, no country specified", - request: { - packageName: "languages", - language: "xx" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "example.filename" - } - }, - { - id: "Multi-language package, full language specified", - request: { - packageName: "languages", - language: "es-ES" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es-es", - "language": "es-ES" - } - }, - { - id: "Multi-language package, full language specified 2", - request: { - packageName: "languages", - language: "es-MX" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es-mx", - "language": "es-MX" - } - }, - { - id: "Multi-language package, no country specified", - request: { - packageName: "languages", - language: "es" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es", - "language": "es" - } - }, - { - id: "Multi-language package, unknown country specified", - request: { - packageName: "languages", - language: "es-YY" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es", - "language": "es" - } - }, - { - id: "Multi-language package, no country specified, no non-country package", - request: { - packageName: "languages", - language: "zh" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.zh-cn", - "language": "zh-CN" - } - }, - { - id: "Multi-language package, unknown country specified, no non-country package", - request: { - packageName: "languages", - language: "zh-YY" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.zh-cn", - "language": "zh-CN" - } - } -]); - gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ { packageRequest: "no-such-package", @@ -272,8 +134,19 @@ fluid.defaults("gpii.tests.iod", { listeners: { "onCreate.discoverServer": null, - "onCreate.readInstallations": null, - "{lifecycleManager}.events.onSessionStop": null + "onCreate.readInstallations": null + }, + distributeOptions: { + packageData: { + record: { + gradeNames: ["kettle.dataSource.file.moduleTerms"], + path: __dirname + "/testPackages/%packageName.json", + termMap: { + "packageName": "%packageName" + } + }, + target: "{that packages packageData}.options" + } }, components: { "testInstaller1": { @@ -284,17 +157,6 @@ fluid.defaults("gpii.tests.iod", { }, "testInstallerFail": { type: "gpii.tests.iod.testInstallerFail" - }, - "packageDataFallback": { - createOnEvent: null, - type: "kettle.dataSource.file", - options: { - gradeNames: [ "kettle.dataSource.file.moduleTerms"], - path: __dirname + "/testPackages/%packageName.json", - termMap: { - "packageName": "%packageName" - } - } } }, invokers: { @@ -450,47 +312,6 @@ jqUnit.test("test getInstaller", function () { }); }); -// Test getPackageInfo returns correct information -jqUnit.asyncTest("test getPackageInfo", function () { - - var tests = gpii.tests.iod.getPackageInfoTests; - jqUnit.expect(tests.length * 2); - - var iod = gpii.tests.iod(); - - var testIndex = -1; - var nextTest = function () { - if (++testIndex >= tests.length) { - jqUnit.start(); - return; - } - - var test = tests[testIndex]; - var suffix = " - test:" + test.id; - - fluid.log("getPackage: " + test.request.packageName + ", " + test.request.language); - - var p = iod.getPackageInfo(test.request); - - jqUnit.assertTrue("getPackageInfo must return a promise" + suffix, fluid.isPromise(p)); - - p.then(function (packageInfo) { - delete packageInfo.languages; - jqUnit.assertDeepEq("packageInfo must match expected" + suffix, test.expect, packageInfo); - nextTest(); - }, function (e) { - if (test.expect !== "reject") { - fluid.log(e); - } - jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); - nextTest(); - }); - - }; - - nextTest(); -}); - // Test requirePackage correctly starts the installer. jqUnit.asyncTest("test requirePackage", function () { var tests = gpii.tests.iod.startInstallerTests; @@ -539,9 +360,9 @@ jqUnit.asyncTest("test installation storage", function () { var iod = gpii.tests.iod({ invokers: { - uninstallPackages: { + uninstallPackage: { funcName: "gpii.tests.iod.testInstaller1.testFunctionCalled", - args: ["{that}", "{iod}", "uninstallPackages"] + args: ["{that}", "{iod}", "uninstallPackage"] } } }); @@ -565,7 +386,7 @@ jqUnit.asyncTest("test installation storage", function () { expectRead: { id: "new-installation", remove: true, - uninstalling: false + uninstalling: true } }, updatedInst: { @@ -581,7 +402,7 @@ jqUnit.asyncTest("test installation storage", function () { id: "new-installation", test1: "something", remove: true, - uninstalling: false + uninstalling: true } } }; @@ -600,7 +421,7 @@ jqUnit.asyncTest("test installation storage", function () { jqUnit.assertDeepEq("readInstallation on an empty directory shouldn't change anything", origInstallations, iod.installations); - jqUnit.assertFalse("uninstallPackages should not have been called", iod.funcCalled.uninstallPackages); + jqUnit.assertFalse("uninstallPackage should not have been called", iod.funcCalled.uninstallPackage); // Write the installation data. var file = gpii.iod.writeInstallation(iod, dir, testData.newInst.input); @@ -620,8 +441,8 @@ jqUnit.asyncTest("test installation storage", function () { jqUnit.assertDeepEq("readInstallations should read the correct data", testData.newInst.expectRead, iod.installations[testData.newInst.input.id]); - jqUnit.assertTrue("uninstallPackages should have been called", !!iod.funcCalled.uninstallPackages); - iod.funcCalled.uninstallPackages = null; + jqUnit.assertTrue("uninstallPackage should have been called", !!iod.funcCalled.uninstallPackage); + iod.funcCalled.uninstallPackage = null; // Overwrite the existing file. var file = gpii.iod.writeInstallation(iod, dir, testData.updatedInst.input); @@ -637,8 +458,8 @@ jqUnit.asyncTest("test installation storage", function () { jqUnit.assertDeepEq("readInstallations should update with the correct data", testData.updatedInst.expectRead, iod.installations[testData.newInst.input.id]); - jqUnit.assertTrue("uninstallPackages should have been called again", - !!iod.funcCalled.uninstallPackages); + jqUnit.assertTrue("uninstallPackage should have been called again", + !!iod.funcCalled.uninstallPackage); jqUnit.start(); }); @@ -667,9 +488,9 @@ jqUnit.asyncTest("test uninstallation", function () { jqUnit.assertTrue("Package should have been set to be removed", installation.remove); - iod.uninstallPackages(0); + iod.uninstallPackage(packageName); - var retries = 10; + var retries = 100; var waitForRemoval = function () { // There's no promise or event, so just poll. if (iod.installations[installation.id]) { @@ -752,7 +573,7 @@ jqUnit.asyncTest("test uninstallation after restart", function () { jqUnit.assertTrue("Package should have been set to be removed", installation.remove); - iod.uninstallPackages(0); + iod.uninstallPackage(packageName); waitForUninstall().then(function () { // Fake a restart by creating a new instance of iod. diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index e3d97fc40..8cd26d896 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -49,10 +49,17 @@ fluid.defaults("gpii.tests.iod", { "testInstaller": { type: "gpii.tests.iod.installer" }, - "packageDataSource": { - type: "kettle.dataSource.file", + "packages": { + type: "gpii.iod.packages", options: { - path: __dirname + "/testPackages/%packageName.json" + components: { + "packageDataSource": { + type: "kettle.dataSource.file", + options: { + path: __dirname + "/testPackages/%packageName.json" + } + } + } } } } diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js new file mode 100644 index 000000000..4dd66ca2e --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -0,0 +1,220 @@ +/* + * IoD Tests - packages. + * + * Copyright 2019 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); +var kettle = fluid.require("kettle"); +kettle.loadTestingSupport(); + +var jqUnit = fluid.require("node-jqunit"); +var gpii = fluid.registerNamespace("gpii"); + +fluid.registerNamespace("gpii.tests.iod"); + +require("../index.js"); + +var teardowns = []; + +jqUnit.module("gpii.tests.iod", { + teardown: function () { + while (teardowns.length) { + teardowns.pop()(); + } + } +}); + + +gpii.tests.iod.getPackageInfoTests = fluid.freezeRecursive([ + { + id: "No matching package", + request: { + packageName: "package-not-exists" + }, + expect: "reject" + }, + { + id: "Single language package", + request: { + packageName: "package1" + }, + expect: require("./testPackages/package1.json") + }, + { + id: "Single language package, with language specified", + request: { + packageName: "package1", + language: "fr-FR" + }, + expect: require("./testPackages/package1.json") + }, + { + id: "Multi-language package, language not specified", + request: { + packageName: "languages" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, unknown language specified", + request: { + packageName: "languages", + language: "xx-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, unknown language, no country specified", + request: { + packageName: "languages", + language: "xx" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, full language specified", + request: { + packageName: "languages", + language: "es-ES" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es-es", + "language": "es-ES" + } + }, + { + id: "Multi-language package, full language specified 2", + request: { + packageName: "languages", + language: "es-MX" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es-mx", + "language": "es-MX" + } + }, + { + id: "Multi-language package, no country specified", + request: { + packageName: "languages", + language: "es" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es", + "language": "es" + } + }, + { + id: "Multi-language package, unknown country specified", + request: { + packageName: "languages", + language: "es-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es", + "language": "es" + } + }, + { + id: "Multi-language package, no country specified, no non-country package", + request: { + packageName: "languages", + language: "zh" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.zh-cn", + "language": "zh-CN" + } + }, + { + id: "Multi-language package, unknown country specified, no non-country package", + request: { + packageName: "languages", + language: "zh-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.zh-cn", + "language": "zh-CN" + } + } +]); + +// Test getPackageInfo returns correct information +jqUnit.asyncTest("test getPackageInfo", function () { + + var tests = gpii.tests.iod.getPackageInfoTests; + jqUnit.expect(tests.length * 2); + + var iod = gpii.tests.iod(); + + var testIndex = -1; + var nextTest = function () { + if (++testIndex >= tests.length) { + jqUnit.start(); + return; + } + + var test = tests[testIndex]; + var suffix = " - test:" + test.id; + + fluid.log("getPackage: " + test.request.packageName + ", " + test.request.language); + + var p = iod.getPackageInfo(test.request); + + jqUnit.assertTrue("getPackageInfo must return a promise" + suffix, fluid.isPromise(p)); + + p.then(function (packageInfo) { + delete packageInfo.languages; + jqUnit.assertDeepEq("packageInfo must match expected" + suffix, test.expect, packageInfo); + nextTest(); + }, function (e) { + if (test.expect !== "reject") { + fluid.log(e); + } + jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); + nextTest(); + }); + + }; + + nextTest(); +}); diff --git a/testData/installOnDemand/demo-package.json5 b/testData/installOnDemand/demo-package.json5 index 9e7dc24bf..e5589e375 100644 --- a/testData/installOnDemand/demo-package.json5 +++ b/testData/installOnDemand/demo-package.json5 @@ -3,5 +3,8 @@ "description": "Installs a small demo application", "url": "https://github.com/stegru/gpii-iod/raw/GPII-2972/testData/packages/demo-package.1.0.0.nupkg", "filename": "demo-package.1.0.0.nupkg", - "packageType": "chocolatey" + "packageType": "chocolatey", + "isInstalled": { + "installed": true + } } diff --git a/testData/solutions/win32.json5 b/testData/solutions/win32.json5 index 9e229d38f..67c1b9993 100644 --- a/testData/solutions/win32.json5 +++ b/testData/solutions/win32.json5 @@ -12,24 +12,14 @@ "settingsHandlers": { "install": { "type": "gpii.iod.settingsHandler", - "capabilities": [], + "capabilities": [ + "http://registry\.gpii\.net/applications/net\.gpii\.test\.iod" + ], "options": { - "package1": { - "isInstalled": { - "type": "gpii.deviceReporter.alwaysInstalled" - }, - "packageName": "demo-package" - } }, - "capabilitiesTransformations": { - "package1": { - "transform": { - "type": "fluid.transforms.literalValue", - "input": "hello", - "outputPath": "a" - } - } - } +// "capabilitiesTransformations": { +// "package1": "http://registry\\.gpii\\.net/applications/net\\.gpii\\.test\\.iod" +// } } }, "isInstalled": [ From 64baa0d2520ba3b2c391e857adf623935d25eb5d Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 11 Sep 2019 10:37:15 +0100 Subject: [PATCH 27/56] GPII-2971: Start/stop application --- .../gpii-iod/src/installOnDemand.js | 70 +++++++++-------- .../gpii-iod/src/packageInstaller.js | 78 ++++++++++++++++++- gpii/node_modules/gpii-iod/src/packages.js | 9 ++- testData/installOnDemand/nvda.json5 | 15 ++++ 4 files changed, 138 insertions(+), 34 deletions(-) create mode 100644 testData/installOnDemand/nvda.json5 diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 03f94f9f2..af8baad86 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -182,9 +182,9 @@ gpii.iod.readInstallations = function (that, directory) { var installation = JSON.parse(content); if (installation && installation.id) { fluid.log("IoD: Existing installation file '" + file + "': ", installation); - installation.remove = true; + installation.required = false; installation.uninstalling = false; - needRemove = needRemove || installation.remove; + needRemove = needRemove || !installation.required; that.installations[installation.id] = installation; } } catch (e) { @@ -218,7 +218,11 @@ gpii.iod.writeInstallation = function (that, directory, installation) { var filename = path.join(directory, "iod-installation." + installation.id + ".json"); if (installation.removed) { - fs.unlinkSync(filename); + try { + fs.unlinkSync(filename); + } catch (e) { + // ignore + } } else { // Don't write the installer component. var out = Object.assign({}, installation); @@ -310,25 +314,20 @@ gpii.iod.requirePackage = function (that, packageRequest) { var promise = fluid.promise(); - var installation = fluid.find(that.installations, function (inst) { - return inst.packageName === packageRequest.packageName ? inst : undefined; - }); - - if (installation) { - // Package is already installed by IoD. - installation.remove = false; - that.writeInstallation(installation); - promise.resolve(false); - } else { - that.initialiseInstallation(packageRequest).then(function (installation) { - installation.installer.startInstaller().then(function () { - installation.installed = true; - // Store the installation info so it can still get removed if gpii restarts. - that.writeInstallation(installation); - promise.resolve(true); - }, promise.reject); + that.initialiseInstallation(packageRequest).then(function (installation) { + installation.required = true; + + installation.installer.startInstaller().then(function () { + installation.installed = true; + // Store the installation info so it can still get removed if gpii restarts. + that.writeInstallation(installation); + // Destroy the installer + var destroy = installation.installer.destroy; + delete installation.installer; + process.nextTick(destroy); + promise.resolve(true); }, promise.reject); - } + }, promise.reject); return promise.then(function () { fluid.log("IoD: Installation of " + packageRequest.packageName + " complete"); @@ -346,12 +345,20 @@ gpii.iod.initialiseInstallation = function (that, packageRequest) { fluid.log("IoD: Initialising installation for " + packageRequest.packageName); - var installation = { - id: fluid.allocateGuid(), - packageName: packageRequest.packageName, - packageRequest: packageRequest, - cleanupPaths: [] - }; + // See if it's already been loaded + var installation = fluid.find(that.installations, function (inst) { + return inst.packageName === packageRequest.packageName ? inst : undefined; + }); + + if (!installation) { + installation = { + id: fluid.allocateGuid(), + packageName: packageRequest.packageName, + packageRequest: packageRequest, + cleanupPaths: [] + }; + } + that.installations[installation.id] = installation; var promise = fluid.promise(); @@ -391,7 +398,7 @@ gpii.iod.unrequirePackage = function (that, packageName) { }); if (installation) { - installation.remove = true; + installation.required = false; that.writeInstallation(installation); } @@ -412,8 +419,9 @@ gpii.iod.uninitialiseInstallation = function (that, wait) { var uninstall = function () { var inSession = false;// that.model.logonChange.inProgress && that.model.logonChange.type !== "login"; if (!inSession) { + // Get the first installation var installation = fluid.find(that.installations, function (inst) { - return inst.remove && !inst.removed ? inst : undefined; + return !inst.required && !inst.removed ? inst : undefined; }); if (installation && !installation.uninstalling) { @@ -457,14 +465,14 @@ gpii.iod.uninstallPackage = function (that, installation) { var promiseTogo = fluid.promise(); initPromise.then(function (installation) { - var result = installation.installer.uninstallPackage(); + var result = installation.installer.startUninstaller(); fluid.promise.follow(result, promiseTogo); }); return promiseTogo.then(function () { fluid.log("IoD: Uninstallation of " + packageName + " complete"); - installation.remove = false; + installation.required = false; installation.removed = true; that.writeInstallation(installation); delete that.installations[installation.id]; diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 7e9f37d33..4c02a7fa3 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -20,7 +20,8 @@ var path = require("path"), fs = require("fs"), - request = require("request"); + request = require("request"), + child_process = require("child_process"); var fluid = require("infusion"); var gpii = fluid.registerNamespace("gpii"); @@ -38,6 +39,10 @@ fluid.defaults("gpii.iod.packageInstaller", { funcName: "gpii.iod.startInstaller", args: ["{that}", "{iod}"] }, + startUninstaller: { + funcName: "gpii.iod.startUninstaller", + args: ["{that}", "{iod}"] + }, // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns // a installation, either directly or via a promise. initialise: { @@ -61,6 +66,14 @@ fluid.defaults("gpii.iod.packageInstaller", { funcName: "gpii.iod.cleanup", args: ["{that}", "{iod}"] }, + startApplication: { + funcName: "gpii.iod.startApplication", + args: ["{that}", "{iod}"] + }, + stopApplication: { + funcName: "gpii.iod.stopApplication", + args: ["{that}", "{iod}"] + }, uninstallPackage: "fluid.notImplemented" }, events: { @@ -94,10 +107,18 @@ fluid.defaults("gpii.iod.packageInstaller", { func: "{that}.cleanup", priority: "after:install" }, + "onInstallPackage.startApplication": { + func: "{that}.startApplication", + priority: "after:cleanup" + }, + "onRemovePackage.stopApplication": { + func: "{that}.stopApplication", + priority: "first" + }, "onRemovePackage.uninstallPackage": { func: "{that}.uninstallPackage", - priority: "first" + priority: "after:stopApplication" } }, @@ -136,6 +157,17 @@ gpii.iod.startInstaller = function (that) { return fluid.promise.fireTransformEvent(that.events.onInstallPackage); }; +/** + * Starts the un-installation pipeline. + * + * @param {Component} that The gpii.iod.installer instance. + * @param {Component} iod The gpii.iod instance. + * @return {Promise} Resolves when complete. + */ +gpii.iod.startUninstaller = function (that) { + return fluid.promise.fireTransformEvent(that.events.onRemovePackage); +}; + /** * Initialises the installation. * @@ -281,3 +313,45 @@ gpii.iod.cleanup = function (that) { promise.resolve(); return promise; }; + +/** + * Starts the application. + * + * @param {Component} that The gpii.iod.installer instance. + * @return {Promise} Resolves when the application has been started. + */ +gpii.iod.startApplication = function (that) { + var promise = fluid.promise(); + fluid.log("IoD: Starting application " + that.packageInfo.name); + if (that.packageInfo.start) { + child_process.exec(that.packageInfo.start, function (err, stdout, stderr) { + if (err) { + fluid.log("IoD: startApplication error: ", err); + } + fluid.log("IoD: startApplication: ", { stdout: stdout, stderr: stderr }); + }); + } + promise.resolve(); + return promise; +}; + +/** + * Stops the application (for uninstallation). + * + * @param {Component} that The gpii.iod.installer instance. + * @return {Promise} Resolves when the command has completed. + */ +gpii.iod.stopApplication = function (that) { + var promise = fluid.promise(); + fluid.log("IoD: Stopping application " + that.packageInfo.name); + if (that.packageInfo.start) { + child_process.exec(that.packageInfo.start, function (err, stdout, stderr) { + if (err) { + fluid.log("IoD: stopApplication error: ", err); + } + fluid.log("IoD: stopApplication: ", { stdout: stdout, stderr: stderr }); + promise.resolve(); + }); + } + return promise; +}; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 1ba5742d4..4bd6af397 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -45,7 +45,14 @@ fluid.defaults("gpii.iod.packages", { gradeNames: ["fluid.component"], components: { packageData: { - type: "kettle.dataSource.file" + type: "kettle.dataSource.file", + options: { + "gradeNames": ["kettle.dataSource.file.moduleTerms"], + "path": "%gpii-universal/testData/installOnDemand/%packageName.json5", + "termMap": { + "packageName": "%packageName" + } + } }, remotePackageData: { createOnEvent: "onServiceFound", diff --git a/testData/installOnDemand/nvda.json5 b/testData/installOnDemand/nvda.json5 new file mode 100644 index 000000000..88ff7d8de --- /dev/null +++ b/testData/installOnDemand/nvda.json5 @@ -0,0 +1,15 @@ +{ + "name": "nvda", + "description": "NVDA 2019.2", + "url": "https://chocolatey.org/api/v2/package/nvda/2019.2", + "filename": "nvda.2019.2.nupkg", + "packageType": "chocolatey", + "isInstalled": { + "installed": true + }, + "start": "\"c:\\Program Files (x86)\\NVDA\\nvda.exe\"", + "stop": { + cmd: "taskkill", + args: "/f /im nvda.exe" + } +} From 0206f9e643d4593e279025640fa081ec716f07ba Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 12 Sep 2019 19:02:12 +0100 Subject: [PATCH 28/56] GPII-2971: All tests working. --- .../gpii-iod/src/installOnDemand.js | 108 +++++++++++------- .../gpii-iod/src/packageInstaller.js | 2 + gpii/node_modules/gpii-iod/src/packages.js | 20 ++-- gpii/node_modules/gpii-iod/test/all-tests.js | 5 + .../gpii-iod/test/installOnDemandTests.js | 78 ++++++------- .../gpii-iod/test/packageInstallerTests.js | 73 ++++++++---- .../gpii-iod/test/packagesTests.js | 32 +++++- .../test/testPackages/installed1.json | 6 + .../gpii-iod/test/testPackages/package2.json | 2 +- tests/all-tests.js | 4 +- 10 files changed, 201 insertions(+), 129 deletions(-) create mode 100644 gpii/node_modules/gpii-iod/test/all-tests.js create mode 100644 gpii/node_modules/gpii-iod/test/testPackages/installed1.json diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index af8baad86..095bcdca5 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -301,7 +301,8 @@ gpii.iod.getInstaller = function (that, packageType) { * @param {String|Object} packageRequest Package name, or object containing packageName, language, version. * @param {String} packageRequest.packageName Name of the package. * @param {String|String[]} packageRequest.language Language. - * @return {Promise} Resolves when the installation is complete. + * @return {Promise} Resolves with true when the installation is complete, or with false if the package was already + * installed. */ gpii.iod.requirePackage = function (that, packageRequest) { if (typeof(packageRequest) === "string") { @@ -314,19 +315,36 @@ gpii.iod.requirePackage = function (that, packageRequest) { var promise = fluid.promise(); - that.initialiseInstallation(packageRequest).then(function (installation) { - installation.required = true; - - installation.installer.startInstaller().then(function () { - installation.installed = true; - // Store the installation info so it can still get removed if gpii restarts. - that.writeInstallation(installation); - // Destroy the installer - var destroy = installation.installer.destroy; - delete installation.installer; - process.nextTick(destroy); - promise.resolve(true); - }, promise.reject); + that.packages.getPackageInfo(packageRequest).then(function (packageInfo) { + var isInstalled = that.packages.checkInstalled(packageInfo); + if (isInstalled) { + promise.resolve(false); + } else { + that.initialiseInstallation(packageInfo).then(function (installation) { + installation.required = true; + if (installation.installed) { + promise.resolve(false); + } else { + installation.installer.startInstaller().then(function () { + installation.installed = true; + installation.gpiiInstalled = true; + // Store the installation info so it can still get removed if gpii restarts. + that.writeInstallation(installation); + promise.resolve(true); + }, promise.reject); + } + + // Destroy the installer + var destroy = function () { + if (installation.installer) { + installation.installer.destroy(); + } + delete installation.installer; + }; + promise.then(destroy, destroy); + + }, promise.reject); + } }, promise.reject); return promise.then(function () { @@ -336,25 +354,24 @@ gpii.iod.requirePackage = function (that, packageRequest) { }); }; -gpii.iod.initialiseInstallation = function (that, packageRequest) { - if (typeof(packageRequest) === "string") { - packageRequest = { - packageName: packageRequest - }; - } - - fluid.log("IoD: Initialising installation for " + packageRequest.packageName); +/** + * Creates the installer component for the given package. + * @param {Component} that The gpii.iod instance. + * @param {PackageInfo} packageInfo The package data. + * @return {Promise} Resolves with a gpii.iod.installer instance. + */ +gpii.iod.initialiseInstallation = function (that, packageInfo) { + fluid.log("IoD: Initialising installation for " + packageInfo.name); // See if it's already been loaded var installation = fluid.find(that.installations, function (inst) { - return inst.packageName === packageRequest.packageName ? inst : undefined; + return inst.packageName === packageInfo.name ? inst : undefined; }); if (!installation) { installation = { id: fluid.allocateGuid(), - packageName: packageRequest.packageName, - packageRequest: packageRequest, + packageName: packageInfo.name, cleanupPaths: [] }; } @@ -363,22 +380,19 @@ gpii.iod.initialiseInstallation = function (that, packageRequest) { var promise = fluid.promise(); - // Get the package info. - that.packages.getPackageInfo(packageRequest).then(function (packageInfo) { - // Create the installer instance. - installation.packageInfo = packageInfo; - var installerGrade = that.getInstaller(packageInfo.packageType); - if (installerGrade) { - // Load the installer. - that.events.onInstallerLoad.fire(installerGrade, installation.id); - promise.resolve(installation); - } else { - promise.reject({ - isError: true, - error: "Unable to find an installer for package type " + packageInfo.packageType - }); - } - }, promise.reject); + // Create the installer instance. + installation.packageInfo = packageInfo; + var installerGrade = that.getInstaller(packageInfo.packageType); + if (installerGrade) { + // Load the installer. + that.events.onInstallerLoad.fire(installerGrade, installation.id); + promise.resolve(installation); + } else { + promise.reject({ + isError: true, + error: "Unable to find an installer for package type " + packageInfo.packageType + }); + } return promise; }; @@ -424,7 +438,7 @@ gpii.iod.uninitialiseInstallation = function (that, wait) { return !inst.required && !inst.removed ? inst : undefined; }); - if (installation && !installation.uninstalling) { + if (installation && !installation.uninstalling && installation.gpiiInstalled) { installation.uninstalling = true; that.uninstallPackage(installation).then(uninstall, uninstall); } @@ -456,11 +470,19 @@ gpii.iod.uninstallPackage = function (that, installation) { packageName = installation.packageName; } + if (!installation.gpiiInstalled) { + return fluid.promise().reject({ + isError: true, + message: "Not uninstalling package '" + packageName + "': Installed externally" + }); + } + + var initPromise; if (installation && installation.installer) { initPromise = fluid.toPromise(installation); } else { - initPromise = that.initialiseInstallation(installation); + initPromise = that.initialiseInstallation(installation.packageInfo); } var promiseTogo = fluid.promise(); diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 4c02a7fa3..297836845 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -352,6 +352,8 @@ gpii.iod.stopApplication = function (that) { fluid.log("IoD: stopApplication: ", { stdout: stdout, stderr: stderr }); promise.resolve(); }); + } else { + promise.resolve(); } return promise; }; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 4bd6af397..3c2c95169 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -20,9 +20,6 @@ var fluid = require("infusion"); -var path = require("path"), - fs = require("fs"); - require("kettle"); var gpii = fluid.registerNamespace("gpii"); @@ -62,7 +59,11 @@ fluid.defaults("gpii.iod.packages", { invokers: { getPackageInfo: { funcName: "gpii.iod.getPackageInfo", - args: ["{that}", "{arguments}.0"] + args: ["{that}", "{arguments}.0"] // packageRequest + }, + checkInstalled: { + funcName: "gpii.iod.checkInstalled", + args: ["{that}", "{arguments.0}"] // packageInfo } }, listeners: { @@ -158,10 +159,13 @@ gpii.iod.matchLanguage = function (languages, language) { }; /** - * A bad way of checking if a package is installed. - * @param {String} packageName Package name + * Determines if a package is installed. + * + * @param {Component} that The gpii.iod.packages instance. + * @param {PackageInfo} packageInfo The package data. * @return {Boolean} true if the package is installed. */ -gpii.iod.isInstalled = function (packageName) { - return fs.existsSync(path.join(process.env.ProgramData, "chocolatey\\lib", packageName)); +gpii.iod.checkInstalled = function (that, packageInfo) { + packageInfo; + return false; }; diff --git a/gpii/node_modules/gpii-iod/test/all-tests.js b/gpii/node_modules/gpii-iod/test/all-tests.js new file mode 100644 index 000000000..fc55cebe9 --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/all-tests.js @@ -0,0 +1,5 @@ +"use strict"; + +require("./installOnDemandTests.js"); +require("./packageInstallerTests.js"); +require("./packagesTests.js"); diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 65e9a1416..3efd226c4 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -83,16 +83,15 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ expect: { installer: "gpii.tests.iod.testInstaller1", packageName: "package1" - } + }, + resolveValue: true }, { packageRequest: { packageName: "package1" }, - expect: { - installer: "gpii.tests.iod.testInstaller1", - packageName: "package1" - } + expect: null, + resolveValue: false }, { packageRequest: { @@ -101,7 +100,8 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ expect: { installer: "gpii.tests.iod.testInstaller2", packageName: "package2" - } + }, + resolveValue: true }, { packageRequest: { @@ -111,17 +111,16 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ expect: { installer: "gpii.tests.iod.testInstaller1", packageName: "languages" - } + }, + resolveValue: true }, { packageRequest: { packageName: "languages", language: "nl-NL" }, - expect: { - installer: "gpii.tests.iod.testInstaller2", - packageName: "languages" - } + expect: null, + resolveValue: false }, { packageRequest: "failInstall", @@ -321,6 +320,7 @@ jqUnit.asyncTest("test requirePackage", function () { var testIndex = -1; var nextTest = function () { if (++testIndex >= tests.length) { + iod.destroy(); jqUnit.start(); return; } @@ -333,17 +333,17 @@ jqUnit.asyncTest("test requirePackage", function () { jqUnit.assertTrue("requirePackage must return a promise" + suffix, fluid.isPromise(p)); - p.then(function () { - jqUnit.assertTrue("startInstaller must have been called" + suffix, !!iod.funcCalled.startInstaller); + p.then(function (value) { + jqUnit.assertDeepEq("requirePackage must resolve with the expected value" + suffix, + test.resolveValue, value); jqUnit.assertDeepEq("startInstaller must have been called correctly" + suffix, test.expect, iod.funcCalled.startInstaller); - nextTest(); + process.nextTick(nextTest); }, function () { jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); jqUnit.assert("balance the assert count"); - nextTest(); + process.nextTick(nextTest); }); - }; nextTest(); @@ -385,8 +385,8 @@ jqUnit.asyncTest("test installation storage", function () { }, expectRead: { id: "new-installation", - remove: true, - uninstalling: true + required: false, + uninstalling: false } }, updatedInst: { @@ -401,8 +401,8 @@ jqUnit.asyncTest("test installation storage", function () { expectRead: { id: "new-installation", test1: "something", - remove: true, - uninstalling: true + required: false, + uninstalling: false } } }; @@ -441,8 +441,8 @@ jqUnit.asyncTest("test installation storage", function () { jqUnit.assertDeepEq("readInstallations should read the correct data", testData.newInst.expectRead, iod.installations[testData.newInst.input.id]); - jqUnit.assertTrue("uninstallPackage should have been called", !!iod.funcCalled.uninstallPackage); - iod.funcCalled.uninstallPackage = null; + // jqUnit.assertTrue("uninstallPackage should have been called", !!iod.funcCalled.uninstallPackage); + // iod.funcCalled.uninstallPackage = null; // Overwrite the existing file. var file = gpii.iod.writeInstallation(iod, dir, testData.updatedInst.input); @@ -458,8 +458,8 @@ jqUnit.asyncTest("test installation storage", function () { jqUnit.assertDeepEq("readInstallations should update with the correct data", testData.updatedInst.expectRead, iod.installations[testData.newInst.input.id]); - jqUnit.assertTrue("uninstallPackage should have been called again", - !!iod.funcCalled.uninstallPackage); + // jqUnit.assertTrue("uninstallPackage should have been called again", + // !!iod.funcCalled.uninstallPackage); jqUnit.start(); }); @@ -486,30 +486,22 @@ jqUnit.asyncTest("test uninstallation", function () { iod.unrequirePackage(packageName); - jqUnit.assertTrue("Package should have been set to be removed", installation.remove); + jqUnit.assertTrue("Package should have been set to be not required", !installation.require); - iod.uninstallPackage(packageName); + var promise = iod.uninstallPackage(packageName); - var retries = 100; - var waitForRemoval = function () { + promise.then(function () { // There's no promise or event, so just poll. if (iod.installations[installation.id]) { - if (--retries > 0) { - setTimeout(waitForRemoval, 100); - } else { - fluid.fail("Package was not removed"); - } + fluid.fail("Package was not removed"); } else { jqUnit.assertTrue("packageInstaller.uninstallPackage should have been called", !!iod.funcCalled.uninstallPackage); jqUnit.start(); } - }; - - process.nextTick(waitForRemoval); + }, jqUnit.fail); }, jqUnit.fail); - }); @@ -571,20 +563,16 @@ jqUnit.asyncTest("test uninstallation after restart", function () { iod.unrequirePackage(packageName); - jqUnit.assertTrue("Package should have been set to be removed", installation.remove); + jqUnit.assertTrue("Package should have been set to be not required", !installation.require); - iod.uninstallPackage(packageName); - - waitForUninstall().then(function () { + process.nextTick(function () { // Fake a restart by creating a new instance of iod. iod.destroy(); iodOptions.testReject = null; iod = gpii.tests.iod(iodOptions); iod.readInstallations(); + }); - waitForUninstall().then(jqUnit.start, jqUnit.fail); - - - }, jqUnit.fail); + waitForUninstall().then(jqUnit.start, jqUnit.fail); }, jqUnit.fail); }); diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index 8cd26d896..40c4b48b4 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -23,31 +23,31 @@ var os = require("os"), path = require("path"), crypto = require("crypto"); -var fluid = require("gpii-universal"); +var fluid = require("infusion"); var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); var jqUnit = fluid.require("node-jqunit"); var gpii = fluid.registerNamespace("gpii"); -fluid.registerNamespace("gpii.tests.iod.installer"); +fluid.registerNamespace("gpii.tests.iodInstaller"); require("../index.js"); -gpii.tests.iod.teardowns = []; +gpii.tests.iodInstaller.teardowns = []; -jqUnit.module("gpii.tests.iod.installer", { +jqUnit.module("gpii.tests.iodInstaller", { teardown: function () { - while (gpii.tests.iod.teardowns.length) { - gpii.tests.iod.teardowns.pop()(); + while (gpii.tests.iodInstaller.teardowns.length) { + gpii.tests.iodInstaller.teardowns.pop()(); } } }); -fluid.defaults("gpii.tests.iod", { +fluid.defaults("gpii.tests.iodInstaller", { gradeNames: [ "gpii.iod" ], components: { "testInstaller": { - type: "gpii.tests.iod.installer" + type: "gpii.tests.iodInstaller.installer" }, "packages": { type: "gpii.iod.packages", @@ -62,35 +62,42 @@ fluid.defaults("gpii.tests.iod", { } } } + }, + invokers: { + readInstallations: "fluid.identity", + writeInstallation: "fluid.identity" } }); -fluid.defaults("gpii.tests.iod.installer", { +fluid.defaults("gpii.tests.iodInstaller.installer", { gradeNames: ["fluid.component", "gpii.iod.packageInstaller"], invokers: { - initialise: "gpii.tests.iod.installer.stage({that}, initialise)", - downloadPackage: "gpii.tests.iod.installer.stage({that}, downloadPackage)", - checkPackage: "gpii.tests.iod.installer.stage({that}, checkPackage)", - prepareInstall: "gpii.tests.iod.installer.stage({that}, prepareInstall)", - installPackage: "gpii.tests.iod.installer.stage({that}, installPackage)", - cleanup: "gpii.tests.iod.installer.stage({that}, cleanup)", - uninstallPackage: "gpii.tests.iod.installer.stage({that}, uninstallPackage)" + initialise: "gpii.tests.iodInstaller.stage({that}, initialise)", + downloadPackage: "gpii.tests.iodInstaller.stage({that}, downloadPackage)", + checkPackage: "gpii.tests.iodInstaller.stage({that}, checkPackage)", + prepareInstall: "gpii.tests.iodInstaller.stage({that}, prepareInstall)", + installPackage: "gpii.tests.iodInstaller.stage({that}, installPackage)", + cleanup: "gpii.tests.iodInstaller.stage({that}, cleanup)", + startApplication: "gpii.tests.iodInstaller.stage({that}, startApplication)", + uninstallPackage: "gpii.tests.iodInstaller.stage({that}, uninstallPackage)", + stopApplication: "gpii.tests.iodInstaller.stage({that}, stopApplication)" }, packageTypes: "testPackageType1" }); -gpii.tests.iod.installer.stage = function (that, stage) { +gpii.tests.iodInstaller.stage = function (that, stage) { that.stages.push(stage); }; // Test startInstaller starts the installation pipe-line. -jqUnit.test("test getInstaller", function () { +jqUnit.asyncTest("test installation pipe-line", function () { - var iod = gpii.tests.iod(); - var installer = iod.getInstaller("testPackageType1"); + var iod = gpii.tests.iodInstaller(); + var installer = iod.testInstaller; + jqUnit.expect(2); installer.stages = []; installer.startInstaller({}).then(function () { @@ -100,18 +107,36 @@ jqUnit.test("test getInstaller", function () { "checkPackage", "prepareInstall", "installPackage", - "cleanup" + "cleanup", + "startApplication" ]; jqUnit.assertDeepEq("All stages of the installation should be called in order.", expect, installer.stages); + installer.stages = []; + installer.startUninstaller().then(function () { + var expect = [ + "stopApplication", + "uninstallPackage" + ]; + + jqUnit.assertDeepEq("All stages of the uninstallation should be called in order.", expect, installer.stages); + jqUnit.start(); + }); }, jqUnit.fail); }); jqUnit.asyncTest("test https download", function () { - gpii.tests.iod.installer.downloadTests = fluid.freezeRecursive([ + if (process.env.GPII_QUICKTEST) { + fluid.log("Skipping download tests"); + jqUnit.assert(); + jqUnit.start(); + return; + } + + gpii.tests.iodInstaller.downloadTests = fluid.freezeRecursive([ { url: "https://raw.githubusercontent.com/GPII/universal/108be0f5f0377eaec9100c1926647e7550efc2ea/gpii.js", expect: "8cb82683c931e15995b2573fda41c41eaacab59e" @@ -179,7 +204,7 @@ jqUnit.asyncTest("test https download", function () { var files = []; // Remove all temporary files. - gpii.tests.iod.teardowns.push(function () { + gpii.tests.iodInstaller.teardowns.push(function () { fluid.each(files, function (file) { try { fs.unlinkSync(file); @@ -190,7 +215,7 @@ jqUnit.asyncTest("test https download", function () { }); - var tests = gpii.tests.iod.installer.downloadTests; + var tests = gpii.tests.iodInstaller.downloadTests; jqUnit.expect(tests.length * 3); var testIndex = -1; diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index 4dd66ca2e..f584e9a6d 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -25,13 +25,13 @@ kettle.loadTestingSupport(); var jqUnit = fluid.require("node-jqunit"); var gpii = fluid.registerNamespace("gpii"); -fluid.registerNamespace("gpii.tests.iod"); +fluid.registerNamespace("gpii.tests.iodPackages"); require("../index.js"); var teardowns = []; -jqUnit.module("gpii.tests.iod", { +jqUnit.module("gpii.tests.iodPackages", { teardown: function () { while (teardowns.length) { teardowns.pop()(); @@ -40,7 +40,7 @@ jqUnit.module("gpii.tests.iod", { }); -gpii.tests.iod.getPackageInfoTests = fluid.freezeRecursive([ +gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ { id: "No matching package", request: { @@ -178,13 +178,33 @@ gpii.tests.iod.getPackageInfoTests = fluid.freezeRecursive([ } ]); +fluid.defaults("gpii.tests.iodPackages", { + gradeNames: [ "gpii.iod" ], + distributeOptions: { + packageData: { + record: { + gradeNames: ["kettle.dataSource.file.moduleTerms"], + path: __dirname + "/testPackages/%packageName.json", + termMap: { + "packageName": "%packageName" + } + }, + target: "{that packages packageData}.options" + } + }, + invokers: { + readInstallations: "fluid.identity", + writeInstallation: "fluid.identity" + } +}); + // Test getPackageInfo returns correct information jqUnit.asyncTest("test getPackageInfo", function () { - var tests = gpii.tests.iod.getPackageInfoTests; + var tests = gpii.tests.iodPackages.getPackageInfoTests; jqUnit.expect(tests.length * 2); - var iod = gpii.tests.iod(); + var iod = gpii.tests.iodPackages(); var testIndex = -1; var nextTest = function () { @@ -198,7 +218,7 @@ jqUnit.asyncTest("test getPackageInfo", function () { fluid.log("getPackage: " + test.request.packageName + ", " + test.request.language); - var p = iod.getPackageInfo(test.request); + var p = iod.packages.getPackageInfo(test.request); jqUnit.assertTrue("getPackageInfo must return a promise" + suffix, fluid.isPromise(p)); diff --git a/gpii/node_modules/gpii-iod/test/testPackages/installed1.json b/gpii/node_modules/gpii-iod/test/testPackages/installed1.json new file mode 100644 index 000000000..688bd0549 --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/testPackages/installed1.json @@ -0,0 +1,6 @@ +{ + "name": "isInstalledTest1", + "filename": "example.filename", + "url": "test://example", + "packageType": "testPackageType1" +} diff --git a/gpii/node_modules/gpii-iod/test/testPackages/package2.json b/gpii/node_modules/gpii-iod/test/testPackages/package2.json index 2507a5291..188c6f6e7 100644 --- a/gpii/node_modules/gpii-iod/test/testPackages/package2.json +++ b/gpii/node_modules/gpii-iod/test/testPackages/package2.json @@ -1,5 +1,5 @@ { - "name": "package1", + "name": "package2", "filename": "example.filename", "url": "test://example", "packageType": "testPackageType2a" diff --git a/tests/all-tests.js b/tests/all-tests.js index 5e172e8ab..e5017d5a9 100644 --- a/tests/all-tests.js +++ b/tests/all-tests.js @@ -85,8 +85,8 @@ var testIncludes = [ "../gpii/node_modules/singleInstance/test/SingleInstanceTests.js", "../gpii/node_modules/solutionsRegistry/test/all-tests.js", "../gpii/node_modules/transformer/test/TransformerTests.js", - "../gpii/node_modules/userListeners/test/all-tests.js" - "../gpii/node_modules/gpii-iod/test/installOnDemandTests.js" + "../gpii/node_modules/userListeners/test/all-tests.js", + "../gpii/node_modules/gpii-iod/test/all-tests.js" ]; fluid.each(testIncludes, function (path) { From ea354a454a6968197cd702a11647c3007ad3bc3f Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 12 Sep 2019 20:11:36 +0100 Subject: [PATCH 29/56] GPII-2971: Changed tests to json5 --- gpii/node_modules/gpii-iod/test/installOnDemandTests.js | 2 +- gpii/node_modules/gpii-iod/test/packageInstallerTests.js | 2 +- gpii/node_modules/gpii-iod/test/packagesTests.js | 6 +++--- .../testPackages/{failInstall.json => failInstall.json5} | 0 .../gpii-iod/test/testPackages/installed1.json | 6 ------ .../gpii-iod/test/testPackages/installed1.json5 | 9 +++++++++ .../testPackages/{languages.json => languages.json5} | 0 .../test/testPackages/{package1.json => package1.json5} | 0 .../test/testPackages/{package2.json => package2.json5} | 0 .../testPackages/{unknownType.json => unknownType.json5} | 0 10 files changed, 14 insertions(+), 11 deletions(-) rename gpii/node_modules/gpii-iod/test/testPackages/{failInstall.json => failInstall.json5} (100%) delete mode 100644 gpii/node_modules/gpii-iod/test/testPackages/installed1.json create mode 100644 gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 rename gpii/node_modules/gpii-iod/test/testPackages/{languages.json => languages.json5} (100%) rename gpii/node_modules/gpii-iod/test/testPackages/{package1.json => package1.json5} (100%) rename gpii/node_modules/gpii-iod/test/testPackages/{package2.json => package2.json5} (100%) rename gpii/node_modules/gpii-iod/test/testPackages/{unknownType.json => unknownType.json5} (100%) diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 3efd226c4..dac6e6329 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -139,7 +139,7 @@ fluid.defaults("gpii.tests.iod", { packageData: { record: { gradeNames: ["kettle.dataSource.file.moduleTerms"], - path: __dirname + "/testPackages/%packageName.json", + path: __dirname + "/testPackages/%packageName.json5", termMap: { "packageName": "%packageName" } diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index 40c4b48b4..ccf56d2c3 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -56,7 +56,7 @@ fluid.defaults("gpii.tests.iodInstaller", { "packageDataSource": { type: "kettle.dataSource.file", options: { - path: __dirname + "/testPackages/%packageName.json" + path: __dirname + "/testPackages/%packageName.json5" } } } diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index f584e9a6d..f61da19be 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -53,7 +53,7 @@ gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ request: { packageName: "package1" }, - expect: require("./testPackages/package1.json") + expect: require("./testPackages/package1.json5") }, { id: "Single language package, with language specified", @@ -61,7 +61,7 @@ gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ packageName: "package1", language: "fr-FR" }, - expect: require("./testPackages/package1.json") + expect: require("./testPackages/package1.json5") }, { id: "Multi-language package, language not specified", @@ -184,7 +184,7 @@ fluid.defaults("gpii.tests.iodPackages", { packageData: { record: { gradeNames: ["kettle.dataSource.file.moduleTerms"], - path: __dirname + "/testPackages/%packageName.json", + path: __dirname + "/testPackages/%packageName.json5", termMap: { "packageName": "%packageName" } diff --git a/gpii/node_modules/gpii-iod/test/testPackages/failInstall.json b/gpii/node_modules/gpii-iod/test/testPackages/failInstall.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/failInstall.json rename to gpii/node_modules/gpii-iod/test/testPackages/failInstall.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/installed1.json b/gpii/node_modules/gpii-iod/test/testPackages/installed1.json deleted file mode 100644 index 688bd0549..000000000 --- a/gpii/node_modules/gpii-iod/test/testPackages/installed1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "isInstalledTest1", - "filename": "example.filename", - "url": "test://example", - "packageType": "testPackageType1" -} diff --git a/gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 b/gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 new file mode 100644 index 000000000..31777072f --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 @@ -0,0 +1,9 @@ +{ + "name": "installed1", + "filename": "example.filename", + "url": "test://example", + "packageType": "testPackageType1", + "isInstalled": { + "installed": 1 + } +} diff --git a/gpii/node_modules/gpii-iod/test/testPackages/languages.json b/gpii/node_modules/gpii-iod/test/testPackages/languages.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/languages.json rename to gpii/node_modules/gpii-iod/test/testPackages/languages.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/package1.json b/gpii/node_modules/gpii-iod/test/testPackages/package1.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/package1.json rename to gpii/node_modules/gpii-iod/test/testPackages/package1.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/package2.json b/gpii/node_modules/gpii-iod/test/testPackages/package2.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/package2.json rename to gpii/node_modules/gpii-iod/test/testPackages/package2.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/unknownType.json b/gpii/node_modules/gpii-iod/test/testPackages/unknownType.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/unknownType.json rename to gpii/node_modules/gpii-iod/test/testPackages/unknownType.json5 From 3b92b646db94b3b32702f12592e9ccfc8d42290f Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 19 Sep 2019 16:14:53 +0100 Subject: [PATCH 30/56] GPII-2971: Packages support transforms, resolvers, isInstalled --- gpii/node_modules/gpii-iod/src/packages.js | 123 +++++- .../gpii-iod/test/packagesTests.js | 403 +++++++++++++++++- .../gpii-iod/test/testPackages/env.json5 | 5 + .../test/testPackages/installed1.json5 | 9 - 4 files changed, 520 insertions(+), 20 deletions(-) create mode 100644 gpii/node_modules/gpii-iod/test/testPackages/env.json5 delete mode 100644 gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 3c2c95169..7c1baa52f 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -18,13 +18,16 @@ "use strict"; -var fluid = require("infusion"); +var fluid = require("infusion"), + fs = require("fs"); require("kettle"); var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.iod.packages"); +fluid.registerNamespace("gpii.iod.packages.resolvers"); +fluid.require("%lifecycleManager"); require("./packageInstaller.js"); require("./iodSettingsHandler.js"); @@ -38,22 +41,32 @@ require("./iodSettingsHandler.js"); * */ +gpii.iod.resolvers = {}; + fluid.defaults("gpii.iod.packages", { gradeNames: ["fluid.component"], components: { packageData: { type: "kettle.dataSource.file", options: { - "gradeNames": ["kettle.dataSource.file.moduleTerms"], - "path": "%gpii-universal/testData/installOnDemand/%packageName.json5", - "termMap": { + gradeNames: ["kettle.dataSource.file.moduleTerms", "kettle.dataSource.encoding.JSON5"], + path: "%gpii-universal/testData/installOnDemand/%packageName.json5", + termMap: { "packageName": "%packageName" + }, + components: { + encoding: { + type: "kettle.dataSource.encoding.JSON5" + } } } }, remotePackageData: { createOnEvent: "onServiceFound", type: "kettle.dataSource.URL" + }, + variableResolver: { + type: "gpii.lifecycleManager.variableResolver" } }, invokers: { @@ -63,14 +76,99 @@ fluid.defaults("gpii.iod.packages", { }, checkInstalled: { funcName: "gpii.iod.checkInstalled", - args: ["{that}", "{arguments.0}"] // packageInfo + args: ["{that}", "{arguments}.0"] // packageInfo + }, + resolvePackage: { + funcName: "gpii.iod.resolvePackage", + args: ["{that}", "{arguments}.0"] // packageInfo } }, listeners: { onCreate: "fluid.identity" + }, + members: { + resolvers: { + expander: { + func: "fluid.transform", + args: [ "{that}.options.resolvers", fluid.getGlobalValue ] + } + }, + fetcher: { + expander: { + func: "gpii.resolversToFetcher", + args: "{that}.resolvers" + } + } + }, + resolvers: { + exists: "gpii.iod.existsResolver" } + }); +/** + * Resolver for ${{exists}.path}, determines if a filesystem path exists. The path can include environment variables, + * named between two '%' symbols (like %this%), to work around the resolvers not supporting nested expressions. + * + * @param {String} path The path to test. Environment variables within two '%' symbols are expanded. + * @return {Boolean} true if the path exists. + */ +gpii.iod.existsResolver = function (path) { + var expandedPath = path.replace(/%([^% ]+)%/g, function (match, name) { + return process.env[name]; + }); + return fs.existsSync(expandedPath); +}; + +/** + * Resolves ${} variables of fields in a packageInfo. + * + * @param {Component} that The gpii.iod.packages instance. + * @param {PackageInfo} packageInfo The package. + * @return {PackageInfo} A copy of the package data, with resolved fields. + */ +gpii.iod.resolvePackage = function (that, packageInfo) { + var result; + + if (packageInfo._original) { + // This package has already been resolved; work on the original copy. + result = gpii.iod.resolvePackage(that, packageInfo._original); + } else { + result = fluid.copy(packageInfo); + + // Allow references to the package itself via "${{this}.field}". + var fetchers = gpii.combineFetchers(that.fetcher, gpii.resolversToFetcher({"this": result})); + // Run the resolvers first, so the real values can be used in the transforms + result = that.variableResolver.resolve(result, fetchers); + + // Transform the package data. Because the same object is being used as the rules, just transform each field which + // have a transform rule, to avoid using malformed rules. + fluid.each(result, function (value, key) { + if (value && (value.transform || value.literalValue) && key !== "_original") { + var newValue = value; + var count = 0; + // Transform the field, until it's no longer a transform rule. Meaning, if the output is another field + // which has yet to be transformed, then transform it. + do { + if (++count > fluid.strategyRecursionBailout) { + fluid.log(fluid.logLevel.WARN, "ERROR: resolvePackage transform got too deep"); + newValue = undefined; + break; + } else { + newValue = fluid.model.transformWithRules(result, {out: newValue}).out; + } + } while (newValue && (newValue.transform || newValue.literalValue)); + result[key] = newValue; + } + }); + + // Stash the original, so it can be re-resolved. + result._original = fluid.freezeRecursive(packageInfo); + } + + return result; +}; + /** * Retrieve the package metadata. * @@ -104,7 +202,8 @@ gpii.iod.getPackageInfo = function (that, packageRequest) { } } - promise.resolve(packageInfo); + var resolvedPackageInfo = that.resolvePackage(packageInfo); + promise.resolve(resolvedPackageInfo); }, function (err) { promise.reject({ isError: true, @@ -166,6 +265,14 @@ gpii.iod.matchLanguage = function (languages, language) { * @return {Boolean} true if the package is installed. */ gpii.iod.checkInstalled = function (that, packageInfo) { - packageInfo; - return false; + + // Update the isInstalled, to reflect the current situation. + packageInfo = that.resolvePackage(packageInfo); + + var isInstalled = packageInfo.isInstalled; + if (fluid.isPlainObject(isInstalled)) { + isInstalled = isInstalled.isInstalled || isInstalled.value; + } + + return !!fluid.coerceToPrimitive(isInstalled); }; diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index f61da19be..011b28cbf 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -22,6 +22,11 @@ var fluid = require("infusion"); var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); +var JSON5 = require("json5"), + fs = require("fs"), + path = require("path"), + os = require("os"); + var jqUnit = fluid.require("node-jqunit"); var gpii = fluid.registerNamespace("gpii"); @@ -39,7 +44,6 @@ jqUnit.module("gpii.tests.iodPackages", { } }); - gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ { id: "No matching package", @@ -48,12 +52,23 @@ gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ }, expect: "reject" }, + { + id: "variables resolved", + request: { + packageName: "env" + }, + expect: { + name: "env", + test: process.env.PATH, + packageType: "testPackageType1" + } + }, { id: "Single language package", request: { packageName: "package1" }, - expect: require("./testPackages/package1.json5") + expect: JSON5.parse(fs.readFileSync(__dirname + "/testPackages/package1.json5", "utf8")) }, { id: "Single language package, with language specified", @@ -61,7 +76,7 @@ gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ packageName: "package1", language: "fr-FR" }, - expect: require("./testPackages/package1.json5") + expect: JSON5.parse(fs.readFileSync(__dirname + "/testPackages/package1.json5", "utf8")) }, { id: "Multi-language package, language not specified", @@ -178,6 +193,308 @@ gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ } ]); +gpii.tests.iodPackages.resolvePackageTests = fluid.freezeRecursive([ + // Resolver + { + name: "environment", + result: "${{environment}.PATH", + expect: process.PATH + }, + { + name: "exists", + result: "${{exists}./}", + expect: true + }, + { + name: "exists (not)", + result: "${{exists}./gpii-test/not/exist}", + expect: false + }, + { + name: "this", + anotherValue: "it works", + result: "${{this}.anotherValue}", + expect: "it works" + }, + { + name: "this (object)", + anotherValue: { + deepValue: "it works" + }, + result: "${{this}.anotherValue}", + expect: { + deepValue: "it works" + } + }, + { + name: "this (deep field)", + anotherValue: { + deepValue: "it works" + }, + result: "${{this}.anotherValue.deepValue}", + expect: "it works" + }, + { + name: "this (multiple)", + first: "it works", + second: "${{this}.first}", + result: "${{this}.second}", + expect: "it works" + }, + { + name: "this (multiple, reverse order)", + result: "${{this}.second}", + second: "${{this}.first}", + first: "it works", + expect: "it works" + }, + { + name: "unknown resolver", + result: "${{stupid}}", + expect: undefined + }, + // Transforms + { + name: "basic transform", + result: { + transform: { + type: "fluid.transforms.literalValue", + input: "it works" + } + }, + expect: "it works" + }, + { + name: "basic transform, object result", + result: { + transform: { + type: "fluid.transforms.literalValue", + input: "it works", + outputPath: "nested" + } + }, + expect: { + nested: "it works" + } + }, + { + name: "basic transform, null result", + result: { + transform: { + type: "fluid.transforms.literalValue", + input: null + } + }, + expect: null + }, + { + name: "literal transform", + result: { + literalValue: "it works" + }, + expect: "it works" + }, + { + name: "transform, outer reference", + otherValue: "it works", + result: { + transform: { + type: "fluid.transforms.value", + inputPath: "otherValue" + } + }, + expect: "it works" + }, + { + name: "transform, outer deep reference", + otherValue: { + nested: "it works" + }, + result: { + transform: { + type: "fluid.transforms.value", + inputPath: "otherValue.nested" + } + }, + expect: "it works" + }, + { + name: "transform, reference to transformed", + otherValue: { + transform: { + type: "fluid.transforms.value", + input: "it works" + } + }, + result: { + transform: { + type: "fluid.transforms.value", + inputPath: "otherValue" + } + }, + expect: "it works" + }, + { + name: "transform, reference to transformed (looking ahead)", + result: { + transform: { + type: "fluid.transforms.value", + inputPath: "otherValue" + } + }, + otherValue: { + transform: { + type: "fluid.transforms.value", + input: "it works" + } + }, + expect: "it works" + }, + { + name: "transform, self reference", + result: { + transform: { + type: "fluid.transforms.value", + inputPath: "value" + } + }, + expect: undefined + }, + { + name: "transform, circular reference", + result: { + transform: { + type: "fluid.transforms.value", + inputPath: "value2" + } + }, + value2: { + transform: { + type: "fluid.transforms.value", + inputPath: "value3" + } + }, + value3: { + transform: { + type: "fluid.transforms.value", + inputPath: "value" + } + }, + expect: undefined + }, + { + name: "resolving transformed value", + result: "${{this}.value2}", + value2: { + transform: { + type: "fluid.transforms.value", + input: "it works" + } + }, + expect: "it works" + }, + { + name: "transforms operate on the resolved variables", + result: { + transform: { + type: "fluid.transforms.condition", + // will be true if it's not resolved + condition: "${{exists}./does/not/exist1}", + false: "it works", + true: "hide the evidence" + } + }, + expect: "it works" + } +]); + +gpii.tests.iodPackages.checkInstalledTests = fluid.freezeRecursive([ + { + name: "literal true", + isInstalled: true, + expect: true + }, + { + name: "literal false", + isInstalled: false, + expect: false + }, + { + name: "string true", + isInstalled: "true", + expect: true + }, + { + name: "string false", + isInstalled: "false", + expect: false + }, + { + name: "literal 1", + isInstalled: 1, + expect: true + }, + { + name: "literal 0", + isInstalled: 0, + expect: false + }, + { + name: "string 1", + isInstalled: "1", + expect: true + }, + { + name: "string 0", + isInstalled: "0", + expect: false + }, + { + name: "word string", + isInstalled: "hello", + expect: true + }, + { + name: "empty string", + isInstalled: "", + expect: false + }, + { + name: "null", + isInstalled: null, + expect: false + }, + { + name: "undefined", + isInstalled: undefined, + expect: false + }, + { + name: "no value", + expect: false + }, + { + name: "empty object", + isInstalled: {}, + expect: false + }, + { + name: "object", + isInstalled: {something: "hello"}, + expect: false + }, + { + name: "object containing isInstalled:true", + isInstalled: {isInstalled: true}, + expect: true + }, + { + name: "object containing isInstalled:0", + isInstalled: {isInstalled: "0"}, + expect: false + } +]); + fluid.defaults("gpii.tests.iodPackages", { gradeNames: [ "gpii.iod" ], distributeOptions: { @@ -198,6 +515,54 @@ fluid.defaults("gpii.tests.iodPackages", { } }); +jqUnit.test("test the 'exists' resolver function", function () { + + // Test it works on a non-existent file + var result = gpii.iod.existsResolver(path.join(os.tmpdir(), "not-exist" + Math.random())); + jqUnit.assertFalse("existsResolver should return false for a non-existing file", result); + + // Test it works on an existing file + var result2 = gpii.iod.existsResolver(__filename); + jqUnit.assertTrue("existsResolver should return true for an existing file", result2); + + // Test it works on an existing directory + var result3 = gpii.iod.existsResolver(__dirname); + jqUnit.assertTrue("existsResolver should return true for an existing directory", result3); + + // Test environment variables are expanded + var result4 = gpii.iod.existsResolver("%HOME%"); + jqUnit.assertTrue("existsResolver should return true, with an environment variable", result4); + process.env.GPII_TEST_RESOLVER1 = __dirname; + process.env.GPII_TEST_RESOLVER2 = path.basename(__filename, "js"); + var result5 = gpii.iod.existsResolver("%GPII_TEST_RESOLVER1%/%GPII_TEST_RESOLVER2%js"); + jqUnit.assertTrue("existsResolver should return true, with multiple environment variables", result5); +}); + +jqUnit.test("test the package resolver", function () { + + var iod = gpii.tests.iodPackages(); + + fluid.each(gpii.tests.iodPackages.resolvePackageTests, function (test) { + + var current = test; + // Resolve the package more than once, to show it can be re-resolved. + for (var i = 1; i <= 3; i++) { + var resolved = iod.packages.resolvePackage(current); + var suffix = " - " + test.name + " (pass " + i + ")"; + + jqUnit.assertDeepEq("return of resolvePackage should contain the original package" + suffix, + test, resolved._original); + + jqUnit.assertDeepEq("resolvePackage should return the expected value" + suffix, + test.expect, resolved.result); + + current = resolved; + } + }); + + iod.destroy(); +}); + // Test getPackageInfo returns correct information jqUnit.asyncTest("test getPackageInfo", function () { @@ -224,6 +589,7 @@ jqUnit.asyncTest("test getPackageInfo", function () { p.then(function (packageInfo) { delete packageInfo.languages; + delete packageInfo._original; jqUnit.assertDeepEq("packageInfo must match expected" + suffix, test.expect, packageInfo); nextTest(); }, function (e) { @@ -238,3 +604,34 @@ jqUnit.asyncTest("test getPackageInfo", function () { nextTest(); }); + +// Test checkInstalled works +jqUnit.test("test checkInstalled", function () { + var iod = gpii.tests.iodPackages(); + + fluid.each(gpii.tests.iodPackages.checkInstalledTests, function (test) { + var result = iod.packages.checkInstalled(test); + jqUnit.assertEquals("checkInstalled should return the expected result - " + test.name, test.expect, result); + }); + + // Ensure the same package can return a different result - ie, the result is live. + var testEnv = "_gpii_test_checkInstalled"; + var testPackage = iod.packages.resolvePackage({ + name: "checkInstalledTest", + isInstalled: "${{environment}." + testEnv + "}" + }); + + var testValues = [ false, true, false, false, true, true, false ]; + fluid.each(testValues, function (value, index) { + // Change what isInstalled resolves to. + process.env[testEnv] = value.toString(); + + var result = iod.packages.checkInstalled(testPackage); + + jqUnit.assertEquals("checkInstalled should return the expected result - index=" + index, value, result); + }); + + delete process.env[testEnv]; + + +}); diff --git a/gpii/node_modules/gpii-iod/test/testPackages/env.json5 b/gpii/node_modules/gpii-iod/test/testPackages/env.json5 new file mode 100644 index 000000000..9b2b6ac67 --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/testPackages/env.json5 @@ -0,0 +1,5 @@ +{ + "name": "env", + "test": "${{environment}.PATH}", + "packageType": "testPackageType1", +} diff --git a/gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 b/gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 deleted file mode 100644 index 31777072f..000000000 --- a/gpii/node_modules/gpii-iod/test/testPackages/installed1.json5 +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "installed1", - "filename": "example.filename", - "url": "test://example", - "packageType": "testPackageType1", - "isInstalled": { - "installed": 1 - } -} From eb8e9f25ebe581927f14f3a8d2df881258ba078d Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 25 Oct 2019 14:27:47 +0100 Subject: [PATCH 31/56] GPII-2971: IoD server/client connectivity --- .../configs/gpii.iod.config.development.json | 2 +- .../gpii-iod/src/installOnDemand.js | 84 +++---------- .../gpii-iod/src/packageInstaller.js | 118 +++++++++--------- gpii/node_modules/gpii-iod/src/packages.js | 69 +++++----- .../gpii-iod/test/installOnDemandTests.js | 6 +- .../gpii-iod/test/packagesTests.js | 26 ++-- 6 files changed, 128 insertions(+), 177 deletions(-) diff --git a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json index 197a886e5..a793116ee 100644 --- a/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json +++ b/gpii/node_modules/gpii-iod/configs/gpii.iod.config.development.json @@ -4,7 +4,7 @@ "distributeOptions": { "packageData.dev": { "record": { - "endpoint": "http://localhost:8087" + "endpoint": "http://vagrant.iod-test.net" }, "target": "{that gpii.iod}.options" } diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 095bcdca5..1da592b5a 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -33,8 +33,8 @@ fluid.registerNamespace("gpii.iod"); * Installation state. * @typedef {Object} Installation * @property {id} - Installation ID - * @property {packageInfo} packageInfo - Package data. - * @property {string} packageName - packageInfo.name + * @property {packageData} packageData - Package data. + * @property {string} packageName - packageData.name * @property {Component} installer - The gpii.iod.installer instance. * @property {boolean} failed - true if the installation had failed. * @property {string} tmpDir - Temporary working directory. @@ -76,14 +76,12 @@ fluid.defaults("gpii.iod", { }, events: { onServiceFound: null, // [ endpoint address ] - onServiceLost: null, // [ endpoint address ] onInstallerLoad: null // [ packageInstaller grade name, installation ID ] }, listeners: { "onCreate.discoverServer": "{that}.discoverServer", "onCreate.readInstallations": "{that}.readInstallations", "onServiceFound": "{that}.serviceFound", - "onServiceLost": "{that}.serviceLost" }, invokers: { discoverServer: { @@ -110,10 +108,6 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.serviceFound", args: ["{that}", "{arguments}.0"] }, - serviceLost: { - funcName: "gpii.iod.serviceLost", - args: ["{that}", "{arguments}.0"] - }, readInstallations: { funcName: "gpii.iod.readInstallations", args: ["{that}", "{gpii.journal}.settingsDir.gpiiSettingsDir" ] @@ -315,12 +309,12 @@ gpii.iod.requirePackage = function (that, packageRequest) { var promise = fluid.promise(); - that.packages.getPackageInfo(packageRequest).then(function (packageInfo) { - var isInstalled = that.packages.checkInstalled(packageInfo); + that.packages.getPackageData(packageRequest).then(function (packageData) { + var isInstalled = that.packages.checkInstalled(packageData); if (isInstalled) { promise.resolve(false); } else { - that.initialiseInstallation(packageInfo).then(function (installation) { + that.initialiseInstallation(packageData).then(function (installation) { installation.required = true; if (installation.installed) { promise.resolve(false); @@ -357,21 +351,21 @@ gpii.iod.requirePackage = function (that, packageRequest) { /** * Creates the installer component for the given package. * @param {Component} that The gpii.iod instance. - * @param {PackageInfo} packageInfo The package data. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves with a gpii.iod.installer instance. */ -gpii.iod.initialiseInstallation = function (that, packageInfo) { - fluid.log("IoD: Initialising installation for " + packageInfo.name); +gpii.iod.initialiseInstallation = function (that, packageData) { + fluid.log("IoD: Initialising installation for " + packageData.name); // See if it's already been loaded var installation = fluid.find(that.installations, function (inst) { - return inst.packageName === packageInfo.name ? inst : undefined; + return inst.packageName === packageData.name ? inst : undefined; }); if (!installation) { installation = { id: fluid.allocateGuid(), - packageName: packageInfo.name, + packageName: packageData.name, cleanupPaths: [] }; } @@ -381,8 +375,8 @@ gpii.iod.initialiseInstallation = function (that, packageInfo) { var promise = fluid.promise(); // Create the installer instance. - installation.packageInfo = packageInfo; - var installerGrade = that.getInstaller(packageInfo.packageType); + installation.packageData = packageData; + var installerGrade = that.getInstaller(packageData.packageType); if (installerGrade) { // Load the installer. that.events.onInstallerLoad.fire(installerGrade, installation.id); @@ -390,7 +384,7 @@ gpii.iod.initialiseInstallation = function (that, packageInfo) { } else { promise.reject({ isError: true, - error: "Unable to find an installer for package type " + packageInfo.packageType + error: "Unable to find an installer for package type " + packageData.packageType }); } @@ -482,7 +476,7 @@ gpii.iod.uninstallPackage = function (that, installation) { if (installation && installation.installer) { initPromise = fluid.toPromise(installation); } else { - initPromise = that.initialiseInstallation(installation.packageInfo); + initPromise = that.initialiseInstallation(installation.packageData); } var promiseTogo = fluid.promise(); @@ -513,43 +507,8 @@ gpii.iod.uninstallPackage = function (that, installation) { * @param {Component} that The gpii.iod instance. */ gpii.iod.discoverServer = function (that) { - var addr = process.env.GPII_IOD_ENDPOINT || that.options.endpoint; - - if (addr === "auto") { - var bonjour = that.bonjourInstance || (that.bonjourInstance = require("bonjour")()); - if (bonjour) { - var browser = bonjour.find({type: "gpii-iod"}); - browser.on("up", function (service) { - fluid.log("IoD: Service up: " + service.fqdn); - if (that.endpoint) { - that.events.onServiceLost.fire(that.endpoint); - } - var endpoint = service.txt.url || ("https://" + service.host + ":" + service.port); - gpii.iod.checkService(endpoint).then(that.events.onServiceFound.fire); - }); - - browser.on("down", function (service) { - if (that.endpoint && that.endpointService === service.fqdn) { - fluid.log("IoD: Service down: " + service.fqdn); - var oldEndpoint = service.txt.url || ("https://" + service.host + ":" + service.port); - if (oldEndpoint === that.endpoint) { - that.events.onServiceLost.fire(that.endpoint); - } - } - }); - - // After a timeout use the default endpoint (if configured) - if (that.options.defaultEndpoint) { - setTimeout(function () { - if (!that.endpoint) { - fluid.log("IoD: No endpoint detected, trying " + that.options.defaultEndpoint); - gpii.iod.checkService(that.options.defaultEndpoint).then(that.events.onServiceFound.fire); - } - }, 5000); - } - } - } else if (addr) { + if (addr) { gpii.iod.checkService(addr).then(that.events.onServiceFound.fire); } }; @@ -566,24 +525,13 @@ gpii.iod.checkService = function (endpoint) { if (response) { promise.resolve(endpoint); } else { - fluid.log("IoD: Unable to connect to endpoint " + endpoint); + fluid.log("IoD: Unable to connect to endpoint " + endpoint + ": ", error); promise.reject(error); } }); return promise; }; -/** - * Invoked when the service endpoint is down. - * - * @param {Component} that The gpii.iod instance. - * @param {String} endpoint The endpoint address. - */ -gpii.iod.serviceLost = function (that, endpoint) { - fluid.log("IoD: Endpoint lost: " + endpoint); - that.endpoint = null; -}; - /** * Invoked when a service endpoint is up. * diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 297836845..4b13ab940 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -21,6 +21,7 @@ var path = require("path"), fs = require("fs"), request = require("request"), + crypto = require("crypto"), child_process = require("child_process"); var fluid = require("infusion"); @@ -49,8 +50,8 @@ fluid.defaults("gpii.iod.packageInstaller", { funcName: "gpii.iod.initialise", args: ["{that}", "{iod}"] }, - downloadPackage: { - funcName: "gpii.iod.downloadPackage", + downloadInstaller: { + funcName: "gpii.iod.downloadInstaller", args: ["{that}", "{iod}"] }, checkPackage: { @@ -88,7 +89,7 @@ fluid.defaults("gpii.iod.packageInstaller", { priority: "first" }, "onInstallPackage.download": { - func: "{that}.downloadPackage", + func: "{that}.downloadInstaller", priority: "after:initialise" }, "onInstallPackage.check": { @@ -127,7 +128,7 @@ fluid.defaults("gpii.iod.packageInstaller", { members: { // Package information from the server. - packageInfo: null + packageData: null } }); @@ -142,7 +143,7 @@ gpii.iod.installerCreated = function (that, iod) { that.installation = iod.installations[that.options.installationID]; if (that.installation) { that.installation.installer = that; - that.packageInfo = that.installation.packageInfo; + that.packageData = that.installation.packageData; } }; @@ -175,31 +176,33 @@ gpii.iod.startUninstaller = function (that) { * @param {Component} iod The gpii.iod instance. */ gpii.iod.initialise = function (that, iod) { - var tempDir = iod.getWorkingPath(that.packageInfo.name); + var tempDir = iod.getWorkingPath(that.packageData.name); that.installation.tempDir = tempDir.fullPath; that.installation.cleanupPaths.push(tempDir.createdPath); }; /** - * Downloads a package from the server. + * Downloads an installer from the server. * * @param {Component} that The gpii.iod.installer instance. * @param {Object} iod The gpii.iod instance. * @return {Promise} Resolves when complete. */ -gpii.iod.downloadPackage = function (that) { - fluid.log("IoD: Downloading package " + that.packageInfo.url); +gpii.iod.downloadInstaller = function (that) { + + + fluid.log("IoD: Downloading installer " + that.packageData.url); var promise = fluid.promise(); - that.installation.localPackage = path.join(that.installation.tempDir, that.packageInfo.filename); + that.installation.localPackage = path.join(that.installation.tempDir, that.packageData.filename); - if (that.packageInfo.url.startsWith("https://")) { + if (that.packageData.url.startsWith("https://")) { // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). - var downloadPromise = gpii.iod.httpsDownload(that.packageInfo.url, that.installation.localPackage); + var downloadPromise = gpii.iod.fileDownload(that.packageData.url, that.installation.localPackage); fluid.promise.follow(downloadPromise, promise); } else { - fs.copyFile(that.packageInfo.url, that.installation.localPackage, function (err) { + fs.copyFile(that.packageData.url, that.installation.localPackage, function (err) { if (err) { promise.reject({ isError: true, @@ -215,58 +218,53 @@ gpii.iod.downloadPackage = function (that) { }; /** - * Downloads a file, trying extra hard to use only https. + * Downloads a file while generating its hash. * * @param {String} url The remote uri. * @param {String} localPath Destination path. - * @return {Promise} Resolves when done. + * @param {Object} options Options + * @param {String} options.hash The hash algorithm (default: sha512) + * @return {Promise} Resolves with the hash when the download is complete. */ -gpii.iod.httpsDownload = function (url, localPath) { +gpii.iod.fileDownload = function (url, localPath, options) { + options = Object.assign({ + hash: "sha512" + }, options); + var promise = fluid.promise(); + var output = fs.createWriteStream(localPath); - output.on("finish", function () { - promise.resolve(); + var hash = crypto.createHash(options.hash); + + hash.on("finish", function () { + promise.resolve(hash.digest()); }); - if (url.startsWith("https:")) { - var req = request.get({ + var req = request.get({ + url: url + }); + + req.on("error", function (err) { + promise.reject({ + isError: true, + message: "Unable to download package: " + err.message, url: url, - strictSSL: true, - // Force https (and fail) if http is attempted. - httpModules: {"http:": require("https")}, - // Don't permit redirecting to non-https. - followRedirect: function (response) { - var allow = response.caseless.get("location").startsWith("https:"); - if (!allow) { - fluid.log("IoD: Denying non-https redirect"); - } - return allow; - } + error: err }); + }); - req.on("error", function (err) { + req.on("response", function (response) { + if (response.statusCode === 200) { + response.pipe(output); + response.pipe(hash); + } else { promise.reject({ isError: true, - message: "Unable to download package: " + err.message, - error: err + message: "Unable to download package: " + response.statusCode + " " + response.statusMessage, + url: url }); - }); - - req.on("response", function (response) { - if ((response.statusCode >= 300) && (response.statusCode < 400)) { - req.emit("error", { - message: "Redirect failed" - }); - } - }); - - req.pipe(output); - } else { - promise.reject({ - isError: true, - message: "IoD only supports HTTPS" - }); - } + } + }); return promise; }; @@ -280,7 +278,7 @@ gpii.iod.httpsDownload = function (url, localPath) { */ gpii.iod.checkPackage = function (that) { var promise = fluid.promise(); - fluid.log("IoD: Checking downloaded package file " + that.packageInfo.filename); + fluid.log("IoD: Checking downloaded package file " + that.packageData.filename); // TODO: It shouldn't be checked here - another process may over-write it before the high privilege executes it. // Instead, take ownership then check the integrity in the same context as it's being ran. promise.resolve(); @@ -296,7 +294,7 @@ gpii.iod.checkPackage = function (that) { */ gpii.iod.prepareInstall = function (that) { var promise = fluid.promise(); - fluid.log("IoD: Preparing installation for " + that.packageInfo.name); + fluid.log("IoD: Preparing installation for " + that.packageData.name); promise.resolve(); return promise; }; @@ -309,7 +307,7 @@ gpii.iod.prepareInstall = function (that) { */ gpii.iod.cleanup = function (that) { var promise = fluid.promise(); - fluid.log("IoD: Cleaning installation of " + that.packageInfo.name); + fluid.log("IoD: Cleaning installation of " + that.packageData.name); promise.resolve(); return promise; }; @@ -322,9 +320,9 @@ gpii.iod.cleanup = function (that) { */ gpii.iod.startApplication = function (that) { var promise = fluid.promise(); - fluid.log("IoD: Starting application " + that.packageInfo.name); - if (that.packageInfo.start) { - child_process.exec(that.packageInfo.start, function (err, stdout, stderr) { + fluid.log("IoD: Starting application " + that.packageData.name); + if (that.packageData.start) { + child_process.exec(that.packageData.start, function (err, stdout, stderr) { if (err) { fluid.log("IoD: startApplication error: ", err); } @@ -343,9 +341,9 @@ gpii.iod.startApplication = function (that) { */ gpii.iod.stopApplication = function (that) { var promise = fluid.promise(); - fluid.log("IoD: Stopping application " + that.packageInfo.name); - if (that.packageInfo.start) { - child_process.exec(that.packageInfo.start, function (err, stdout, stderr) { + fluid.log("IoD: Stopping application " + that.packageData.name); + if (that.packageData.start) { + child_process.exec(that.packageData.start, function (err, stdout, stderr) { if (err) { fluid.log("IoD: stopApplication error: ", err); } diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 7c1baa52f..6692b30de 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -33,11 +33,16 @@ require("./iodSettingsHandler.js"); /** * Information about a package. - * @typedef {Object} PackageInfo - * @property {string} name - The package name. - * @property {string} url - The package location. - * @property {string} filename - The package filename. - * @property {string} packageType - Type of installer to use. + * @typedef {Object} PackageData + * @property {String} name The package name. + * @property {String} url The package location. + * @property {String} filename The package filename. + * @property {String} packageType Type of installer to use. + * + * @property {String} installerFilename Original filename of the installer file. + * @property {String} installerSize Size of installer. + * @property {String} installerHash Installer sha512 hash. + * @property {String} publicKey Public key used to verify the package data. * */ @@ -46,7 +51,7 @@ gpii.iod.resolvers = {}; fluid.defaults("gpii.iod.packages", { gradeNames: ["fluid.component"], components: { - packageData: { + packageDataSource: { type: "kettle.dataSource.file", options: { gradeNames: ["kettle.dataSource.file.moduleTerms", "kettle.dataSource.encoding.JSON5"], @@ -61,7 +66,7 @@ fluid.defaults("gpii.iod.packages", { } } }, - remotePackageData: { + remotePackageDataSource: { createOnEvent: "onServiceFound", type: "kettle.dataSource.URL" }, @@ -70,17 +75,17 @@ fluid.defaults("gpii.iod.packages", { } }, invokers: { - getPackageInfo: { - funcName: "gpii.iod.getPackageInfo", + getPackageData: { + funcName: "gpii.iod.getPackageData", args: ["{that}", "{arguments}.0"] // packageRequest }, checkInstalled: { funcName: "gpii.iod.checkInstalled", - args: ["{that}", "{arguments}.0"] // packageInfo + args: ["{that}", "{arguments}.0"] // packageData }, resolvePackage: { funcName: "gpii.iod.resolvePackage", - args: ["{that}", "{arguments}.0"] // packageInfo + args: ["{that}", "{arguments}.0"] // packageData } }, listeners: { @@ -121,20 +126,20 @@ gpii.iod.existsResolver = function (path) { }; /** - * Resolves ${} variables of fields in a packageInfo. + * Resolves ${} variables of fields in a packageData. * * @param {Component} that The gpii.iod.packages instance. - * @param {PackageInfo} packageInfo The package. - * @return {PackageInfo} A copy of the package data, with resolved fields. + * @param {PackageData} packageData The package. + * @return {PackageData} A copy of the package data, with resolved fields. */ -gpii.iod.resolvePackage = function (that, packageInfo) { +gpii.iod.resolvePackage = function (that, packageData) { var result; - if (packageInfo._original) { + if (packageData._original) { // This package has already been resolved; work on the original copy. - result = gpii.iod.resolvePackage(that, packageInfo._original); + result = gpii.iod.resolvePackage(that, packageData._original); } else { - result = fluid.copy(packageInfo); + result = fluid.copy(packageData); // Allow references to the package itself via "${{this}.field}". var fetchers = gpii.combineFetchers(that.fetcher, gpii.resolversToFetcher({"this": result})); @@ -163,7 +168,7 @@ gpii.iod.resolvePackage = function (that, packageInfo) { }); // Stash the original, so it can be re-resolved. - result._original = fluid.freezeRecursive(packageInfo); + result._original = fluid.freezeRecursive(packageData); } return result; @@ -179,12 +184,12 @@ gpii.iod.resolvePackage = function (that, packageInfo) { * @param {String} packageRequest.language [optional] Language code with optional country code (en, en-US, es-ES). * @return {Promise} Resolves to an object containing package information. */ -gpii.iod.getPackageInfo = function (that, packageRequest) { +gpii.iod.getPackageData = function (that, packageRequest) { fluid.log("IoD: Getting package info for " + packageRequest.packageName); var promise = fluid.promise(); - var dataSource = that.remotePackageData || that.packageData; + var dataSource = that.remotepackageDataSource || that.packageDataSource; if (dataSource) { dataSource.get({ @@ -192,18 +197,18 @@ gpii.iod.getPackageInfo = function (that, packageRequest) { language: packageRequest.language, version: packageRequest.version, server: that.remoteServer - }).then(function (packageInfo) { - if (packageRequest.language && packageInfo.languages) { + }).then(function (packageData) { + if (packageRequest.language && packageData.languages) { // Merge the language-specific info. - var lang = gpii.iod.matchLanguage(Object.keys(packageInfo.languages), packageRequest.language); + var lang = gpii.iod.matchLanguage(Object.keys(packageData.languages), packageRequest.language); if (lang) { - Object.assign(packageInfo, packageInfo.languages[lang]); - packageInfo.language = lang; + Object.assign(packageData, packageData.languages[lang]); + packageData.language = lang; } } - var resolvedPackageInfo = that.resolvePackage(packageInfo); - promise.resolve(resolvedPackageInfo); + var resolvedPackageData = that.resolvePackage(packageData); + promise.resolve(resolvedPackageData); }, function (err) { promise.reject({ isError: true, @@ -261,15 +266,15 @@ gpii.iod.matchLanguage = function (languages, language) { * Determines if a package is installed. * * @param {Component} that The gpii.iod.packages instance. - * @param {PackageInfo} packageInfo The package data. + * @param {PackageData} packageData The package data. * @return {Boolean} true if the package is installed. */ -gpii.iod.checkInstalled = function (that, packageInfo) { +gpii.iod.checkInstalled = function (that, packageData) { // Update the isInstalled, to reflect the current situation. - packageInfo = that.resolvePackage(packageInfo); + packageData = that.resolvePackage(packageData); - var isInstalled = packageInfo.isInstalled; + var isInstalled = packageData.isInstalled; if (fluid.isPlainObject(isInstalled)) { isInstalled = isInstalled.isInstalled || isInstalled.value; } diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index dac6e6329..d6361610f 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -136,7 +136,7 @@ fluid.defaults("gpii.tests.iod", { "onCreate.readInstallations": null }, distributeOptions: { - packageData: { + packageDataSource: { record: { gradeNames: ["kettle.dataSource.file.moduleTerms"], path: __dirname + "/testPackages/%packageName.json5", @@ -144,7 +144,7 @@ fluid.defaults("gpii.tests.iod", { "packageName": "%packageName" } }, - target: "{that packages packageData}.options" + target: "{that packages packageDataSource}.options" } }, components: { @@ -340,7 +340,7 @@ jqUnit.asyncTest("test requirePackage", function () { test.expect, iod.funcCalled.startInstaller); process.nextTick(nextTest); }, function () { - jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); + jqUnit.assertEquals("packageData must only reject if expected" + suffix, test.expect, "reject"); jqUnit.assert("balance the assert count"); process.nextTick(nextTest); }); diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index 011b28cbf..7e0e7273c 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -44,7 +44,7 @@ jqUnit.module("gpii.tests.iodPackages", { } }); -gpii.tests.iodPackages.getPackageInfoTests = fluid.freezeRecursive([ +gpii.tests.iodPackages.getPackageDataTests = fluid.freezeRecursive([ { id: "No matching package", request: { @@ -498,7 +498,7 @@ gpii.tests.iodPackages.checkInstalledTests = fluid.freezeRecursive([ fluid.defaults("gpii.tests.iodPackages", { gradeNames: [ "gpii.iod" ], distributeOptions: { - packageData: { + packageDataSource: { record: { gradeNames: ["kettle.dataSource.file.moduleTerms"], path: __dirname + "/testPackages/%packageName.json5", @@ -506,7 +506,7 @@ fluid.defaults("gpii.tests.iodPackages", { "packageName": "%packageName" } }, - target: "{that packages packageData}.options" + target: "{that packages packageDataSource}.options" } }, invokers: { @@ -563,10 +563,10 @@ jqUnit.test("test the package resolver", function () { iod.destroy(); }); -// Test getPackageInfo returns correct information -jqUnit.asyncTest("test getPackageInfo", function () { +// Test getPackageData returns correct information +jqUnit.asyncTest("test getPackageData", function () { - var tests = gpii.tests.iodPackages.getPackageInfoTests; + var tests = gpii.tests.iodPackages.getPackageDataTests; jqUnit.expect(tests.length * 2); var iod = gpii.tests.iodPackages(); @@ -583,20 +583,20 @@ jqUnit.asyncTest("test getPackageInfo", function () { fluid.log("getPackage: " + test.request.packageName + ", " + test.request.language); - var p = iod.packages.getPackageInfo(test.request); + var p = iod.packages.getPackageData(test.request); - jqUnit.assertTrue("getPackageInfo must return a promise" + suffix, fluid.isPromise(p)); + jqUnit.assertTrue("getPackageData must return a promise" + suffix, fluid.isPromise(p)); - p.then(function (packageInfo) { - delete packageInfo.languages; - delete packageInfo._original; - jqUnit.assertDeepEq("packageInfo must match expected" + suffix, test.expect, packageInfo); + p.then(function (packageData) { + delete packageData.languages; + delete packageData._original; + jqUnit.assertDeepEq("packageData must match expected" + suffix, test.expect, packageData); nextTest(); }, function (e) { if (test.expect !== "reject") { fluid.log(e); } - jqUnit.assertEquals("packageInfo must only reject if expected" + suffix, test.expect, "reject"); + jqUnit.assertEquals("packageData must only reject if expected" + suffix, test.expect, "reject"); nextTest(); }); From c29c3bbb2492c37570dc19c4b285a624b52f4934 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 7 Nov 2019 10:14:22 +0000 Subject: [PATCH 32/56] GPII-2971: Serve & download package data + installer --- gpii/node_modules/gpii-iod/index.js | 1 + .../gpii-iod/src/installOnDemand.js | 32 ++---- .../gpii-iod/src/packageDataSource.js | 100 ++++++++++++++++++ .../gpii-iod/src/packageInstaller.js | 44 ++++---- gpii/node_modules/gpii-iod/src/packages.js | 57 +++++++--- .../gpii-iod/test/packagesTests.js | 80 ++++++++++++++ 6 files changed, 262 insertions(+), 52 deletions(-) create mode 100644 gpii/node_modules/gpii-iod/src/packageDataSource.js diff --git a/gpii/node_modules/gpii-iod/index.js b/gpii/node_modules/gpii-iod/index.js index 85f14a61d..a074646e4 100644 --- a/gpii/node_modules/gpii-iod/index.js +++ b/gpii/node_modules/gpii-iod/index.js @@ -26,3 +26,4 @@ require("./src/iodSettingsHandler.js"); require("./src/installOnDemand.js"); require("./src/packageInstaller.js"); require("./src/packages.js"); +require("./src/packageDataSource.js"); diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 1da592b5a..a3caa7eb6 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -33,7 +33,7 @@ fluid.registerNamespace("gpii.iod"); * Installation state. * @typedef {Object} Installation * @property {id} - Installation ID - * @property {packageData} packageData - Package data. + * @property {PackageData} packageData - Package data. * @property {string} packageName - packageData.name * @property {Component} installer - The gpii.iod.installer instance. * @property {boolean} failed - true if the installation had failed. @@ -60,7 +60,7 @@ fluid.defaults("gpii.iod", { type: "gpii.iod.packages", options: { events: { - "onServiceFound": "onServiceFound" + "onServerFound": "{gpii.iod}.events.onServerFound" } } } @@ -75,13 +75,16 @@ fluid.defaults("gpii.iod", { } }, events: { - onServiceFound: null, // [ endpoint address ] + onServerFound: null, // [ endpoint address ] onInstallerLoad: null // [ packageInstaller grade name, installation ID ] }, listeners: { "onCreate.discoverServer": "{that}.discoverServer", "onCreate.readInstallations": "{that}.readInstallations", - "onServiceFound": "{that}.serviceFound", + "onServerFound.setEndpoint": { + funcName: "fluid.set", + args: [ "{that}", "endpoint", "{arguments}.0"] + } }, invokers: { discoverServer: { @@ -104,10 +107,6 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.getWorkingPath", args: ["{arguments}.0"] }, - serviceFound: { - funcName: "gpii.iod.serviceFound", - args: ["{that}", "{arguments}.0"] - }, readInstallations: { funcName: "gpii.iod.readInstallations", args: ["{that}", "{gpii.journal}.settingsDir.gpiiSettingsDir" ] @@ -509,7 +508,7 @@ gpii.iod.uninstallPackage = function (that, installation) { gpii.iod.discoverServer = function (that) { var addr = process.env.GPII_IOD_ENDPOINT || that.options.endpoint; if (addr) { - gpii.iod.checkService(addr).then(that.events.onServiceFound.fire); + gpii.iod.checkService(addr).then(that.events.onServerFound.fire); } }; @@ -517,12 +516,14 @@ gpii.iod.discoverServer = function (that) { * Check if an endpoint is listening for connections. * * @param {String} endpoint The service end point URI - * @return {Promise} Resolves + * @return {Promise} Resolves with the endpoint address, rejects if it can't be connected to. */ gpii.iod.checkService = function (endpoint) { var promise = fluid.promise(); + endpoint = gpii.iod.joinUrl(endpoint, "iod"); request(endpoint, function (error, response) { if (response) { + fluid.log("IoD: Endpoint found: " + endpoint); promise.resolve(endpoint); } else { fluid.log("IoD: Unable to connect to endpoint " + endpoint + ": ", error); @@ -531,14 +532,3 @@ gpii.iod.checkService = function (endpoint) { }); return promise; }; - -/** - * Invoked when a service endpoint is up. - * - * @param {Component} that The gpii.iod instance. - * @param {String} endpoint The endpoint address. - */ -gpii.iod.serviceFound = function (that, endpoint) { - fluid.log("IoD: Endpoint found: " + endpoint); - that.endpoint = endpoint; -}; diff --git a/gpii/node_modules/gpii-iod/src/packageDataSource.js b/gpii/node_modules/gpii-iod/src/packageDataSource.js new file mode 100644 index 000000000..353b7f1f7 --- /dev/null +++ b/gpii/node_modules/gpii-iod/src/packageDataSource.js @@ -0,0 +1,100 @@ +/* + * Install on Demand package data source. + * + * Copyright 2019 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"), + crypto = require("crypto"); + +require("kettle"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.iod.packages"); + +fluid.defaults("gpii.iod.remotePackageDataSource", { + gradeNames: ["kettle.dataSource.URL"], + listeners: { + "onRead.checkSignature": { + funcName: "gpii.iod.checkPackageSignature" + } + } +}); + +/** + * Verifies the packageData JSON string against the packageDataSignature, returns the response with the packageData + * field de-serialised. + * @param {PackageResponse} packageResponse The response from the data source. + * @return {Promise} Resolves with the response from the data source, with de-serialised packageData. + */ +gpii.iod.checkPackageSignature = function (packageResponse) { + var promise = fluid.promise(); + gpii.iod.verifySignedJSON(packageResponse.packageData, packageResponse.packageDataSignature).then(function (obj) { + var result = fluid.copy(packageResponse); + result.packageData = obj; + promise.resolve(result); + }, promise.reject); + + return promise; +}; + +/** + * Verifies a serialised object against a signature. The base64 encoded public key is inside the object under the + * `publicKey` field. + * + * @param {String} data The JSON data to verify (expects a `publicKey` field) + * @param {String} signature The signature (base64) + * @return {Promise} Resolves, with the de-serialised object, when complete. + */ +gpii.iod.verifySignedJSON = function (data, signature) { + var promise = fluid.promise(); + + try { + var verified = false; + var obj = JSON.parse(data); + if (obj.publicKey) { + // PEM encode the key - it's already base64 encoded, so just surround with the header and footer. + var publicKey = "-----BEGIN PUBLIC KEY-----\n" + + obj.publicKey.trim() + + "\n-----END PUBLIC KEY-----\n"; + + // Verify the package data with the signature. + var verify = crypto.createVerify("RSA-SHA512"); + verify.update(data); + + verified = verify.verify({key: publicKey}, signature, "base64"); + } + + if (verified) { + promise.resolve(obj); + } else { + var extra = obj.publicKey ? "" : ": JSON object did not contain a publicKey field."; + promise.reject({ + isError: true, + message: "Signed JSON data failed verification" + extra + }); + } + } catch (e) { + promise.reject({ + isError: true, + message: "Error while verifying signed JSON data: " + (e.message || ""), + error: e + }); + } + + return promise; +}; diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 4b13ab940..f3774aabd 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -45,7 +45,7 @@ fluid.defaults("gpii.iod.packageInstaller", { args: ["{that}", "{iod}"] }, // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns - // a installation, either directly or via a promise. + // an installation, either directly or via a promise. initialise: { funcName: "gpii.iod.initialise", args: ["{that}", "{iod}"] @@ -190,31 +190,36 @@ gpii.iod.initialise = function (that, iod) { */ gpii.iod.downloadInstaller = function (that) { - - fluid.log("IoD: Downloading installer " + that.packageData.url); + fluid.log("IoD: Downloading installer " + that.packageData.installerSource); var promise = fluid.promise(); - that.installation.localPackage = path.join(that.installation.tempDir, that.packageData.filename); + that.installation.localPackage = path.join(that.installation.tempDir, that.packageData.installer); - if (that.packageData.url.startsWith("https://")) { - // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). - var downloadPromise = gpii.iod.fileDownload(that.packageData.url, that.installation.localPackage); - fluid.promise.follow(downloadPromise, promise); + if (that.packageData.installerSource) { + if (/^https?:\/\//.test(that.packageData.installerSource)) { + // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). + var downloadPromise = gpii.iod.fileDownload(that.packageData.installerSource, that.installation.localPackage); + fluid.promise.follow(downloadPromise, promise); + } else { + fs.copyFile(that.packageData.url, that.installation.localPackage, function (err) { + if (err) { + promise.reject({ + isError: true, + message: "Unable to copy package" + }); + } else { + promise.resolve(); + } + }); + } } else { - fs.copyFile(that.packageData.url, that.installation.localPackage, function (err) { - if (err) { - promise.reject({ - isError: true, - message: "Unable to copy package" - }); - } else { - promise.resolve(); - } - }); + promise.resolve(); } - return promise; + return promise.then(null, function (err) { + fluid.log("IoD: Failed download of " + that.packageData.installerSource + ": ", err); + }); }; /** @@ -224,6 +229,7 @@ gpii.iod.downloadInstaller = function (that) { * @param {String} localPath Destination path. * @param {Object} options Options * @param {String} options.hash The hash algorithm (default: sha512) + * @param {Function} options.process Callback for the progress, called with current and total. * @return {Promise} Resolves with the hash when the download is complete. */ gpii.iod.fileDownload = function (url, localPath, options) { diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 6692b30de..a26ffff2a 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -35,13 +35,12 @@ require("./iodSettingsHandler.js"); * Information about a package. * @typedef {Object} PackageData * @property {String} name The package name. - * @property {String} url The package location. - * @property {String} filename The package filename. * @property {String} packageType Type of installer to use. * - * @property {String} installerFilename Original filename of the installer file. + * @property {String} installer Original filename of the installer file. * @property {String} installerSize Size of installer. * @property {String} installerHash Installer sha512 hash. + * @property {String} installerSource The installer location (where to download/copy it from). * @property {String} publicKey Public key used to verify the package data. * */ @@ -67,8 +66,15 @@ fluid.defaults("gpii.iod.packages", { } }, remotePackageDataSource: { - createOnEvent: "onServiceFound", - type: "kettle.dataSource.URL" + createOnEvent: "onServerFound", + type: "gpii.iod.remotePackageDataSource", + options: { + url: "@expand:gpii.iod.joinUrl({arguments}.0, {that}.options.urlPath)", + urlPath: "/packages/%packageName", + termMap: { + packageName: "%packageName" + } + } }, variableResolver: { type: "gpii.lifecycleManager.variableResolver" @@ -89,7 +95,11 @@ fluid.defaults("gpii.iod.packages", { } }, listeners: { - onCreate: "fluid.identity" + onCreate: "fluid.identity", + onServerFound: { + funcName: "fluid.set", + args: [ "{that}", "endpoint", "{arguments}.0"] + } }, members: { resolvers: { @@ -103,7 +113,9 @@ fluid.defaults("gpii.iod.packages", { func: "gpii.resolversToFetcher", args: "{that}.resolvers" } - } + }, + // The IoD server, set from onServerFound + endpoint: null }, resolvers: { exists: "gpii.iod.existsResolver" @@ -111,6 +123,18 @@ fluid.defaults("gpii.iod.packages", { }); +/** + * Convenience function to concatenate two parts of a URL together, ensuring there's a single '/' between each part + * (like `path.join`, but performs no normalisation and always uses slash). + * + * @param {String} front The first part of the URL + * @param {String} end The final part of the URL + * @return {String} `font` and `end` combined. + */ +gpii.iod.joinUrl = function (front, end) { + return front.replace(/\/+$/, "") + "/" + end.replace(/^\/+/, ""); +}; + /** * Resolver for ${{exists}.path}, determines if a filesystem path exists. The path can include environment variables, * named between two '%' symbols (like %this%), to work around the resolvers not supporting nested expressions. @@ -189,16 +213,25 @@ gpii.iod.getPackageData = function (that, packageRequest) { var promise = fluid.promise(); - var dataSource = that.remotepackageDataSource || that.packageDataSource; + var remote = !!that.remotePackageDataSource; + var dataSource = remote ? that.remotePackageDataSource : that.packageDataSource; if (dataSource) { dataSource.get({ packageName: packageRequest.packageName, language: packageRequest.language, - version: packageRequest.version, - server: that.remoteServer - }).then(function (packageData) { - if (packageRequest.language && packageData.languages) { + version: packageRequest.version + }).then(function (packageResponse) { + // Remote datasource wraps the packageData, local doesn't. + /** @type PackageData */ + var packageData = remote ? packageResponse.packageData : packageResponse; + + if (remote && packageResponse.installer) { + packageData.installerSource = gpii.iod.joinUrl(that.endpoint, packageResponse.installer); + } + + if (packageRequest.language && packageData.languages) + { // Merge the language-specific info. var lang = gpii.iod.matchLanguage(Object.keys(packageData.languages), packageRequest.language); if (lang) { diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index 7e0e7273c..3c1e5e01a 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -23,6 +23,7 @@ var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); var JSON5 = require("json5"), + crypto = require("crypto"), fs = require("fs"), path = require("path"), os = require("os"); @@ -632,6 +633,85 @@ jqUnit.test("test checkInstalled", function () { }); delete process.env[testEnv]; +}); + +jqUnit.asyncTest("testing package data signature verification", function () { + // Create a key pair + var passphrase = "test"; + var keyPair = crypto.generateKeyPairSync("rsa", { + modulusLength: 4096, + publicKeyEncoding: { + type: "spki", + format: "pem" + }, + privateKeyEncoding: { + type: "pkcs1", + format: "pem", + cipher: "aes-128-cbc", + passphrase: passphrase + } + }); + + // The key is already base64 encoded - just remove the PEM header and footer. + var re = new RegExp(".*(:?\n|^)-----BEGIN[^\n]*\n(.*\n)?-----END.*", "s"); + var publicKey = re.exec(keyPair.publicKey)[2]; + + // The object to be signed + var obj = { + a: "this object will be signed", + b: { + c: "extra", + d: Math.random() + }, + publicKey: publicKey + }; + + var json = JSON.stringify(obj); + var buffer = Buffer.from(json, "utf8"); + + // Sign it + var sign = crypto.createSign("RSA-SHA512"); + sign.update(buffer); + var signature = sign.sign({ + key: keyPair.privateKey, + passphrase: passphrase + }).toString("base64"); + + jqUnit.expect(3); + var tests = [ + function () { + // Verify it. + return gpii.iod.verifySignedJSON(json, signature).then(function () { + jqUnit.assert("verifySignedJSON should resolve with correctly signed data"); + }); + }, + function () { + // Run it through checkPackageSignature + /** @type PackageResponse */ + var packageResponse = { + installer: true, + packageData: json, + packageDataSignature: signature + }; + return gpii.iod.checkPackageSignature(packageResponse).then(function (value) { + jqUnit.assertDeepEq("packageData resolved by checkPackageSignature should be de-serialised", + obj, value.packageData); + }); + }, + function () { + // Make a change to the signed data + var modified = json.replace(/this object/, "that object"); + var promise = fluid.promise(); + gpii.iod.verifySignedJSON(modified, signature).then(function () { + jqUnit.fail("verifySignedJSON should have rejected with modified data"); + promise.reject(); + }, function () { + jqUnit.assert("verifySignedJSON should reject with modified data"); + }); + return promise; + } + ]; + fluid.promise.sequence(tests).then(jqUnit.start, jqUnit.fail); }); From f84c2131e5f4a0b290b0bec5f5f157daed0c59da Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 11 Nov 2019 14:34:17 +0000 Subject: [PATCH 33/56] GPII-2971: Authorised code signing keys --- .../gpii-iod/src/installOnDemand.js | 3 + .../gpii-iod/src/packageDataSource.js | 79 +++++++++++++------ .../gpii-iod/test/packagesTests.js | 20 ++++- 3 files changed, 75 insertions(+), 27 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index a3caa7eb6..6532e360a 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -129,7 +129,10 @@ fluid.defaults("gpii.iod", { } }, + // The IoD server address endpoint: undefined, + // Map of recognised keys that sign the packages. + allowedKeys: {}, members: { installations: {} diff --git a/gpii/node_modules/gpii-iod/src/packageDataSource.js b/gpii/node_modules/gpii-iod/src/packageDataSource.js index 353b7f1f7..adf19f189 100644 --- a/gpii/node_modules/gpii-iod/src/packageDataSource.js +++ b/gpii/node_modules/gpii-iod/src/packageDataSource.js @@ -28,9 +28,16 @@ fluid.registerNamespace("gpii.iod.packages"); fluid.defaults("gpii.iod.remotePackageDataSource", { gradeNames: ["kettle.dataSource.URL"], + invokers: { + "checkPackageSignature": { + funcName: "gpii.iod.checkPackageSignature", + args: [ "{arguments}.0", "{gpii.iod}.options.allowedKeys" ] + } + }, listeners: { "onRead.checkSignature": { - funcName: "gpii.iod.checkPackageSignature" + func: "{that}.checkPackageSignature", + args: "{arguments}.0" // packageResponse } } }); @@ -39,56 +46,82 @@ fluid.defaults("gpii.iod.remotePackageDataSource", { * Verifies the packageData JSON string against the packageDataSignature, returns the response with the packageData * field de-serialised. * @param {PackageResponse} packageResponse The response from the data source. + * @param {Object} allowedKeys Map of sha256 fingerprints (base64 encoded) of the public keys that are authorised. * @return {Promise} Resolves with the response from the data source, with de-serialised packageData. */ -gpii.iod.checkPackageSignature = function (packageResponse) { +gpii.iod.checkPackageSignature = function (packageResponse, allowedKeys) { var promise = fluid.promise(); - gpii.iod.verifySignedJSON(packageResponse.packageData, packageResponse.packageDataSignature).then(function (obj) { - var result = fluid.copy(packageResponse); - result.packageData = obj; - promise.resolve(result); - }, promise.reject); + gpii.iod.verifySignedJSON(packageResponse.packageData, packageResponse.packageDataSignature, allowedKeys) + .then(function (packageData) { + var result = fluid.copy(packageResponse); + result.packageData = packageData; + promise.resolve(result); + }, promise.reject); return promise; }; /** - * Verifies a serialised object against a signature. The base64 encoded public key is inside the object under the - * `publicKey` field. + * Verifies a serialised object against a signature, and the public key is one of those specified. + * + * The base64 encoded public key is inside the object under the `publicKey` field. * * @param {String} data The JSON data to verify (expects a `publicKey` field) - * @param {String} signature The signature (base64) - * @return {Promise} Resolves, with the de-serialised object, when complete. + * @param {String} signature The signature (base64). + * @param {Object} allowedKeys Map of sha256 fingerprints (base64 encoded) of the public keys that are authorised. + * + * @return {Promise} Resolves, with the de-serialised object, when complete. Rejects if the signature doesn't validate, + * or if one of the keys aren't in the given list. */ -gpii.iod.verifySignedJSON = function (data, signature) { +gpii.iod.verifySignedJSON = function (data, signature, allowedKeys) { var promise = fluid.promise(); + var failureMessage; + var verified = false; try { - var verified = false; var obj = JSON.parse(data); if (obj.publicKey) { - // PEM encode the key - it's already base64 encoded, so just surround with the header and footer. - var publicKey = "-----BEGIN PUBLIC KEY-----\n" - + obj.publicKey.trim() - + "\n-----END PUBLIC KEY-----\n"; + // Get the sha256 fingerprint of the public key, and check if it's one of the allowed keys. + var fingerprint = crypto.createHash("sha256").update(Buffer.from(obj.publicKey, "base64")).digest("base64"); + var keyName = fluid.keyForValue(allowedKeys, fingerprint); + var authorised = keyName !== undefined; + + if (authorised) { + fluid.log("Package signing key: " + keyName); + // PEM encode the key - it's already base64 encoded, so just surround with the header and footer. + var publicKey = "-----BEGIN PUBLIC KEY-----\n" + + obj.publicKey.trim() + + "\n-----END PUBLIC KEY-----\n"; - // Verify the package data with the signature. - var verify = crypto.createVerify("RSA-SHA512"); - verify.update(data); + // Verify the package data with the signature. + var verify = crypto.createVerify("RSA-SHA512"); + verify.update(data); - verified = verify.verify({key: publicKey}, signature, "base64"); + verified = verify.verify({key: publicKey}, signature, "base64"); + + if (!verified) { + failureMessage = "Signature could not be verified."; + } + + } else { + verified = false; + failureMessage = "Signed by an unknown key."; + } + } else { + verified = false; + failureMessage = "JSON object did not contain a publicKey field."; } if (verified) { promise.resolve(obj); } else { - var extra = obj.publicKey ? "" : ": JSON object did not contain a publicKey field."; promise.reject({ isError: true, - message: "Signed JSON data failed verification" + extra + message: "Signed JSON data failed verification: " + failureMessage }); } } catch (e) { + verified = false; promise.reject({ isError: true, message: "Error while verifying signed JSON data: " + (e.message || ""), diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index 3c1e5e01a..1e05b1019 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -656,6 +656,9 @@ jqUnit.asyncTest("testing package data signature verification", function () { var re = new RegExp(".*(:?\n|^)-----BEGIN[^\n]*\n(.*\n)?-----END.*", "s"); var publicKey = re.exec(keyPair.publicKey)[2]; + // Get the key fingerprint + var fingerprint = crypto.createHash("sha256").update(Buffer.from(publicKey, "base64")).digest("base64"); + // The object to be signed var obj = { a: "this object will be signed", @@ -677,14 +680,23 @@ jqUnit.asyncTest("testing package data signature verification", function () { passphrase: passphrase }).toString("base64"); - jqUnit.expect(3); + jqUnit.expect(4); var tests = [ function () { // Verify it. - return gpii.iod.verifySignedJSON(json, signature).then(function () { + return gpii.iod.verifySignedJSON(json, signature, [fingerprint]).then(function () { jqUnit.assert("verifySignedJSON should resolve with correctly signed data"); }); }, + function () { + // Verify it, without a valid fingerprint. + return gpii.iod.verifySignedJSON(json, signature, ["xxx"]).then(function () { + jqUnit.fail("verifySignedJSON should have rejected with unknown fingerprint"); + }, + function () { + jqUnit.assert("verifySignedJSON should reject with unknown fingerprint"); + }); + }, function () { // Run it through checkPackageSignature /** @type PackageResponse */ @@ -693,7 +705,7 @@ jqUnit.asyncTest("testing package data signature verification", function () { packageData: json, packageDataSignature: signature }; - return gpii.iod.checkPackageSignature(packageResponse).then(function (value) { + return gpii.iod.checkPackageSignature(packageResponse, [fingerprint]).then(function (value) { jqUnit.assertDeepEq("packageData resolved by checkPackageSignature should be de-serialised", obj, value.packageData); }); @@ -702,7 +714,7 @@ jqUnit.asyncTest("testing package data signature verification", function () { // Make a change to the signed data var modified = json.replace(/this object/, "that object"); var promise = fluid.promise(); - gpii.iod.verifySignedJSON(modified, signature).then(function () { + gpii.iod.verifySignedJSON(modified, signature, [fingerprint]).then(function () { jqUnit.fail("verifySignedJSON should have rejected with modified data"); promise.reject(); }, function () { From 825a3044cdbb2b28890406fe4e63f0c1f4d3b537 Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 11 Nov 2019 15:52:44 +0000 Subject: [PATCH 34/56] GPII-2971: Removed endpoint modification --- gpii/node_modules/gpii-iod/src/installOnDemand.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 6532e360a..f5ba629ce 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -523,7 +523,7 @@ gpii.iod.discoverServer = function (that) { */ gpii.iod.checkService = function (endpoint) { var promise = fluid.promise(); - endpoint = gpii.iod.joinUrl(endpoint, "iod"); + request(endpoint, function (error, response) { if (response) { fluid.log("IoD: Endpoint found: " + endpoint); From 92f92a14622623230511f82d4d8321302ef8292b Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 13 Nov 2019 13:59:33 +0000 Subject: [PATCH 35/56] GPII-2971: Added IoD endpoint to siteconfig --- gpii/node_modules/gpii-iod/src/installOnDemand.js | 2 +- gpii/node_modules/gpii-iod/src/packageDataSource.js | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index f5ba629ce..87c433ec9 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -509,7 +509,7 @@ gpii.iod.uninstallPackage = function (that, installation) { * @param {Component} that The gpii.iod instance. */ gpii.iod.discoverServer = function (that) { - var addr = process.env.GPII_IOD_ENDPOINT || that.options.endpoint; + var addr = process.env.GPII_IOD_ENDPOINT || that.options.config.endpoint; if (addr) { gpii.iod.checkService(addr).then(that.events.onServerFound.fire); } diff --git a/gpii/node_modules/gpii-iod/src/packageDataSource.js b/gpii/node_modules/gpii-iod/src/packageDataSource.js index adf19f189..b05c1a0a0 100644 --- a/gpii/node_modules/gpii-iod/src/packageDataSource.js +++ b/gpii/node_modules/gpii-iod/src/packageDataSource.js @@ -31,7 +31,7 @@ fluid.defaults("gpii.iod.remotePackageDataSource", { invokers: { "checkPackageSignature": { funcName: "gpii.iod.checkPackageSignature", - args: [ "{arguments}.0", "{gpii.iod}.options.allowedKeys" ] + args: [ "{arguments}.0", "{gpii.iod}.options.config.allowedKeys" ] } }, listeners: { @@ -85,9 +85,10 @@ gpii.iod.verifySignedJSON = function (data, signature, allowedKeys) { var fingerprint = crypto.createHash("sha256").update(Buffer.from(obj.publicKey, "base64")).digest("base64"); var keyName = fluid.keyForValue(allowedKeys, fingerprint); var authorised = keyName !== undefined; + fluid.log("IoD: Package signing key: " + fingerprint + " - ", (keyName || "unknown")); if (authorised) { - fluid.log("Package signing key: " + keyName); + fluid.log("IoD: Package signing key name: " + keyName); // PEM encode the key - it's already base64 encoded, so just surround with the header and footer. var publicKey = "-----BEGIN PUBLIC KEY-----\n" + obj.publicKey.trim() From f8f68da4d77ab4ca45a655c6dcda7c8e6d380514 Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 13 Nov 2019 14:06:44 +0000 Subject: [PATCH 36/56] GPII-2971: Added support for windows installer (msi) --- .../gpii-iod/src/installOnDemand.js | 60 ++++--------------- .../gpii-iod/src/packageInstaller.js | 6 +- gpii/node_modules/gpii-iod/src/packages.js | 11 +++- .../gpii-iod/test/installOnDemandTests.js | 45 -------------- 4 files changed, 25 insertions(+), 97 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 87c433ec9..82af5eac0 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -38,23 +38,13 @@ fluid.registerNamespace("gpii.iod"); * @property {Component} installer - The gpii.iod.installer instance. * @property {boolean} failed - true if the installation had failed. * @property {string} tmpDir - Temporary working directory. - * @property {string} localPackage - Path to the downloaded package file. + * @property {string} installerFile - Path to the downloaded package file. * @property {string[]} cleanupPaths - The directories to remove during cleanup. * */ fluid.defaults("gpii.iod", { - gradeNames: ["fluid.component", "fluid.contextAware", "fluid.modelComponent"], - contextAwareness: { - platform: { - checks: { - windows: { - contextValue: "{gpii.contexts.windows}", - gradeNames: "gpii.windows.iod" - } - } - } - }, + gradeNames: ["fluid.component", "fluid.modelComponent"], components: { packages: { type: "gpii.iod.packages", @@ -99,10 +89,6 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.initialiseInstallation", args: ["{that}", "{arguments}.0"] }, - getInstaller: { - funcName: "gpii.iod.getInstaller", - args: ["{that}", "{arguments}.0"] - }, getWorkingPath: { funcName: "gpii.iod.getWorkingPath", args: ["{arguments}.0"] @@ -133,6 +119,8 @@ fluid.defaults("gpii.iod", { endpoint: undefined, // Map of recognised keys that sign the packages. allowedKeys: {}, + // Map of installer type -> grade name, for each type of installer. + installerGrades: {}, members: { installations: {} @@ -270,26 +258,6 @@ gpii.iod.getWorkingPath = function (packageName) { }; }; -/** - * Finds a package installer component that handles the given type of package. - * - * @param {Component} that The gpii.iod instance. - * @param {String} packageType The package type identifier. - * @return {String} The grade name of the package installer. - */ -gpii.iod.getInstaller = function (that, packageType) { - var packageInstallers = fluid.queryIoCSelector(that, "gpii.iod.packageInstaller"); - - var installerComponent = fluid.find(packageInstallers, function (installer) { - var packageTypes = fluid.makeArray(installer.options.packageTypes); - return packageTypes.indexOf(packageType) >= 0 - ? installer - : undefined; - }); - - return installerComponent && installerComponent.typeName; -}; - /** * Starts the process of installing a package. * @@ -318,17 +286,13 @@ gpii.iod.requirePackage = function (that, packageRequest) { } else { that.initialiseInstallation(packageData).then(function (installation) { installation.required = true; - if (installation.installed) { - promise.resolve(false); - } else { - installation.installer.startInstaller().then(function () { - installation.installed = true; - installation.gpiiInstalled = true; - // Store the installation info so it can still get removed if gpii restarts. - that.writeInstallation(installation); - promise.resolve(true); - }, promise.reject); - } + installation.installer.startInstaller().then(function () { + installation.installed = true; + installation.gpiiInstalled = true; + // Store the installation info so it can still get removed if gpii restarts. + that.writeInstallation(installation); + promise.resolve(true); + }, promise.reject); // Destroy the installer var destroy = function () { @@ -378,7 +342,7 @@ gpii.iod.initialiseInstallation = function (that, packageData) { // Create the installer instance. installation.packageData = packageData; - var installerGrade = that.getInstaller(packageData.packageType); + var installerGrade = that.options.installerGrades[packageData.packageType]; if (installerGrade) { // Load the installer. that.events.onInstallerLoad.fire(installerGrade, installation.id); diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index f3774aabd..3a82e7596 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -194,15 +194,15 @@ gpii.iod.downloadInstaller = function (that) { var promise = fluid.promise(); - that.installation.localPackage = path.join(that.installation.tempDir, that.packageData.installer); + that.installation.installerFile = path.join(that.installation.tempDir, that.packageData.installer); if (that.packageData.installerSource) { if (/^https?:\/\//.test(that.packageData.installerSource)) { // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). - var downloadPromise = gpii.iod.fileDownload(that.packageData.installerSource, that.installation.localPackage); + var downloadPromise = gpii.iod.fileDownload(that.packageData.installerSource, that.installation.installerFile); fluid.promise.follow(downloadPromise, promise); } else { - fs.copyFile(that.packageData.url, that.installation.localPackage, function (err) { + fs.copyFile(that.packageData.url, that.installation.installerFile, function (err) { if (err) { promise.reject({ isError: true, diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index a26ffff2a..188ce8bb0 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -37,11 +37,19 @@ require("./iodSettingsHandler.js"); * @property {String} name The package name. * @property {String} packageType Type of installer to use. * + * @property {String} publicKey Public key used to verify the package data. + * * @property {String} installer Original filename of the installer file. * @property {String} installerSize Size of installer. * @property {String} installerHash Installer sha512 hash. * @property {String} installerSource The installer location (where to download/copy it from). - * @property {String} publicKey Public key used to verify the package data. + * + * @property {String|String[]} installerArgs Additional arguments to pass to the installer. + * @property {String|String[]} uninstallerArgs Additional arguments to pass to the installer, when uninstalling. + * + * @property {Boolean} elevate true to install as the administrator (installer specific). + * @property {String} uiLevel How much is displayed (if possible): "none" (default), "progress" (non-interactive), + * "progress-cancel" (progress, can be cancelled), "full" (fully interactive, asks questions). * */ @@ -222,6 +230,7 @@ gpii.iod.getPackageData = function (that, packageRequest) { language: packageRequest.language, version: packageRequest.version }).then(function (packageResponse) { + fluid.log("IoD: Package response: ", packageResponse); // Remote datasource wraps the packageData, local doesn't. /** @type PackageData */ var packageData = remote ? packageResponse.packageData : packageResponse; diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index d6361610f..57f81fd72 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -44,31 +44,6 @@ jqUnit.module("gpii.tests.iod", { } }); - -gpii.tests.iod.getInstallerTests = fluid.freezeRecursive([ - { - packageType: "testPackageType1", - expect: "gpii.tests.iod.testInstaller1" - }, - { - packageType: "testPackageType2a", - expect: "gpii.tests.iod.testInstaller2" - }, - { - packageType: "testPackageType2b", - expect: "gpii.tests.iod.testInstaller2" - }, - { - // Fails at installation, not during initialisation. - packageType: "testFailPackageType", - expect: "gpii.tests.iod.testInstallerFail" - }, - { - packageType: "testPackageType-not-exist", - expect: undefined - } -]); - gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ { packageRequest: "no-such-package", @@ -291,26 +266,6 @@ jqUnit.test("test getWorkingPath", function () { } }); -// Test getInstaller returns the correct installer -jqUnit.test("test getInstaller", function () { - - var tests = gpii.tests.iod.getInstallerTests; - - var iod = gpii.tests.iod(); - - fluid.each(tests, function (test) { - - var installer = iod.getInstaller(test.packageType); - - if (test.expect) { - jqUnit.assertEquals("getInstaller should return the correct installer for packageType=" + test.packageType, - test.expect, installer); - } else { - jqUnit.assertFalse("getInstaller should return nothing for packageType=" + test.packageType, !!installer); - } - }); -}); - // Test requirePackage correctly starts the installer. jqUnit.asyncTest("test requirePackage", function () { var tests = gpii.tests.iod.startInstallerTests; From 4ac05d4908c724e38400410c8f7867cf67eea3bf Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 14 Nov 2019 11:37:11 +0000 Subject: [PATCH 37/56] GPII-2971: added uninstallTime to package data --- .../gpii-iod/src/installOnDemand.js | 22 +++++++++++-------- gpii/node_modules/gpii-iod/src/packages.js | 3 +++ 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 82af5eac0..0b7c5b113 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -105,8 +105,8 @@ fluid.defaults("gpii.iod", { funcName: "gpii.iod.unrequirePackage", args: ["{that}", "{arguments}.0"] }, - uninitialiseInstallation: { - funcName: "gpii.iod.uninitialiseInstallation", + autoRemove: { + funcName: "gpii.iod.autoRemove", args: ["{that}", "{arguments}.0"] }, uninstallPackage: { @@ -130,7 +130,10 @@ fluid.defaults("gpii.iod", { fluid.defaults("gpii.iodLifeCycleManager", { gradeNames: ["fluid.component"], listeners: { - "{lifecycleManager}.events.onSessionStop": "{that}.uninitialiseInstallation" + "{lifecycleManager}.events.onSessionStop": { + func: "{that}.autoRemove", + args: [false] + } }, model: { @@ -182,7 +185,7 @@ gpii.iod.readInstallations = function (that, directory) { promise.then(function () { if (needRemove) { // Loaded some uninstalled installation. - that.uninitialiseInstallation(0); + that.autoRemove(); } }); @@ -386,16 +389,17 @@ gpii.iod.unrequirePackage = function (that, packageName) { * a short time if there is no active session, to avoid giving the computer too much to do while it's in use. * * @param {Component} that The gpii.iod instance. - * @param {Number} wait Number of seconds to wait until uninstalling (default: 30). + * @param {Boolean} immediate Uninstall immediately. */ -gpii.iod.uninitialiseInstallation = function (that, wait) { +gpii.iod.autoRemove = function (that, immediate) { var uninstall = function () { var inSession = false;// that.model.logonChange.inProgress && that.model.logonChange.type !== "login"; if (!inSession) { // Get the first installation var installation = fluid.find(that.installations, function (inst) { - return !inst.required && !inst.removed ? inst : undefined; + return (!inst.required && !inst.removed && inst.packageData.uninstallTime !== "never") + ? inst : undefined; }); if (installation && !installation.uninstalling && installation.gpiiInstalled) { @@ -405,10 +409,10 @@ gpii.iod.uninitialiseInstallation = function (that, wait) { } }; - if (wait === 0) { + if (immediate) { uninstall(); } else { - setTimeout(uninstall, (wait || 30) * 1000); + setTimeout(uninstall, 20000); } }; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 188ce8bb0..58068362e 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -47,6 +47,9 @@ require("./iodSettingsHandler.js"); * @property {String|String[]} installerArgs Additional arguments to pass to the installer. * @property {String|String[]} uninstallerArgs Additional arguments to pass to the installer, when uninstalling. * + * @property {String} uninstallTime When to uninstall this package, after it's no longer required: "immediate", "idle", + * "never". + * * @property {Boolean} elevate true to install as the administrator (installer specific). * @property {String} uiLevel How much is displayed (if possible): "none" (default), "progress" (non-interactive), * "progress-cancel" (progress, can be cancelled), "full" (fully interactive, asks questions). From bc868e1ce24c02c7c6686bb6cd3714de843c5221 Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 15 Nov 2019 12:09:23 +0000 Subject: [PATCH 38/56] GPII-2971: Auto-install, from siteconfig --- .../gpii-iod/src/installOnDemand.js | 43 ++++++++++++++++--- .../gpii-iod/src/packageInstaller.js | 15 ++++++- gpii/node_modules/gpii-iod/src/packages.js | 5 +++ 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 0b7c5b113..12e602cec 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -74,6 +74,10 @@ fluid.defaults("gpii.iod", { "onServerFound.setEndpoint": { funcName: "fluid.set", args: [ "{that}", "endpoint", "{arguments}.0"] + }, + "onServerFound.autoInstall": { + funcName: "gpii.iod.autoInstall", + args: ["{that}", "{that}.options.config.autoInstall"] } }, invokers: { @@ -115,10 +119,14 @@ fluid.defaults("gpii.iod", { } }, - // The IoD server address - endpoint: undefined, - // Map of recognised keys that sign the packages. - allowedKeys: {}, + config: { + // The IoD server address + endpoint: undefined, + // Map of recognised keys that sign the packages. + allowedKeys: {}, + // Packages to install on startup + autoInstall: [] + }, // Map of installer type -> grade name, for each type of installer. installerGrades: {}, @@ -403,8 +411,12 @@ gpii.iod.autoRemove = function (that, immediate) { }); if (installation && !installation.uninstalling && installation.gpiiInstalled) { - installation.uninstalling = true; - that.uninstallPackage(installation).then(uninstall, uninstall); + var autoInstalled = Array.isArray(that.options.config.autoInstall) + && that.options.config.autoInstall.indexOf(installation.packageData.name) > -1; + if (!autoInstalled) { + installation.uninstalling = true; + that.uninstallPackage(installation).then(uninstall, uninstall); + } } } }; @@ -464,7 +476,7 @@ gpii.iod.uninstallPackage = function (that, installation) { delete that.installations[installation.id]; }, function (err) { - fluid.log("IoD: Uninstallation of " + packageName + " failed:", err.error || err); + fluid.log("IoD: Uninstallation of " + packageName + " failed:", (err && err.error) || err); // Remove it from the list so it's uninstalled again, but the file is kept so it tries again upon restart. that.writeInstallation(installation); delete that.installations[installation.id]; @@ -503,3 +515,20 @@ gpii.iod.checkService = function (endpoint) { }); return promise; }; + +/** + * Installs the packages mentioned in the site config. + * + * @param {Component} that The gpii.iod instance. + * @param {Array} packages The array of package names to install. + */ +gpii.iod.autoInstall = function (that, packages) { + var next = function () { + gpii.iod.autoInstall(that, packages.splice(1)); + }; + if (packages && packages.length > 0) { + setTimeout(function () { + that.requirePackage(packages[0]).then(next, next); + }, 1000); + } +}; diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 3a82e7596..6b4633b42 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -120,6 +120,10 @@ fluid.defaults("gpii.iod.packageInstaller", { "onRemovePackage.uninstallPackage": { func: "{that}.uninstallPackage", priority: "after:stopApplication" + }, + "onRemovePackage.cleanup": { + func: "{that}.cleanup", + priority: "after:uninstallPackage" } }, @@ -128,7 +132,9 @@ fluid.defaults("gpii.iod.packageInstaller", { members: { // Package information from the server. - packageData: null + packageData: null, + // "install" or "uninstall" + currentAction: null } }); @@ -155,6 +161,7 @@ gpii.iod.installerCreated = function (that, iod) { * @return {Promise} Resolves when complete. */ gpii.iod.startInstaller = function (that) { + that.currentAction = "install"; return fluid.promise.fireTransformEvent(that.events.onInstallPackage); }; @@ -166,6 +173,7 @@ gpii.iod.startInstaller = function (that) { * @return {Promise} Resolves when complete. */ gpii.iod.startUninstaller = function (that) { + that.currentAction = "uninstall"; return fluid.promise.fireTransformEvent(that.events.onRemovePackage); }; @@ -313,7 +321,10 @@ gpii.iod.prepareInstall = function (that) { */ gpii.iod.cleanup = function (that) { var promise = fluid.promise(); - fluid.log("IoD: Cleaning installation of " + that.packageData.name); + if (that.packageData.keepInstaller || that.currentAction === "uninstall") { + // TODO + fluid.log("IoD: Cleaning installation of " + that.packageData.name); + } promise.resolve(); return promise; }; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 58068362e..51d3b6577 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -44,6 +44,11 @@ require("./iodSettingsHandler.js"); * @property {String} installerHash Installer sha512 hash. * @property {String} installerSource The installer location (where to download/copy it from). * + * @property {Boolean} keepInstaller `true` to keep the installer file after installing (removes after uninstall). + * + * @property {String} installerArgs Additional arguments for the installer. + * @property {String} uninstallerArgs Additional arguments for the uninstaller. + * * @property {String|String[]} installerArgs Additional arguments to pass to the installer. * @property {String|String[]} uninstallerArgs Additional arguments to pass to the installer, when uninstalling. * From 99f4435dd3837d2ab9f9ae182834883357b1e48b Mon Sep 17 00:00:00 2001 From: ste Date: Tue, 19 Nov 2019 10:04:21 +0000 Subject: [PATCH 39/56] GPII-2971: Added some preference sets for IoD --- testData/preferences/iod_italian.json5 | 16 ++++++++++++++++ testData/preferences/iod_putty.json5 | 16 ++++++++++++++++ testData/preferences/iod_zoomtext.json5 | 15 +++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 testData/preferences/iod_italian.json5 create mode 100644 testData/preferences/iod_putty.json5 create mode 100644 testData/preferences/iod_zoomtext.json5 diff --git a/testData/preferences/iod_italian.json5 b/testData/preferences/iod_italian.json5 new file mode 100644 index 000000000..49accddfc --- /dev/null +++ b/testData/preferences/iod_italian.json5 @@ -0,0 +1,16 @@ + +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/net.gpii.test.iod": { + "language.it-it": {}, + }, + "http://registry.gpii.net/common/language": "it-it" + } + } + } + } +} diff --git a/testData/preferences/iod_putty.json5 b/testData/preferences/iod_putty.json5 new file mode 100644 index 000000000..49accddfc --- /dev/null +++ b/testData/preferences/iod_putty.json5 @@ -0,0 +1,16 @@ + +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/net.gpii.test.iod": { + "language.it-it": {}, + }, + "http://registry.gpii.net/common/language": "it-it" + } + } + } + } +} diff --git a/testData/preferences/iod_zoomtext.json5 b/testData/preferences/iod_zoomtext.json5 new file mode 100644 index 000000000..db87664c9 --- /dev/null +++ b/testData/preferences/iod_zoomtext.json5 @@ -0,0 +1,15 @@ + +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/net.gpii.test.iod": { + "zoomtext": {}, + } + } + } + } + } +} From ca83888de5d5246d16b96a78135010c7cb89137d Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 25 Nov 2019 15:51:01 +0000 Subject: [PATCH 40/56] GPII-2971: Added some iod preference sets --- testData/preferences/iod_nvda.json5 | 15 +++++++++++++++ testData/preferences/iod_putty.json5 | 5 ++--- testData/preferences/iod_zoomtext.json5 | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 testData/preferences/iod_nvda.json5 diff --git a/testData/preferences/iod_nvda.json5 b/testData/preferences/iod_nvda.json5 new file mode 100644 index 000000000..9120cf30c --- /dev/null +++ b/testData/preferences/iod_nvda.json5 @@ -0,0 +1,15 @@ + +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/net.gpii.test.iod": { + "nvda": {} + } + } + } + } + } +} diff --git a/testData/preferences/iod_putty.json5 b/testData/preferences/iod_putty.json5 index 49accddfc..1504dd83b 100644 --- a/testData/preferences/iod_putty.json5 +++ b/testData/preferences/iod_putty.json5 @@ -6,9 +6,8 @@ "name": "Default preferences", "preferences": { "http://registry.gpii.net/applications/net.gpii.test.iod": { - "language.it-it": {}, - }, - "http://registry.gpii.net/common/language": "it-it" + "putty": {} + } } } } diff --git a/testData/preferences/iod_zoomtext.json5 b/testData/preferences/iod_zoomtext.json5 index db87664c9..f9109f6e2 100644 --- a/testData/preferences/iod_zoomtext.json5 +++ b/testData/preferences/iod_zoomtext.json5 @@ -6,7 +6,7 @@ "name": "Default preferences", "preferences": { "http://registry.gpii.net/applications/net.gpii.test.iod": { - "zoomtext": {}, + "zoomtext": {} } } } From 7f99ebb73c5af0acba5d248a90dcd60bc29b737f Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 27 Dec 2019 19:25:16 +0000 Subject: [PATCH 41/56] GPII-2971: Not re-installing previously installed packages. --- .../gpii-iod/src/installOnDemand.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 12e602cec..df9bfdab3 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -297,13 +297,17 @@ gpii.iod.requirePackage = function (that, packageRequest) { } else { that.initialiseInstallation(packageData).then(function (installation) { installation.required = true; - installation.installer.startInstaller().then(function () { - installation.installed = true; - installation.gpiiInstalled = true; - // Store the installation info so it can still get removed if gpii restarts. - that.writeInstallation(installation); - promise.resolve(true); - }, promise.reject); + if (installation.installed) { + promise.resolve(false); + } else { + installation.installer.startInstaller().then(function () { + installation.installed = true; + installation.gpiiInstalled = true; + // Store the installation info so it can still get removed if gpii restarts. + that.writeInstallation(installation); + promise.resolve(true); + }, promise.reject); + } // Destroy the installer var destroy = function () { From a315c639d803879bcda4d5f6cad52f08e73623ad Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 27 Dec 2019 19:25:57 +0000 Subject: [PATCH 42/56] GPII-2971: Configurable auto-remove delay. --- gpii/node_modules/gpii-iod/src/installOnDemand.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index df9bfdab3..e406f3a3b 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -130,6 +130,9 @@ fluid.defaults("gpii.iod", { // Map of installer type -> grade name, for each type of installer. installerGrades: {}, + // Milliseconds to wait after key-out (or start up) before uninstalling any un-required packages + autoRemoveDelay: 20000, + members: { installations: {} } @@ -428,7 +431,7 @@ gpii.iod.autoRemove = function (that, immediate) { if (immediate) { uninstall(); } else { - setTimeout(uninstall, 20000); + setTimeout(uninstall, that.options.autoRemoveDelay); } }; From 378ec7d63b7868fdd16c3b2d4e1bb5e1b966a52c Mon Sep 17 00:00:00 2001 From: ste Date: Fri, 27 Dec 2019 19:27:16 +0000 Subject: [PATCH 43/56] GPII-2971: Updated tests to match new code. --- .../gpii-iod/test/installOnDemandTests.js | 41 +++++++++---------- 1 file changed, 20 insertions(+), 21 deletions(-) diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 57f81fd72..7e56d2890 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -122,17 +122,6 @@ fluid.defaults("gpii.tests.iod", { target: "{that packages packageDataSource}.options" } }, - components: { - "testInstaller1": { - type: "gpii.tests.iod.testInstaller1" - }, - "testInstaller2": { - type: "gpii.tests.iod.testInstaller2" - }, - "testInstallerFail": { - type: "gpii.tests.iod.testInstallerFail" - } - }, invokers: { readInstallations: "fluid.identity", writeInstallation: "fluid.identity" @@ -142,6 +131,12 @@ fluid.defaults("gpii.tests.iod", { }, members: { funcCalled: {} + }, + installerGrades: { + "testPackageType1": "gpii.tests.iod.testInstaller1", + "testPackageType2a": "gpii.tests.iod.testInstaller2", + "testPackageType2b": "gpii.tests.iod.testInstaller2", + "testFailPackageType": "gpii.tests.iod.testInstallerFail" } }); @@ -158,20 +153,16 @@ fluid.defaults("gpii.tests.iod.testInstaller1", { funcName: "gpii.tests.iod.testInstaller1.testFunctionCalled", args: ["{that}", "{iod}", "startInstaller"] } - }, - - packageTypes: "testPackageType1" + } }); fluid.defaults("gpii.tests.iod.testInstaller2", { - gradeNames: [ "gpii.tests.iod.testInstaller1"], - packageTypes: ["testPackageType2a", "testPackageType2b"] + gradeNames: [ "gpii.tests.iod.testInstaller1"] }); fluid.defaults("gpii.tests.iod.testInstallerFail", { gradeNames: ["gpii.tests.iod.testInstaller1"], - testReject: "startInstaller", - packageTypes: "testFailPackageType" + testReject: "startInstaller" }); /** @@ -281,7 +272,7 @@ jqUnit.asyncTest("test requirePackage", function () { } var test = tests[testIndex]; - var suffix = " - test:" + test.id; + var suffix = " - test:" + testIndex; iod.funcCalled.startInstaller = null; var p = iod.requirePackage(test.packageRequest); @@ -294,7 +285,10 @@ jqUnit.asyncTest("test requirePackage", function () { jqUnit.assertDeepEq("startInstaller must have been called correctly" + suffix, test.expect, iod.funcCalled.startInstaller); process.nextTick(nextTest); - }, function () { + }, function (reason) { + if (test.expect !== "reject") { + fluid.log("reject reason: ", reason); + } jqUnit.assertEquals("packageData must only reject if expected" + suffix, test.expect, "reject"); jqUnit.assert("balance the assert count"); process.nextTick(nextTest); @@ -478,9 +472,14 @@ jqUnit.asyncTest("test uninstallation after restart", function () { writeInstallation: { funcName: "gpii.iod.writeInstallation", args: ["{that}", dir, "{arguments}.0"] + }, + uninstallPackage: { + funcName: "gpii.tests.iod.testInstaller1.testFunctionCalled", + args: ["{that}", "{iod}", "uninstallPackage"] } }, - testReject: "uninstallPackage" + testReject: "uninstallPackage", + autoRemoveDelay: 0 }; var iod = gpii.tests.iod(iodOptions); From 6640bb2c3d91323d9f2c59ab5fb9afeb8feb3bbc Mon Sep 17 00:00:00 2001 From: ste Date: Sat, 28 Dec 2019 21:08:33 +0000 Subject: [PATCH 44/56] GPII-2971: Installers now using the same command execution method --- .../gpii-iod/src/packageInstaller.js | 74 +++++++++++++++++++ gpii/node_modules/gpii-iod/src/packages.js | 22 ++++-- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 6b4633b42..b0bfe8aa4 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -44,6 +44,11 @@ fluid.defaults("gpii.iod.packageInstaller", { funcName: "gpii.iod.startUninstaller", args: ["{that}", "{iod}"] }, + executeCommand: { + funcName: "gpii.iod.executeCommand", + // PackageInvocation, command, args + args: ["{that}", "{iod}", "{arguments}.0", "{arguments}.1", "{arguments}.2"] + }, // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns // an installation, either directly or via a promise. initialise: { @@ -372,3 +377,72 @@ gpii.iod.stopApplication = function (that) { } return promise; }; + +/** + * Executes a command. + * @param {Component} that The gpii.iod.installer instance. + * @param {Component} iod The gpii.iod instance. + * @param {PackageInvocation} invocation How the command is invoked. + * @param {String} command The command. + * @param {Array|String} args [optional] The arguments (overrides `invocation.args`). + * @return {Promise} Resolves when complete. + */ +gpii.iod.executeCommand = function (that, iod, invocation, command, args) { + + if (typeof(invocation) === "string") { + // Can be expressed as a string, where it only specifies the arguments. + invocation = { args: invocation }; + } else { + // Take a copy to modify. + invocation = Object.assign({}, invocation); + } + + if (args) { + invocation.args = fluid.makeArray(args); + } + + var promise; + if (invocation.elevate && that.invokeElevated) { + promise = that.invokeElevated(invocation, command, args); + } else { + if (invocation.elevate) { + fluid.log(fluid.logLevel.WARN, "Running elevated commands is not supported on this operating system."); + } + + promise = fluid.promise(); + + var child = child_process.spawn(command, fluid.makeArray(invocation.args), { + stdio: "inherit" + }); + + child.on("error", function (err) { + if (!promise.disposition) { + promise.reject({ + isError: true, + error: err, + message: "Error running command", + command: command, + invocation: invocation + }); + } + }); + child.on("exit", function (code) { + if (code) { + if (!promise.disposition) { + promise.reject({ + isError: true, + exitCode: code, + message: "Error running command", + command: command, + invocation: invocation + }); + } + } else { + promise.resolve(); + } + }); + + } + + return promise; +}; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 51d3b6577..c782047a4 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -46,21 +46,31 @@ require("./iodSettingsHandler.js"); * * @property {Boolean} keepInstaller `true` to keep the installer file after installing (removes after uninstall). * - * @property {String} installerArgs Additional arguments for the installer. - * @property {String} uninstallerArgs Additional arguments for the uninstaller. - * - * @property {String|String[]} installerArgs Additional arguments to pass to the installer. - * @property {String|String[]} uninstallerArgs Additional arguments to pass to the installer, when uninstalling. + * @property {PackageInvocation|String} installerArgs Additional options used when executing the installer. + * @property {PackageInvocation|String} uninstallerArgs Additional options used when executing the uninstaller. * * @property {String} uninstallTime When to uninstall this package, after it's no longer required: "immediate", "idle", * "never". * - * @property {Boolean} elevate true to install as the administrator (installer specific). * @property {String} uiLevel How much is displayed (if possible): "none" (default), "progress" (non-interactive), * "progress-cancel" (progress, can be cancelled), "full" (fully interactive, asks questions). * */ +/** + * Describes how something is invoked. + * @typedef {Object} PackageInvocation + * @property {String|Array} args arguments passed to the command. + * @property {Boolean} elevate true to run as administrator. + * @property {Boolean} desktop true to run in the context of the desktop, if elevate is true. + */ + +/** + * A command + * @typedef {PackageInvocation} PackageCommand + * @property {String} command The command to invoke. + */ + gpii.iod.resolvers = {}; fluid.defaults("gpii.iod.packages", { From 33d18bb2d6e27ab9ee1b61b38be38e532d48d65c Mon Sep 17 00:00:00 2001 From: ste Date: Sat, 28 Dec 2019 21:10:06 +0000 Subject: [PATCH 45/56] GPII-2971: Improved IoD tests, increased coverage --- .nycrc | 7 +- .../gpii-iod/src/installOnDemand.js | 8 +- gpii/node_modules/gpii-iod/test/all-tests.js | 1 + .../gpii-iod/test/installOnDemandTests.js | 64 +++++- .../{testPackages => packageData}/env.json5 | 0 .../failInstall.json5 | 0 .../languages.json5 | 0 .../package1.json5 | 0 .../package2.json5 | 0 .../unknownType.json5 | 0 .../gpii-iod/test/packageDataSourceTests.js | 183 ++++++++++++++++++ .../gpii-iod/test/packageInstallerTests.js | 2 +- .../gpii-iod/test/packagesTests.js | 100 +--------- package.json | 1 + 14 files changed, 263 insertions(+), 103 deletions(-) rename gpii/node_modules/gpii-iod/test/{testPackages => packageData}/env.json5 (100%) rename gpii/node_modules/gpii-iod/test/{testPackages => packageData}/failInstall.json5 (100%) rename gpii/node_modules/gpii-iod/test/{testPackages => packageData}/languages.json5 (100%) rename gpii/node_modules/gpii-iod/test/{testPackages => packageData}/package1.json5 (100%) rename gpii/node_modules/gpii-iod/test/{testPackages => packageData}/package2.json5 (100%) rename gpii/node_modules/gpii-iod/test/{testPackages => packageData}/unknownType.json5 (100%) create mode 100644 gpii/node_modules/gpii-iod/test/packageDataSourceTests.js diff --git a/.nycrc b/.nycrc index 640be7eb6..76b43f1b9 100644 --- a/.nycrc +++ b/.nycrc @@ -116,6 +116,11 @@ "!**/gpii/node_modules/userListeners/src/listeners.js", "!**/gpii/node_modules/userListeners/src/pcsc.js", "!**/gpii/node_modules/userListeners/src/usb.js", + "!**/gpii/node_modules/gpii-iod/src/installOnDemand.js", + "!**/gpii/node_modules/gpii-iod/src/iodSettingsHandler.js", + "!**/gpii/node_modules/gpii-iod/src/packageDataSource.js", + "!**/gpii/node_modules/gpii-iod/src/packageInstaller.js", + "!**/gpii/node_modules/gpii-iod/src/packages.js", "testData", "tests", "reports", @@ -128,7 +133,7 @@ "gpii.js", "Gruntfile.js" ], - "reporter": "none", + "reporter": "lcov", "report-dir": "reports", "temp-directory": "coverage", "clean": false diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index e406f3a3b..73ac1028a 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -413,8 +413,10 @@ gpii.iod.autoRemove = function (that, immediate) { if (!inSession) { // Get the first installation var installation = fluid.find(that.installations, function (inst) { - return (!inst.required && !inst.removed && inst.packageData.uninstallTime !== "never") - ? inst : undefined; + return (!inst.required && !inst.removed && + (!inst.packageData || inst.packageData.uninstallTime !== "never")) + ? inst + : undefined; }); if (installation && !installation.uninstalling && installation.gpiiInstalled) { @@ -439,7 +441,7 @@ gpii.iod.autoRemove = function (that, immediate) { * Uninstall a package. * * @param {Component} that The gpii.iod instance. - * @param {Installation|String} installation The installation state, or installation ID. + * @param {Installation|String} installation The installation state, or package name. * @return {Promise} Resolves when the package is removed. */ gpii.iod.uninstallPackage = function (that, installation) { diff --git a/gpii/node_modules/gpii-iod/test/all-tests.js b/gpii/node_modules/gpii-iod/test/all-tests.js index fc55cebe9..2c40e6c08 100644 --- a/gpii/node_modules/gpii-iod/test/all-tests.js +++ b/gpii/node_modules/gpii-iod/test/all-tests.js @@ -3,3 +3,4 @@ require("./installOnDemandTests.js"); require("./packageInstallerTests.js"); require("./packagesTests.js"); +require("./packageDataSourceTests.js"); diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 7e56d2890..0abf0c2bd 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -114,7 +114,7 @@ fluid.defaults("gpii.tests.iod", { packageDataSource: { record: { gradeNames: ["kettle.dataSource.file.moduleTerms"], - path: __dirname + "/testPackages/%packageName.json5", + path: __dirname + "/packageData/%packageName.json5", termMap: { "packageName": "%packageName" } @@ -530,3 +530,65 @@ jqUnit.asyncTest("test uninstallation after restart", function () { waitForUninstall().then(jqUnit.start, jqUnit.fail); }, jqUnit.fail); }); + +jqUnit.asyncTest("test service discovery", function () { + + jqUnit.expect(4); + + var server = require("http").createServer(); + server.listen(0, "127.0.0.1"); + + server.on("request", function (req, res) { + fluid.log("request: ", req.url); + jqUnit.assert("request made"); + res.end("hello"); + }); + + server.on("listening", function () { + var localUrl = "http://" + server.address().address + ":" + server.address().port + "/"; + fluid.log("listening: ", localUrl); + + var timeout = setTimeout(function () { + jqUnit.fail("Timeout waiting for endpoint request/reply"); + }, 5000); + + var iodOptions = { + listeners: { + onServerFound: function () { + clearTimeout(timeout); + jqUnit.start(); + } + }, + config: { + endpoint: localUrl + } + }; + + // try checkService directly + var successPromise = gpii.iod.checkService(localUrl).then(function () { + jqUnit.assert("checkService should resolve"); + }, function (err) { + fluid.log(err); + jqUnit.fail("checkService should not reject"); + }); + + var failPromise = fluid.promise(); + // Check a local port (Gopher) which is probably closed. + gpii.iod.checkService("http://127.0.0.3:70").then(function () { + jqUnit.fail("checkService (fail test) should not resolve"); + }, function () { + jqUnit.assert("checkService (fail test) should reject"); + failPromise.resolve(); + }); + + fluid.promise.sequence([ + failPromise, + successPromise, + function () { + // Check that discoverServer fires the event + var iod = gpii.tests.iod(iodOptions); + iod.discoverServer(); + } + ]); + }); +}); diff --git a/gpii/node_modules/gpii-iod/test/testPackages/env.json5 b/gpii/node_modules/gpii-iod/test/packageData/env.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/env.json5 rename to gpii/node_modules/gpii-iod/test/packageData/env.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/failInstall.json5 b/gpii/node_modules/gpii-iod/test/packageData/failInstall.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/failInstall.json5 rename to gpii/node_modules/gpii-iod/test/packageData/failInstall.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/languages.json5 b/gpii/node_modules/gpii-iod/test/packageData/languages.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/languages.json5 rename to gpii/node_modules/gpii-iod/test/packageData/languages.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/package1.json5 b/gpii/node_modules/gpii-iod/test/packageData/package1.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/package1.json5 rename to gpii/node_modules/gpii-iod/test/packageData/package1.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/package2.json5 b/gpii/node_modules/gpii-iod/test/packageData/package2.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/package2.json5 rename to gpii/node_modules/gpii-iod/test/packageData/package2.json5 diff --git a/gpii/node_modules/gpii-iod/test/testPackages/unknownType.json5 b/gpii/node_modules/gpii-iod/test/packageData/unknownType.json5 similarity index 100% rename from gpii/node_modules/gpii-iod/test/testPackages/unknownType.json5 rename to gpii/node_modules/gpii-iod/test/packageData/unknownType.json5 diff --git a/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js b/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js new file mode 100644 index 000000000..781e8ae43 --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js @@ -0,0 +1,183 @@ +/* + * IoD Tests - package data source. + * + * Copyright 2019 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); +var kettle = fluid.require("kettle"); +kettle.loadTestingSupport(); + +var crypto = require("crypto"); + +var jqUnit = fluid.require("node-jqunit"); +var gpii = fluid.registerNamespace("gpii"); + +fluid.registerNamespace("gpii.tests.iodPackageData"); + +require("../index.js"); + +require("gpii-iodServer"); + +gpii.tests.iodPackageData.verifySignedJSONTests = [ + { + id: "correctly signed", + input: { + data: { + testField: "testValue", + publicKey: "$key" + }, + signature: "$signature", + allowedKeys: "$fingerprint" + }, + expect: "resolve" + } +]; + + + +gpii.tests.iodPackageData.generateKey = function (passphrase) { + var keyPair = crypto.generateKeyPairSync("rsa", { + modulusLength: 4096, + publicKeyEncoding: { + type: "spki", + format: "pem" + }, + privateKeyEncoding: { + type: "pkcs1", + format: "pem", + cipher: "aes-128-cbc", + passphrase: passphrase || "test" + } + }); + + // Include the passphrase + keyPair.passphrase = passphrase; + // signPackageData expects the private key to be `key`. + keyPair.key = keyPair.privateKey; + delete keyPair.privateKey; + + // Get the key (without the PEM header+trailer) + var keyBinary = gpii.iodServer.packageFile.readPEM(keyPair.publicKey); + // Generate the finger print + keyPair.fingerprint = crypto.createHash("sha256").update(keyBinary).digest("base64"); + + return keyPair; +}; + + + +gpii.tests.iodPackageData.createPackage = function (packageData, key, packageFile) { + return gpii.iodServer.packageFile.create(packageData, null, key, packageFile); +}; + + +gpii.tests.iodPackageData.assertReject = function (msg, promise) { + var promiseTogo = fluid.promise(); + promise.then(function () { + jqUnit.assertEquals(msg, "reject", "resolve"); + promiseTogo.resolve(); + }, function (reason) { + fluid.log("rejection result: ", reason); + jqUnit.assert("promise rejected"); + promiseTogo.resolve(); + }); + return promiseTogo; +}; + +jqUnit.asyncTest("test verifySignedJSON", function () { + + var pair = gpii.tests.iodPackageData.generateKey("test pass"); + + var data = { + testField: "test value", + publicKey: pair.publicKey + }; + + var signedData = gpii.iodServer.packageFile.signPackageData(data, pair); + var signedString = signedData.buffer.toString("utf8"); + + // The signature is well formed, but for the wrong thing + var wrongSignature = crypto.createSign("RSA-SHA512").update("something else").sign(pair); + // The signature isn't a signature + var badSignature = Buffer.from("bad signature"); + + + var promises = [ + gpii.iod.verifySignedJSON(signedString, signedData.signature, {key1: pair.fingerprint}).then(function (result) { + var expect = JSON.parse(signedString); + jqUnit.assertDeepEq("verifySignedJSON with valid data should resolve with the expected result", + expect, result); + }, function (e) { + fluid.log("reject reason: ", e); + jqUnit.fail("verifySignedJSON with valid data should resolve"); + }), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with invalid signedString should reject", + gpii.iod.verifySignedJSON(signedString.replace("test", "TEST"), signedData.signature, {key1: pair.fingerprint})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with the wrong signature should reject", + gpii.iod.verifySignedJSON(signedString, wrongSignature, {key1: pair.fingerprint})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with a bad signature should reject", + gpii.iod.verifySignedJSON(signedString, badSignature, {key1: pair.fingerprint})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with an empty signature should reject", + gpii.iod.verifySignedJSON(signedString, Buffer.from([]), {key1: pair.fingerprint})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with unknown fingerprint should reject", + gpii.iod.verifySignedJSON(signedString, signedData.signature, {key1: "wrong fingerprint"})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with no known fingerprints should reject", + gpii.iod.verifySignedJSON(signedString, signedData.signature, {})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with no public key in object should reject", + gpii.iod.verifySignedJSON("{\"testField\":123}", signedData.signature, {key1: pair.fingerprint})), + + gpii.tests.iodPackageData.assertReject("verifySignedJSON with invalid json should reject", + gpii.iod.verifySignedJSON("} invalid json:", signedData.signature, {key1: pair.fingerprint})), + + function () { + // test checkPackageSignature with valid data + var packageResponse = { + packageData: signedString, + packageDataSignature: signedData.signature + }; + // The packageData should return as an object. + var expect = { + packageData: JSON.parse(signedString), + packageDataSignature: signedData.signature + }; + + return gpii.iod.checkPackageSignature(packageResponse, {key1: pair.fingerprint}).then(function (result) { + jqUnit.assertDeepEq("checkPackageSignature with valid data should resolve with the expected result", + expect, result); + }, function (reason) { + jqUnit.assertEquals("checkPackageSignature with valid data should have resolved", "resolve", reason); + }); + }, + + gpii.tests.iodPackageData.assertReject("checkPackageSignature with invalid data should reject", + gpii.iod.checkPackageSignature( + {packageData: signedString, packageDataSignature: "wrong"}, {key1: pair.fingerprint})) + + ]; + + fluid.promise.sequence(promises).then(jqUnit.start, jqUnit.fail); + +}); + diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index ccf56d2c3..8bda9904f 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -56,7 +56,7 @@ fluid.defaults("gpii.tests.iodInstaller", { "packageDataSource": { type: "kettle.dataSource.file", options: { - path: __dirname + "/testPackages/%packageName.json5" + path: __dirname + "/packageData/%packageName.json5" } } } diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index 1e05b1019..dfb6e1006 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -23,7 +23,6 @@ var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); var JSON5 = require("json5"), - crypto = require("crypto"), fs = require("fs"), path = require("path"), os = require("os"); @@ -69,7 +68,7 @@ gpii.tests.iodPackages.getPackageDataTests = fluid.freezeRecursive([ request: { packageName: "package1" }, - expect: JSON5.parse(fs.readFileSync(__dirname + "/testPackages/package1.json5", "utf8")) + expect: JSON5.parse(fs.readFileSync(__dirname + "/packageData/package1.json5", "utf8")) }, { id: "Single language package, with language specified", @@ -77,7 +76,7 @@ gpii.tests.iodPackages.getPackageDataTests = fluid.freezeRecursive([ packageName: "package1", language: "fr-FR" }, - expect: JSON5.parse(fs.readFileSync(__dirname + "/testPackages/package1.json5", "utf8")) + expect: JSON5.parse(fs.readFileSync(__dirname + "/packageData/package1.json5", "utf8")) }, { id: "Multi-language package, language not specified", @@ -502,7 +501,7 @@ fluid.defaults("gpii.tests.iodPackages", { packageDataSource: { record: { gradeNames: ["kettle.dataSource.file.moduleTerms"], - path: __dirname + "/testPackages/%packageName.json5", + path: __dirname + "/packageData/%packageName.json5", termMap: { "packageName": "%packageName" } @@ -634,96 +633,3 @@ jqUnit.test("test checkInstalled", function () { delete process.env[testEnv]; }); - -jqUnit.asyncTest("testing package data signature verification", function () { - // Create a key pair - var passphrase = "test"; - var keyPair = crypto.generateKeyPairSync("rsa", { - modulusLength: 4096, - publicKeyEncoding: { - type: "spki", - format: "pem" - }, - privateKeyEncoding: { - type: "pkcs1", - format: "pem", - cipher: "aes-128-cbc", - passphrase: passphrase - } - }); - - // The key is already base64 encoded - just remove the PEM header and footer. - var re = new RegExp(".*(:?\n|^)-----BEGIN[^\n]*\n(.*\n)?-----END.*", "s"); - var publicKey = re.exec(keyPair.publicKey)[2]; - - // Get the key fingerprint - var fingerprint = crypto.createHash("sha256").update(Buffer.from(publicKey, "base64")).digest("base64"); - - // The object to be signed - var obj = { - a: "this object will be signed", - b: { - c: "extra", - d: Math.random() - }, - publicKey: publicKey - }; - - var json = JSON.stringify(obj); - var buffer = Buffer.from(json, "utf8"); - - // Sign it - var sign = crypto.createSign("RSA-SHA512"); - sign.update(buffer); - var signature = sign.sign({ - key: keyPair.privateKey, - passphrase: passphrase - }).toString("base64"); - - jqUnit.expect(4); - var tests = [ - function () { - // Verify it. - return gpii.iod.verifySignedJSON(json, signature, [fingerprint]).then(function () { - jqUnit.assert("verifySignedJSON should resolve with correctly signed data"); - }); - }, - function () { - // Verify it, without a valid fingerprint. - return gpii.iod.verifySignedJSON(json, signature, ["xxx"]).then(function () { - jqUnit.fail("verifySignedJSON should have rejected with unknown fingerprint"); - }, - function () { - jqUnit.assert("verifySignedJSON should reject with unknown fingerprint"); - }); - }, - function () { - // Run it through checkPackageSignature - /** @type PackageResponse */ - var packageResponse = { - installer: true, - packageData: json, - packageDataSignature: signature - }; - return gpii.iod.checkPackageSignature(packageResponse, [fingerprint]).then(function (value) { - jqUnit.assertDeepEq("packageData resolved by checkPackageSignature should be de-serialised", - obj, value.packageData); - }); - }, - function () { - // Make a change to the signed data - var modified = json.replace(/this object/, "that object"); - var promise = fluid.promise(); - gpii.iod.verifySignedJSON(modified, signature, [fingerprint]).then(function () { - jqUnit.fail("verifySignedJSON should have rejected with modified data"); - promise.reject(); - }, function () { - jqUnit.assert("verifySignedJSON should reject with modified data"); - }); - return promise; - } - ]; - - fluid.promise.sequence(tests).then(jqUnit.start, jqUnit.fail); - -}); diff --git a/package.json b/package.json index 01865c706..627222ec5 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "browserify": "16.2.3", "gpii-express": "1.0.15", "gpii-grunt-lint-all": "1.0.5", + "gpii-iodServer": "stegru/gpii-iod#GPII-2972", "gpii-testem": "2.1.7", "grunt": "1.0.3", "grunt-markdownlint": "2.1.0", From a7b33cca0add6b0a074866f6af3fb33322e7a96c Mon Sep 17 00:00:00 2001 From: ste Date: Sun, 29 Dec 2019 16:36:10 +0000 Subject: [PATCH 46/56] GPII-2971: IoD tests pass --- .../gpii-iod/test/packageInstallerTests.js | 31 ++++++++----------- .../gpii-iod/test/packagesTests.js | 4 +-- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index 8bda9904f..c48f40168 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -74,14 +74,16 @@ fluid.defaults("gpii.tests.iodInstaller.installer", { invokers: { initialise: "gpii.tests.iodInstaller.stage({that}, initialise)", - downloadPackage: "gpii.tests.iodInstaller.stage({that}, downloadPackage)", + downloadInstaller: "gpii.tests.iodInstaller.stage({that}, downloadInstaller)", checkPackage: "gpii.tests.iodInstaller.stage({that}, checkPackage)", prepareInstall: "gpii.tests.iodInstaller.stage({that}, prepareInstall)", installPackage: "gpii.tests.iodInstaller.stage({that}, installPackage)", cleanup: "gpii.tests.iodInstaller.stage({that}, cleanup)", startApplication: "gpii.tests.iodInstaller.stage({that}, startApplication)", uninstallPackage: "gpii.tests.iodInstaller.stage({that}, uninstallPackage)", - stopApplication: "gpii.tests.iodInstaller.stage({that}, stopApplication)" + stopApplication: "gpii.tests.iodInstaller.stage({that}, stopApplication)", + installComplete: "gpii.tests.iodInstaller.stage({that}, installComplete)", + uninstallComplete: "gpii.tests.iodInstaller.stage({that}, uninstallComplete)" }, packageTypes: "testPackageType1" @@ -103,11 +105,12 @@ jqUnit.asyncTest("test installation pipe-line", function () { installer.startInstaller({}).then(function () { var expect = [ "initialise", - "downloadPackage", + "downloadInstaller", "checkPackage", "prepareInstall", "installPackage", "cleanup", + "installComplete", "startApplication" ]; @@ -117,7 +120,9 @@ jqUnit.asyncTest("test installation pipe-line", function () { installer.startUninstaller().then(function () { var expect = [ "stopApplication", - "uninstallPackage" + "uninstallPackage", + "cleanup", + "uninstallComplete" ]; jqUnit.assertDeepEq("All stages of the uninstallation should be called in order.", expect, installer.stages); @@ -183,16 +188,6 @@ jqUnit.asyncTest("test https download", function () { url: "https://null.badssl.com/", expect: "reject" }, - // HTTP - { - // This redirects to http - url: "https://http.badssl.com/", - expect: "reject" - }, - { - url: "http://http.badssl.com/", - expect: "reject" - }, { // Unopened port (hopefully) url: "https://127.0.0.1:51749", @@ -231,12 +226,12 @@ jqUnit.asyncTest("test https download", function () { var outFile = filePrefix + testIndex; files.push(outFile); - var p = gpii.iod.httpsDownload(test.url, outFile); + var p = gpii.iod.fileDownload(test.url, outFile); - jqUnit.assertTrue("httpsDownload must return a promise" + suffix, fluid.isPromise(p)); + jqUnit.assertTrue("fileDownload must return a promise" + suffix, fluid.isPromise(p)); p.then(function () { - jqUnit.assertNotEquals("httpsDownload must only succeed if expected" + suffix, test.expect, "reject"); + jqUnit.assertNotEquals("fileDownload must only succeed if expected" + suffix, test.expect, "reject"); if (test.expect === "resolve") { jqUnit.assert("resolved"); @@ -261,7 +256,7 @@ jqUnit.asyncTest("test https download", function () { }); } }, function (err) { - jqUnit.assertEquals("httpsDownload must only reject if expected" + suffix, test.expect, "reject"); + jqUnit.assertEquals("fileDownload must only reject if expected" + suffix, test.expect, "reject"); jqUnit.assert("Balancing the expected assert count"); if (test.expects !== "reject") { fluid.log(err); diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index dfb6e1006..4541f24d4 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -197,8 +197,8 @@ gpii.tests.iodPackages.resolvePackageTests = fluid.freezeRecursive([ // Resolver { name: "environment", - result: "${{environment}.PATH", - expect: process.PATH + result: "${{environment}.PATH}", + expect: process.env.PATH }, { name: "exists", From 2eaa794c4d34fc114d61a2e7c53701e5ef0d6842 Mon Sep 17 00:00:00 2001 From: ste Date: Sun, 29 Dec 2019 20:30:28 +0000 Subject: [PATCH 47/56] GPII-2971: Added installCommands to packageData. --- .../gpii-iod/src/packageInstaller.js | 298 ++++++++++++++---- gpii/node_modules/gpii-iod/src/packages.js | 10 + .../gpii-iod/test/packageInstallerTests.js | 155 +++++++-- 3 files changed, 373 insertions(+), 90 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index b0bfe8aa4..c2aa585c4 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -53,34 +53,42 @@ fluid.defaults("gpii.iod.packageInstaller", { // an installation, either directly or via a promise. initialise: { funcName: "gpii.iod.initialise", - args: ["{that}", "{iod}"] + args: ["{that}", "{iod}", "{that}.installation", "{that}.installation.packageData"] }, downloadInstaller: { funcName: "gpii.iod.downloadInstaller", - args: ["{that}", "{iod}"] + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] }, checkPackage: { funcName: "gpii.iod.checkPackage", - args: ["{that}", "{iod}"] + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] }, prepareInstall: { funcName: "gpii.iod.prepareInstall", - args: ["{that}", "{iod}"] + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] }, installPackage: "fluid.notImplemented", cleanup: { funcName: "gpii.iod.cleanup", - args: ["{that}", "{iod}"] + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] + }, + installComplete: { + funcName: "gpii.iod.installComplete", + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] }, startApplication: { funcName: "gpii.iod.startApplication", - args: ["{that}", "{iod}"] + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] }, stopApplication: { funcName: "gpii.iod.stopApplication", - args: ["{that}", "{iod}"] + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] }, - uninstallPackage: "fluid.notImplemented" + uninstallPackage: "fluid.notImplemented", + uninstallComplete: { + funcName: "gpii.iod.installComplete", + args: ["{that}", "{that}.installation", "{that}.installation.packageData"] + } }, events: { // Dummy events for the installation pipe-lines @@ -113,9 +121,13 @@ fluid.defaults("gpii.iod.packageInstaller", { func: "{that}.cleanup", priority: "after:install" }, + "onInstallPackage.complete": { + func: "{that}.installComplete", + priority: "after:cleanup" + }, "onInstallPackage.startApplication": { func: "{that}.startApplication", - priority: "after:cleanup" + priority: "last" }, "onRemovePackage.stopApplication": { @@ -129,6 +141,10 @@ fluid.defaults("gpii.iod.packageInstaller", { "onRemovePackage.cleanup": { func: "{that}.cleanup", priority: "after:uninstallPackage" + }, + "onRemovePackage.uninstallComplete": { + func: "{that}.uninstallComplete", + priority: "after:cleanup" } }, @@ -155,6 +171,12 @@ gpii.iod.installerCreated = function (that, iod) { if (that.installation) { that.installation.installer = that; that.packageData = that.installation.packageData; + if (!that.packageData.installCommands) { + that.packageData.installCommands = {}; + } + if (!that.packageData.uninstallCommands) { + that.packageData.uninstallCommands = {}; + } } }; @@ -167,6 +189,7 @@ gpii.iod.installerCreated = function (that, iod) { */ gpii.iod.startInstaller = function (that) { that.currentAction = "install"; + gpii.iod.addStageListeners(that, that.events.onInstallPackage); return fluid.promise.fireTransformEvent(that.events.onInstallPackage); }; @@ -179,59 +202,108 @@ gpii.iod.startInstaller = function (that) { */ gpii.iod.startUninstaller = function (that) { that.currentAction = "uninstall"; + gpii.iod.addStageListeners(that, that.events.onRemovePackage); return fluid.promise.fireTransformEvent(that.events.onRemovePackage); }; +/** + * Adds listeners before and after the existing listeners of an event (onInstallPackage or onRemovePackage), which + * update+log the current stage and possibly execute a command specified in the packageData.installCommands for that + * stage. + * + * @param {Component} that The gpii.iod.installer instance. + * @param {Event} event The event. + */ +gpii.iod.addStageListeners = function (that, event) { + // inject some listeners to record the current stage + fluid.each(Object.keys(event.listeners), function (namespace) { + event.addListener(function () { + that.installation.currentStage = namespace; + fluid.log("IoD: Entering stage: ", namespace); + return gpii.iod.customCommand(that, "before"); + }, "_before_" + namespace, "before:" + namespace); + + event.addListener(function () { + fluid.log("IoD: Leaving stage: ", namespace); + return gpii.iod.customCommand(that, "after"); + }, "_after_" + namespace, "after:" + namespace); + + }); +}; + +gpii.iod.customCommand = function (that, when) { + var commands = that.currentAction === "install" + ? that.packageData.installCommands + : that.packageData.uninstallCommands; + var command = commands && commands[that.installation.currentStage + ":" + when]; + var togo; + if (command) { + togo = that.executeCommand(command); + } + return togo; +}; + /** * Initialises the installation. * * @param {Component} that The gpii.iod.installer instance. * @param {Component} iod The gpii.iod instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. + * @return {Promise} Resolves when complete. */ -gpii.iod.initialise = function (that, iod) { - var tempDir = iod.getWorkingPath(that.packageData.name); - that.installation.tempDir = tempDir.fullPath; - that.installation.cleanupPaths.push(tempDir.createdPath); +gpii.iod.initialise = function (that, iod, installation, packageData) { + var tempDir = iod.getWorkingPath(packageData.name); + installation.tempDir = tempDir.fullPath; + installation.cleanupPaths.push(tempDir.createdPath); + return packageData.installCommands.initialise + ? that.executeCommand(packageData.installCommands.initialise) + : fluid.promise().resolve(); }; /** * Downloads an installer from the server. * * @param {Component} that The gpii.iod.installer instance. - * @param {Object} iod The gpii.iod instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves when complete. */ -gpii.iod.downloadInstaller = function (that) { +gpii.iod.downloadInstaller = function (that, installation, packageData) { - fluid.log("IoD: Downloading installer " + that.packageData.installerSource); + fluid.log("IoD: Downloading installer " + packageData.installerSource); var promise = fluid.promise(); + if (packageData.installCommands.download) { + promise = that.executeCommand(packageData.installCommands.initialise); + } else { + promise = fluid.promise(); - that.installation.installerFile = path.join(that.installation.tempDir, that.packageData.installer); + installation.installerFile = path.join(installation.tempDir, packageData.installer); - if (that.packageData.installerSource) { - if (/^https?:\/\//.test(that.packageData.installerSource)) { - // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). - var downloadPromise = gpii.iod.fileDownload(that.packageData.installerSource, that.installation.installerFile); - fluid.promise.follow(downloadPromise, promise); + if (packageData.installerSource) { + if (/^https?:\/\//.test(packageData.installerSource)) { + // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). + var downloadPromise = gpii.iod.fileDownload(packageData.installerSource, installation.installerFile); + fluid.promise.follow(downloadPromise, promise); + } else { + fs.copyFile(packageData.url, installation.installerFile, function (err) { + if (err) { + promise.reject({ + isError: true, + message: "Unable to copy package" + }); + } else { + promise.resolve(); + } + }); + } } else { - fs.copyFile(that.packageData.url, that.installation.installerFile, function (err) { - if (err) { - promise.reject({ - isError: true, - message: "Unable to copy package" - }); - } else { - promise.resolve(); - } - }); + promise.resolve(); } - } else { - promise.resolve(); } - return promise.then(null, function (err) { - fluid.log("IoD: Failed download of " + that.packageData.installerSource + ": ", err); + fluid.log("IoD: Failed download of " + packageData.installerSource + ": ", err); }); }; @@ -292,15 +364,20 @@ gpii.iod.fileDownload = function (url, localPath, options) { * Checks that a downloaded package is ok. * * @param {Component} that The gpii.iod.installer instance. - * @param {Object} iod The gpii.iod instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves when complete. */ -gpii.iod.checkPackage = function (that) { - var promise = fluid.promise(); - fluid.log("IoD: Checking downloaded package file " + that.packageData.filename); - // TODO: It shouldn't be checked here - another process may over-write it before the high privilege executes it. - // Instead, take ownership then check the integrity in the same context as it's being ran. - promise.resolve(); +gpii.iod.checkPackage = function (that, installation, packageData) { + var promise; + fluid.log("IoD: Checking downloaded package file " + packageData.filename); + if (packageData.installCommands.check) { + promise = that.executeCommand(packageData.installCommands.check); + } else { + // TODO: It shouldn't be checked here - another process may over-write it before the high privilege executes it. + // Instead, take ownership then check the integrity in the same context as it's being ran. + promise = fluid.promise().resolve(); + } return promise; }; @@ -308,13 +385,18 @@ gpii.iod.checkPackage = function (that) { * Generate the installation instructions. * * @param {Component} that The gpii.iod.installer instance. - * @param {Object} iod The gpii.iod instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves when complete. */ -gpii.iod.prepareInstall = function (that) { - var promise = fluid.promise(); - fluid.log("IoD: Preparing installation for " + that.packageData.name); - promise.resolve(); +gpii.iod.prepareInstall = function (that, installation, packageData) { + var promise; + fluid.log("IoD: Preparing installation for " + packageData.name); + if (packageData.installCommands.prepareInstall) { + promise = that.executeCommand(packageData.installCommands.prepareInstall); + } else { + promise = fluid.promise().resolve(); + } return promise; }; @@ -322,15 +404,43 @@ gpii.iod.prepareInstall = function (that) { * Cleans up things that are no longer required. * * @param {Component} that The gpii.iod.installer instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves when complete. */ -gpii.iod.cleanup = function (that) { - var promise = fluid.promise(); - if (that.packageData.keepInstaller || that.currentAction === "uninstall") { - // TODO - fluid.log("IoD: Cleaning installation of " + that.packageData.name); +gpii.iod.cleanup = function (that, installation, packageData) { + var promise; + if (that.currentAction === "install") { + promise = packageData.installCommands.cleanup && that.executeCommand(packageData.installCommands.cleanup); + } else { + promise = packageData.uninstallCommands.cleanup && that.executeCommand(packageData.uninstallCommands.cleanup); + } + + if (!promise) { + if (!packageData.keepInstaller || that.currentAction === "uninstall") { + // TODO + fluid.log("IoD: Cleaning installation of " + packageData.name); + } + promise = fluid.promise().resolve(); + } + return promise; +}; + +/** + * Called when the installation has completed. + * @param {Component} that The gpii.iod.installer instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. + * @return {Promise} Resolves when complete. + */ +gpii.iod.installComplete = function (that, installation, packageData) { + var promise; + fluid.log("IoD: Completed installation of " + packageData.name); + if (packageData.installCommands.complete) { + promise = that.executeCommand(packageData.installCommands.complete); + } else { + promise = fluid.promise().resolve(); } - promise.resolve(); return promise; }; @@ -338,13 +448,15 @@ gpii.iod.cleanup = function (that) { * Starts the application. * * @param {Component} that The gpii.iod.installer instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves when the application has been started. */ -gpii.iod.startApplication = function (that) { +gpii.iod.startApplication = function (that, installation, packageData) { var promise = fluid.promise(); - fluid.log("IoD: Starting application " + that.packageData.name); - if (that.packageData.start) { - child_process.exec(that.packageData.start, function (err, stdout, stderr) { + fluid.log("IoD: Starting application " + packageData.name); + if (packageData.start) { + child_process.exec(packageData.start, function (err, stdout, stderr) { if (err) { fluid.log("IoD: startApplication error: ", err); } @@ -359,13 +471,15 @@ gpii.iod.startApplication = function (that) { * Stops the application (for uninstallation). * * @param {Component} that The gpii.iod.installer instance. + * @param {Installation} installation The installation state. + * @param {PackageData} packageData The package data. * @return {Promise} Resolves when the command has completed. */ -gpii.iod.stopApplication = function (that) { +gpii.iod.stopApplication = function (that, installation, packageData) { var promise = fluid.promise(); - fluid.log("IoD: Stopping application " + that.packageData.name); - if (that.packageData.start) { - child_process.exec(that.packageData.start, function (err, stdout, stderr) { + fluid.log("IoD: Stopping application " + packageData.name); + if (packageData.start) { + child_process.exec(packageData.start, function (err, stdout, stderr) { if (err) { fluid.log("IoD: stopApplication error: ", err); } @@ -378,6 +492,59 @@ gpii.iod.stopApplication = function (that) { return promise; }; +/** + * Expands "$(expanders)" in a string, whose content is a path to a field in the given object. + * + * Expanders are in the format of $(path) or $(path?default). + * Examples: + * "${a.b.c}", {a:{b:{c:"result"}}} returns "result". + * "${a.x?no}", {a:{b:{c:"result"}}} returns "no". + * + * @param {String|Object} unexpanded The input string, containing zero or more expanders. If an object, then string + * values within the object are worked on. + * @param {Object} sourceObject The object which the paths in the expanders refer to. + * @param {String} alwaysExpand `true` to make expanders that resolve to null/undefined resolve to an empty + * string, otherwise the function returns null. + * @return {String} The input string, with the expanders replaced by the value of the field they refer to. + */ +gpii.iod.expand = function (unexpanded, sourceObject, alwaysExpand) { + var unresolved = false; + var result; + + if (typeof(unexpanded) === "string") { + // Replace all occurences of "$(...)" + result = unexpanded.replace(/\$\(([^?}]*)(\?([^}]*))?\)/g, function (match, expression, defaultGroup, defaultValue) { + // Resolve the path to a field, deep in the object. + var value = expression.split(".").reduce(function (parent, property) { + return (parent && parent.hasOwnProperty(property)) ? parent[property] : undefined; + }, sourceObject); + + if (value === undefined || (typeof (value) === "object")) { + if (defaultGroup) { + value = defaultValue; + } + if (value === undefined || value === null) { + if (!alwaysExpand) { + unresolved = true; + } + value = ""; + } + } + return value; + }); + } else if (unexpanded === null || unexpanded === undefined) { + result = null; + } else if (fluid.isPlainObject(unexpanded)) { + result = fluid.transform(unexpanded, function (field) { + return gpii.iod.expand(field, sourceObject, alwaysExpand); + }); + } else { + result = unexpanded; + } + + return unresolved ? null : result; +}; + /** * Executes a command. * @param {Component} that The gpii.iod.installer instance. @@ -401,16 +568,19 @@ gpii.iod.executeCommand = function (that, iod, invocation, command, args) { invocation.args = fluid.makeArray(args); } + invocation = gpii.iod.expand(invocation, that.installation); + command = command ? gpii.iod.expand(command, that.installation) : invocation.command; + var promise; if (invocation.elevate && that.invokeElevated) { - promise = that.invokeElevated(invocation, command, args); + promise = that.invokeElevated(invocation, command, invocation.args); } else { if (invocation.elevate) { fluid.log(fluid.logLevel.WARN, "Running elevated commands is not supported on this operating system."); } promise = fluid.promise(); - + fluid.log("spawning: " + command + " ", invocation.args); var child = child_process.spawn(command, fluid.makeArray(invocation.args), { stdio: "inherit" }); diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index c782047a4..05c187f1e 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -55,6 +55,16 @@ require("./iodSettingsHandler.js"); * @property {String} uiLevel How much is displayed (if possible): "none" (default), "progress" (non-interactive), * "progress-cancel" (progress, can be cancelled), "full" (fully interactive, asks questions). * + * @property {Object} installCommands Commands to execute at certain points in the installation, rather than + * perform the default action (if any). + * @property {PackageCommand} installCommands.initialise The initialise command. + * @property {PackageCommand} installCommands.download The download command. + * @property {PackageCommand} installCommands.check The check command. + * @property {PackageCommand} installCommands.prepareInstall The prepareInstall command. + * @property {PackageCommand} installCommands.install The install command. + * @property {PackageCommand} installCommands.cleanup The cleanup command. + * @property {PackageCommand} installCommands.complete The installation is complete. + * */ /** diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index c48f40168..b00c70d76 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -83,51 +83,154 @@ fluid.defaults("gpii.tests.iodInstaller.installer", { uninstallPackage: "gpii.tests.iodInstaller.stage({that}, uninstallPackage)", stopApplication: "gpii.tests.iodInstaller.stage({that}, stopApplication)", installComplete: "gpii.tests.iodInstaller.stage({that}, installComplete)", - uninstallComplete: "gpii.tests.iodInstaller.stage({that}, uninstallComplete)" + uninstallComplete: "gpii.tests.iodInstaller.stage({that}, uninstallComplete)", + executeCommand: "gpii.tests.iodInstaller.stage({that}, execute, {arguments}.0.command, {arguments}.0.args.0, {arguments}.0.args.1)" }, packageTypes: "testPackageType1" }); -gpii.tests.iodInstaller.stage = function (that, stage) { - that.stages.push(stage); +gpii.tests.iodInstaller.stage = function (that) { + that.stages.push(fluid.makeArray(arguments).splice(1).join(",")); }; +gpii.tests.iodInstaller.installStages = [ + "initialise", + "downloadInstaller", + "checkPackage", + "prepareInstall", + "installPackage", + "cleanup", + "installComplete", + "startApplication" +]; + +gpii.tests.iodInstaller.uninstallStages = [ + "stopApplication", + "uninstallPackage", + "cleanup", + "uninstallComplete" +]; + // Test startInstaller starts the installation pipe-line. jqUnit.asyncTest("test installation pipe-line", function () { - var iod = gpii.tests.iodInstaller(); - var installer = iod.testInstaller; - jqUnit.expect(2); - installer.stages = []; + var testStages = function (packageData, expectInstall, expectUninstall) { + jqUnit.expect(2); + var iod = gpii.tests.iodInstaller(); + var installer = iod.testInstaller; + installer.stages = []; + installer.packageData = packageData; + installer.installation = { + packageData: packageData + }; - installer.startInstaller({}).then(function () { - var expect = [ - "initialise", - "downloadInstaller", - "checkPackage", - "prepareInstall", - "installPackage", - "cleanup", - "installComplete", - "startApplication" - ]; + var promise = fluid.promise(); - jqUnit.assertDeepEq("All stages of the installation should be called in order.", expect, installer.stages); + installer.startInstaller({}).then(function () { + var expect = expectInstall; - installer.stages = []; - installer.startUninstaller().then(function () { - var expect = [ + jqUnit.assertDeepEq("All stages of the installation should be called in order.", expect, installer.stages); + + installer.stages = []; + installer.startUninstaller().then(function () { + var expect = expectUninstall; + + jqUnit.assertDeepEq("All stages of the uninstallation should be called in order.", expect, installer.stages); + promise.resolve(); + }); + }, promise.reject); + + promise.then(function () { + iod.destroy(); + }); + + return promise; + }; + + var work = [ + function () { + fluid.log("Testing stages"); + return testStages({}, gpii.tests.iodInstaller.installStages, gpii.tests.iodInstaller.uninstallStages); + }, + function () { + // Test that the "packageData.installCommands.XX:before" and "XX:after" commands (installation) get invoked. + fluid.log("Testing stages with :before and :after commands"); + var packageData = { + installCommands: {}, + uninstallCommands: {} + }; + // install commands + fluid.each(gpii.tests.iodInstaller.installStages, function (stage) { + packageData.installCommands[stage + ":before"] = { + command: "install", + args: ["before", stage] + }; + packageData.installCommands[stage + ":after"] = { + command: "install", + args: ["after", stage] + }; + }); + var expectInstall = [ + "execute,install,before,initialise", + "initialise", + "execute,install,after,initialise", + "downloadInstaller", + "checkPackage", + "execute,install,before,prepareInstall", + "prepareInstall", + "execute,install,after,prepareInstall", + "installPackage", + "execute,install,before,cleanup", + "cleanup", + "execute,install,after,cleanup", + "installComplete", + "execute,install,before,startApplication", + "startApplication", + "execute,install,after,startApplication" + ]; + return testStages(packageData, expectInstall, gpii.tests.iodInstaller.uninstallStages); + }, + function () { + // ":before" and ":after commands (uninstallation) + fluid.log("Testing stages with :before and :after commands - uninstallation"); + var packageData = { + installCommands: {}, + uninstallCommands: {} + }; + + // uninstall commands + fluid.each(gpii.tests.iodInstaller.uninstallStages, function (stage) { + packageData.uninstallCommands[stage + ":before"] = { + command: "uninstall", + args: ["before", stage] + }; + packageData.uninstallCommands[stage + ":after"] = { + command: "uninstall", + args: ["after", stage] + }; + }); + var expectUninstall = [ + "execute,uninstall,before,stopApplication", "stopApplication", + "execute,uninstall,after,stopApplication", + "execute,uninstall,before,uninstallPackage", "uninstallPackage", + "execute,uninstall,after,uninstallPackage", + "execute,uninstall,before,cleanup", "cleanup", - "uninstallComplete" + "execute,uninstall,after,cleanup", + "execute,uninstall,before,uninstallComplete", + "uninstallComplete", + "execute,uninstall,after,uninstallComplete" ]; + return testStages(packageData, gpii.tests.iodInstaller.installStages, expectUninstall); + } + ]; - jqUnit.assertDeepEq("All stages of the uninstallation should be called in order.", expect, installer.stages); - jqUnit.start(); - }); + fluid.promise.sequence(work).then(function () { + jqUnit.start(); }, jqUnit.fail); }); From 5b75e743a0a5677de800d8daeb7538c9b9c9414a Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 30 Dec 2019 11:41:48 +0000 Subject: [PATCH 48/56] GPII-2971: Improved command execution --- .../gpii-iod/src/installOnDemand.js | 6 +- .../gpii-iod/src/packageInstaller.js | 132 +++++---- gpii/node_modules/gpii-iod/src/packages.js | 13 +- .../gpii-iod/test/packageInstallerTests.js | 268 ++++++++++++++++-- 4 files changed, 321 insertions(+), 98 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 73ac1028a..e91fc246e 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -60,13 +60,13 @@ fluid.defaults("gpii.iod", { createOnEvent: "onInstallerLoad", type: "{arguments}.0", options: { - installationID: "{arguments}.1" + installation: "{arguments}.1" } } }, events: { onServerFound: null, // [ endpoint address ] - onInstallerLoad: null // [ packageInstaller grade name, installation ID ] + onInstallerLoad: null // [ packageInstaller grade name, installation ] }, listeners: { "onCreate.discoverServer": "{that}.discoverServer", @@ -363,7 +363,7 @@ gpii.iod.initialiseInstallation = function (that, packageData) { var installerGrade = that.options.installerGrades[packageData.packageType]; if (installerGrade) { // Load the installer. - that.events.onInstallerLoad.fire(installerGrade, installation.id); + that.events.onInstallerLoad.fire(installerGrade, installation); promise.resolve(installation); } else { promise.reject({ diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index c2aa585c4..037d92d39 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -46,9 +46,15 @@ fluid.defaults("gpii.iod.packageInstaller", { }, executeCommand: { funcName: "gpii.iod.executeCommand", - // PackageInvocation, command, args + // PackageCommand, command, args args: ["{that}", "{iod}", "{arguments}.0", "{arguments}.1", "{arguments}.2"] }, + startProcess: { + funcName: "gpii.iod.startProcess", + // command, args + args: ["{that}", "{iod}", "{arguments}.0", "{arguments}.1"] + }, + // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns // an installation, either directly or via a promise. initialise: { @@ -166,8 +172,7 @@ fluid.defaults("gpii.iod.packageInstaller", { * @param {Component} that The gpii.iod.installer instance. * @param {Component} iod The gpii.iod instance. */ -gpii.iod.installerCreated = function (that, iod) { - that.installation = iod.installations[that.options.installationID]; +gpii.iod.installerCreated = function (that) { if (that.installation) { that.installation.installer = that; that.packageData = that.installation.packageData; @@ -514,6 +519,9 @@ gpii.iod.expand = function (unexpanded, sourceObject, alwaysExpand) { if (typeof(unexpanded) === "string") { // Replace all occurences of "$(...)" result = unexpanded.replace(/\$\(([^?}]*)(\?([^}]*))?\)/g, function (match, expression, defaultGroup, defaultValue) { + if (expression === "debug") { + fluid.log("expand object: ", sourceObject); + } // Resolve the path to a field, deep in the object. var value = expression.split(".").reduce(function (parent, property) { return (parent && parent.hasOwnProperty(property)) ? parent[property] : undefined; @@ -546,72 +554,82 @@ gpii.iod.expand = function (unexpanded, sourceObject, alwaysExpand) { }; /** - * Executes a command. - * @param {Component} that The gpii.iod.installer instance. - * @param {Component} iod The gpii.iod instance. - * @param {PackageInvocation} invocation How the command is invoked. - * @param {String} command The command. - * @param {Array|String} args [optional] The arguments (overrides `invocation.args`). - * @return {Promise} Resolves when complete. + * Starts a process, and waits for it to exit. + * @param {String} command The command to run. + * @param {Array} args [optional] The arguments to pass. + * @return {Promise} Resolves when the process terminates. */ -gpii.iod.executeCommand = function (that, iod, invocation, command, args) { - - if (typeof(invocation) === "string") { - // Can be expressed as a string, where it only specifies the arguments. - invocation = { args: invocation }; - } else { - // Take a copy to modify. - invocation = Object.assign({}, invocation); - } - - if (args) { - invocation.args = fluid.makeArray(args); - } - - invocation = gpii.iod.expand(invocation, that.installation); - command = command ? gpii.iod.expand(command, that.installation) : invocation.command; +gpii.iod.startProcess = function (command, args) { + args = fluid.makeArray(args); + var promise = fluid.promise(); + fluid.log("spawning: " + command + " ", args); + var child = child_process.spawn(command, fluid.makeArray(args), { + stdio: "inherit" + }); - var promise; - if (invocation.elevate && that.invokeElevated) { - promise = that.invokeElevated(invocation, command, invocation.args); - } else { - if (invocation.elevate) { - fluid.log(fluid.logLevel.WARN, "Running elevated commands is not supported on this operating system."); + child.on("error", function (err) { + if (!promise.disposition) { + promise.reject({ + isError: true, + error: err, + message: "Error running command", + command: command, + args: args + }); } - - promise = fluid.promise(); - fluid.log("spawning: " + command + " ", invocation.args); - var child = child_process.spawn(command, fluid.makeArray(invocation.args), { - stdio: "inherit" - }); - - child.on("error", function (err) { + }); + child.on("exit", function (code) { + if (code) { if (!promise.disposition) { promise.reject({ isError: true, - error: err, + exitCode: code, message: "Error running command", command: command, - invocation: invocation + args: args }); } + } else { + promise.resolve(); + } + }); + return promise; +}; + +/** + * Executes a command, which was specified in the package data. + * + * @param {Component} that The gpii.iod.installer instance. + * @param {PackageCommand} execOptions How the command is invoked. + * @param {String} command The command (overrides `execOptions.command`. + * @param {Array|String} args [optional] The arguments (overrides `execOptions.args`). + * @return {Promise} Resolves when complete. + */ +gpii.iod.executeCommand = function (that, execOptions, command, args) { + // Take a copy to modify. + execOptions = Object.assign({}, execOptions); + + if (command) { + execOptions.command = command; + } + + execOptions.args = fluid.makeArray(args || execOptions.args); + execOptions = gpii.iod.expand(execOptions, that.installation); + + var promise; + if (!execOptions.command) { + promise = fluid.promise().reject({ + isError: true, + message: "executeCommand called without a command" }); - child.on("exit", function (code) { - if (code) { - if (!promise.disposition) { - promise.reject({ - isError: true, - exitCode: code, - message: "Error running command", - command: command, - invocation: invocation - }); - } - } else { - promise.resolve(); - } - }); + } else if (execOptions.elevate && that.startElevatedProcess) { + promise = that.startElevatedProcess(execOptions.command, execOptions.args, {desktop: execOptions.desktop}); + } else { + if (execOptions.elevate) { + fluid.log(fluid.logLevel.WARN, "Running elevated commands is not supported on this system."); + } + promise = that.startProcess(execOptions.command, execOptions.args); } return promise; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 05c187f1e..4b09762bc 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -46,8 +46,8 @@ require("./iodSettingsHandler.js"); * * @property {Boolean} keepInstaller `true` to keep the installer file after installing (removes after uninstall). * - * @property {PackageInvocation|String} installerArgs Additional options used when executing the installer. - * @property {PackageInvocation|String} uninstallerArgs Additional options used when executing the uninstaller. + * @property {PackageCommand|String} installerArgs Additional options used when executing the installer. + * @property {PackageCommand|String} uninstallerArgs Additional options used when executing the uninstaller. * * @property {String} uninstallTime When to uninstall this package, after it's no longer required: "immediate", "idle", * "never". @@ -69,18 +69,13 @@ require("./iodSettingsHandler.js"); /** * Describes how something is invoked. - * @typedef {Object} PackageInvocation + * @typedef {Object} PackageCommand + * @property {String} command The command to invoke. * @property {String|Array} args arguments passed to the command. * @property {Boolean} elevate true to run as administrator. * @property {Boolean} desktop true to run in the context of the desktop, if elevate is true. */ -/** - * A command - * @typedef {PackageInvocation} PackageCommand - * @property {String} command The command to invoke. - */ - gpii.iod.resolvers = {}; fluid.defaults("gpii.iod.packages", { diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index b00c70d76..c35a247c0 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -43,33 +43,182 @@ jqUnit.module("gpii.tests.iodInstaller", { } }); -fluid.defaults("gpii.tests.iodInstaller", { - gradeNames: [ "gpii.iod" ], - components: { - "testInstaller": { - type: "gpii.tests.iodInstaller.installer" +gpii.tests.iodInstaller.executeCommandTests = fluid.freezeRecursive([ + { + id: "command only", + command: "command", + expect: { + funcName: "startProcess", + command: "command", + args: [], + options: undefined + } + }, + { + id: "command + args", + command: "command", + args: ["arg1", "arg2", "arg3"], + expect: { + funcName: "startProcess", + command: "command", + args: ["arg1", "arg2", "arg3"], + options: undefined + } + }, + { + id: "args in options", + command: "command", + args: undefined, + execOptions: { + args: ["arg1", "arg2", "arg3"] }, - "packages": { - type: "gpii.iod.packages", - options: { - components: { - "packageDataSource": { - type: "kettle.dataSource.file", - options: { - path: __dirname + "/packageData/%packageName.json5" - } - } - } + expect: { + funcName: "startProcess", + command: "command", + args: ["arg1", "arg2", "arg3"], + options: undefined + } + }, + { + id: "command in options", + command: undefined, + args: undefined, + execOptions: { + command: "command", + args: ["arg1", "arg2", "arg3"] + }, + expect: { + funcName: "startProcess", + command: "command", + args: ["arg1", "arg2", "arg3"], + options: undefined + } + }, + { + id: "command + args in options, overridden", + command: "command", + args: ["arg1", "arg2", "arg3"], + execOptions: { + command: "unexpected", + args: ["unexpected1"] + }, + expect: { + funcName: "startProcess", + command: "command", + args: ["arg1", "arg2", "arg3"], + options: undefined + } + }, + { + id: "no command", + command: undefined, + args: ["arg1", "arg2", "arg3"], + execOptions: { + command: undefined, + args: ["unexpected1"] + }, + expect: { + reject: true + } + }, + { + id: "expand", + command: "$(value)", + args: ["a$(value)", "$(value2.value3)", "$(nothing?expected3)", "x $(nothing)", "y $(nothing?)"], + execOptions: { + }, + installation: { + value: "expected", + value2: { + value3: "expected2" + } + }, + expect: { + funcName: "startProcess", + command: "expected", + args: ["aexpected", "expected2", "expected3", null, "y "], + options: undefined + } + }, + { + id: "expand (options)", + command: undefined, + args: undefined, + execOptions: { + command: "$(value)", + args: ["a$(value)", "$(value2.value3)", "$(nothing?expected3)", "x $(nothing)", "y $(nothing?)"] + }, + installation: { + value: "expected", + value2: { + value3: "expected2" } + }, + expect: { + funcName: "startProcess", + command: "expected", + args: ["aexpected", "expected2", "expected3", null, "y "], + options: undefined } }, + { + id: "elevated", + command: "command $(value)", + args: undefined, + execOptions: { + args: ["$(value)", "arg2"], + elevate: true + }, + installation: { + value: "expected" + }, + expect: { + funcName: "startElevatedProcess", + command: "command expected", + args: ["expected", "arg2"], + options: {desktop: undefined} + } + }, + { + id: "elevated (desktop)", + command: "command $(value)", + args: undefined, + execOptions: { + args: ["$(value)", "arg2"], + elevate: true, + desktop: true + }, + installation: { + value: "expected" + }, + expect: { + funcName: "startElevatedProcess", + command: "command expected", + args: ["expected", "arg2"], + options: {desktop: true} + } + } +]); + +fluid.defaults("gpii.tests.iodInstaller.dummyInstaller", { + gradeNames: ["fluid.component", "gpii.iod.packageInstaller"], + invokers: { - readInstallations: "fluid.identity", - writeInstallation: "fluid.identity" + initialise: "fluid.identity", + downloadInstaller: "fluid.identity", + checkPackage: "fluid.identity", + prepareInstall: "fluid.identity", + installPackage: "fluid.identity", + cleanup: "fluid.identity", + startApplication: "fluid.identity", + uninstallPackage: "fluid.identity", + stopApplication: "fluid.identity", + installComplete: "fluid.identity", + uninstallComplete: "fluid.identity", + executeCommand: "fluid.identity" } }); - -fluid.defaults("gpii.tests.iodInstaller.installer", { +fluid.defaults("gpii.tests.iodInstaller.loggingInstaller", { gradeNames: ["fluid.component", "gpii.iod.packageInstaller"], invokers: { @@ -85,9 +234,7 @@ fluid.defaults("gpii.tests.iodInstaller.installer", { installComplete: "gpii.tests.iodInstaller.stage({that}, installComplete)", uninstallComplete: "gpii.tests.iodInstaller.stage({that}, uninstallComplete)", executeCommand: "gpii.tests.iodInstaller.stage({that}, execute, {arguments}.0.command, {arguments}.0.args.0, {arguments}.0.args.1)" - }, - - packageTypes: "testPackageType1" + } }); gpii.tests.iodInstaller.stage = function (that) { @@ -114,12 +261,10 @@ gpii.tests.iodInstaller.uninstallStages = [ // Test startInstaller starts the installation pipe-line. jqUnit.asyncTest("test installation pipe-line", function () { - - var testStages = function (packageData, expectInstall, expectUninstall) { jqUnit.expect(2); - var iod = gpii.tests.iodInstaller(); - var installer = iod.testInstaller; + + var installer = gpii.tests.iodInstaller.loggingInstaller(); installer.stages = []; installer.packageData = packageData; installer.installation = { @@ -143,7 +288,7 @@ jqUnit.asyncTest("test installation pipe-line", function () { }, promise.reject); promise.then(function () { - iod.destroy(); + installer.destroy(); }); return promise; @@ -234,7 +379,7 @@ jqUnit.asyncTest("test installation pipe-line", function () { }, jqUnit.fail); }); - +// Test the file download jqUnit.asyncTest("test https download", function () { if (process.env.GPII_QUICKTEST) { @@ -371,3 +516,68 @@ jqUnit.asyncTest("test https download", function () { nextTest(); }); + +jqUnit.asyncTest("test executeCommand", function () { + + var tests = gpii.tests.iodInstaller.executeCommandTests; + + jqUnit.expect(tests.length * 2); + + var currentTest, messageSuffix; + + var installer = gpii.tests.iodInstaller.dummyInstaller({ + invokers: { + startProcess: { + func: function (args) { + jqUnit.assertDeepEq("startProcess should be called correctly" + messageSuffix, + currentTest.expect, args); + return fluid.promise().resolve(); + }, + args: [{ + funcName: "startProcess", + command: "{arguments}.0", + args: "{arguments}.1", + options: "{arguments}2" + }] + }, + startElevatedProcess: { + func: function (args) { + jqUnit.assertDeepEq("startElevatedProcess should be called correctly" + messageSuffix, + currentTest.expect, args); + return fluid.promise().resolve(); + }, + args: [{ + funcName: "startElevatedProcess", + command: "{arguments}.0", + args: "{arguments}.1", + options: "{arguments}2" + }] + } + } + }); + + + var doTest = function (testIndex) { + currentTest = tests[testIndex]; + if (currentTest) { + messageSuffix = " - test:" + currentTest.id; + installer.installation = currentTest.installation; + var p = gpii.iod.executeCommand(installer, currentTest.execOptions, currentTest.command, currentTest.args); + jqUnit.assertTrue("executeCommand must return a promise" + messageSuffix, fluid.isPromise(p)); + + p.then(function () { + doTest(testIndex + 1); + }, function () { + jqUnit.assertTrue("executeCommand should reject if expected" + messageSuffix, + currentTest.expect.reject); + doTest(testIndex + 1); + }); + + } else { + jqUnit.start(); + } + }; + + doTest(0); + +}); From 684280d7953af00c8eb3a4662ec01346c573c5cd Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 30 Dec 2019 20:42:18 +0000 Subject: [PATCH 49/56] GPII-2971: implemented multiDataSource --- .../gpii-iod/src/multiDataSource.js | 115 +++++++++ .../gpii-iod/test/multiDataSourceTests.js | 244 ++++++++++++++++++ 2 files changed, 359 insertions(+) create mode 100644 gpii/node_modules/gpii-iod/src/multiDataSource.js create mode 100644 gpii/node_modules/gpii-iod/test/multiDataSourceTests.js diff --git a/gpii/node_modules/gpii-iod/src/multiDataSource.js b/gpii/node_modules/gpii-iod/src/multiDataSource.js new file mode 100644 index 000000000..87135d77f --- /dev/null +++ b/gpii/node_modules/gpii-iod/src/multiDataSource.js @@ -0,0 +1,115 @@ +/* + * A readonly data source which encapsulates multiple data sources into one, where each source gets queried until one + * of them returns a result. + * + * Copyright 2019 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); + +require("kettle"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.iod.multiDataSource"); + +fluid.defaults("gpii.iod.multiDataSource", { + gradeNames: ["kettle.dataSource"], + readOnlyGrade: "gpii.iod.multiDataSource", + + components: { + encoding: { + // The root sources provide their own encoding. + type: "kettle.dataSource.encoding.none" + } + }, + dynamicComponents: { + rootDataSources: { + createOnEvent: "onNewDataSource", + type: "{arguments}.0", + options: { + path: "{arguments}.1", + priority: "{arguments}.2", + listeners: { + "onCreate.multiDataSource": { + func: "{gpii.iod.multiDataSource}.addDataSource", + args: ["{that}"] + } + } + } + } + }, + + events: { + onNewDataSource: null // component name, path, priority + }, + + members: { + sortedDataSources: [] + }, + + invokers: { + getImpl: { + funcName: "gpii.iod.multiDataSource.getImpl", + args: [ "{that}", "{arguments}.0", "{arguments}.1" ] + }, + addDataSource: { + funcName: "gpii.iod.multiDataSource.addDataSource", + args: [ "{that}", "{arguments}.0"] + } + } +}); + +gpii.iod.multiDataSource.addDataSource = function (that, dataSource) { + // Insert the new one at the top of the array, so newer sources of the same priority are before the older ones. + that.sortedDataSources.unshift(dataSource); + dataSource.options.priority = parseInt(dataSource.options.priority); + fluid.stableSort(that.sortedDataSources, function (a, b) { + return a.options.priority - b.options.priority; + }); +}; + +gpii.iod.multiDataSource.getImpl = function (that, options, directModel) { + var promise = fluid.promise(); + + var firstReject; + + var next = function (index) { + var source = that.sortedDataSources[index]; + if (source) { + var result = source.get(directModel, options); + result.then(promise.resolve, function (reason) { + if (!firstReject) { + firstReject = reason; + } + next(index + 1); + }); + } else { + promise.reject(firstReject); + } + }; + + if (that.sortedDataSources.length) { + next(0); + } else { + promise.reject({ + isError: true, + message: "no root data sources" + }); + } + + return promise; +}; diff --git a/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js b/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js new file mode 100644 index 000000000..9c84d5bbd --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js @@ -0,0 +1,244 @@ +/* + * Tests for the multiDataSource component. + * + * Copyright 2019 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); +var kettle = fluid.require("kettle"); +kettle.loadTestingSupport(); + +var gpii = fluid.registerNamespace("gpii"); + +fluid.registerNamespace("gpii.tests.multiDataSource"); + +require("../src/multiDataSource.js"); + + +fluid.defaults("gpii.tests.multiDataSource.dataSource", { + gradeNames: ["kettle.dataSource"], + readOnlyGrade: "gpii.tests.multiDataSource", + + components: { + encoding: { + type: "kettle.dataSource.encoding.none" + } + }, + invokers: { + getImpl: { + funcName: "gpii.tests.multiDataSource.getImpl", + args: ["{that}", "{arguments}.0", "{arguments}.1"] + } + } +}); + +gpii.tests.multiDataSource.getImpl = function (that, options, directModel) { + var result; + var req = fluid.makeArray(directModel.request); + + if (req.indexOf(that.options.path) > -1 || directModel.request === "any") { + result = { + from: that.options.path + }; + } + + return result ? fluid.toPromise(result) : fluid.promise().reject({notFound: that.options.path}); +}; + + +fluid.defaults("gpii.tests.multiDataSource.tests", { + gradeNames: ["fluid.test.testEnvironment"], + + components: { + multiDataSource: { + type: "gpii.iod.multiDataSource" + }, + tester: { + type: "gpii.tests.multiDataSource.testCaseHolder" + } + } +}); + +fluid.defaults("gpii.tests.multiDataSource.testCaseHolder", { + gradeNames: ["fluid.test.testCaseHolder"], + modules: [{ + name: "multiDataSource", + tests: [{ + expect: 1, + name: "testing empty multiDataSource", + sequence: [{ + task: "{multiDataSource}.get", + args: [{request: "value"}], + reject: "jqUnit.assert", + rejectArgs: [ + "multiDataSource.get() with no root data sources should reject" + ] + }] + }, { + expect: 3, + name: "adding to multiDataSource", + sequence: [{ + func: "jqUnit.assertEquals", + args: [ + "sortedDataSources member should be empty before adding one", + 0, + "{multiDataSource}.sortedDataSources.length" + ] + }, { + func: "{multiDataSource}.events.onNewDataSource.fire", + args: ["gpii.tests.multiDataSource.dataSource", "first", 1] + }, { + func: "jqUnit.assertEquals", + args: [ + "sortedDataSources member should contain one item after adding it", + 1, + "{multiDataSource}.sortedDataSources.length" + ] + }, { + func: "{multiDataSource}.events.onNewDataSource.fire", + // 3rd is added before the 2nd, to prove the priorities work + args: ["gpii.tests.multiDataSource.dataSource", "third", 3] + }, { + func: "{multiDataSource}.events.onNewDataSource.fire", + args: ["gpii.tests.multiDataSource.dataSource", "second", 2] + }, { + func: "jqUnit.assertEquals", + args: [ + "sortedDataSources member should contain two more items after adding the second and third", + 3, + "{multiDataSource}.sortedDataSources.length" + ] + }] + }, { + expect: 12, + name: "getting from multiDataSource", + sequence: [{ + func: "jqUnit.assertEquals", + args: [ + "sortedDataSources member should be contain the items from the last test", + 3, + "{multiDataSource}.sortedDataSources.length" + ] + }, { + task: "{multiDataSource}.get", + args: [{request: "first"}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from first source should be received for 'first'", {from: "first"}, "{arguments}.0" + ] + }, { + task: "{multiDataSource}.get", + args: [{request: "second"}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from second source should be received for 'second'", {from: "second"}, "{arguments}.0" + ] + }, { + task: "{multiDataSource}.get", + args: [{request: "third"}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from third source should be received for 'third'", {from: "third"}, "{arguments}.0" + ] + }, { + task: "{multiDataSource}.get", + args: [{request: "any"}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from first source should be received for 'any'", {from: "first"}, "{arguments}.0" + ] + }, { + task: "{multiDataSource}.get", + args: [{request: "none"}], + reject: "jqUnit.assertDeepEq", + rejectArgs: [ + "reject from first source should be received for 'none'", {notFound: "first"}, "{arguments}.0" + ] + }, { + // tests ordering + task: "{multiDataSource}.get", + args: [{request: ["first", "third"]}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from first source should be received for 'first or third'", {from: "first"}, "{arguments}.0" + ] + }, { + // tests ordering, even if added later. + task: "{multiDataSource}.get", + args: [{request: ["second", "third"]}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from second source should be received for 'second or third'", + {from: "second"}, + "{arguments}.0" + ] + }, { + // Add a higher priority source + func: "{multiDataSource}.events.onNewDataSource.fire", + args: ["gpii.tests.multiDataSource.dataSource", "zeroth", 0] + }, { + // Check the newly added source is used first + task: "{multiDataSource}.get", + args: [{request: "any"}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from zeroth should be received for 'any'", {from: "zeroth"}, "{arguments}.0" + ] + }, { + // Add a new source at the same priority as 'second' + func: "{multiDataSource}.events.onNewDataSource.fire", + args: ["gpii.tests.multiDataSource.dataSource", "new-second", 2] + }, { + // Check the newly added source is used - the same priority, but it's newer. + task: "{multiDataSource}.get", + args: [{request: ["new-second", "second", "third"]}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from before-second should be received for 'before-second or second'", + {from: "new-second"}, + "{arguments}.0" + ] + }, { + // Check the new source didn't break the higher priority source + task: "{multiDataSource}.get", + args: [{request: "any"}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from zeroth should be received for 'any' #2", {from: "zeroth"}, "{arguments}.0" + ] + }, { + // Check the sorted list is in the right order + func: "jqUnit.assertDeepEq", + args: [ + "Sorted data sources should be in the expected order", + [ + "zeroth", + "first", + "new-second", + "second", + "third" + ], + "@expand:fluid.getMembers({multiDataSource}.sortedDataSources, options.path)" + ] + }] + }] + }] +}); + +module.exports = kettle.test.bootstrap("gpii.tests.multiDataSource.tests"); + + From 9930895019ea445f49a08248dd44a864cf50f461 Mon Sep 17 00:00:00 2001 From: ste Date: Thu, 2 Jan 2020 21:36:36 +0000 Subject: [PATCH 50/56] GPII-2971: multiDataSource root sources can be removed. --- .../gpii-iod/src/multiDataSource.js | 49 ++++++++++++++++--- .../gpii-iod/test/multiDataSourceTests.js | 30 ++++++++++++ 2 files changed, 71 insertions(+), 8 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/multiDataSource.js b/gpii/node_modules/gpii-iod/src/multiDataSource.js index 87135d77f..f949fc230 100644 --- a/gpii/node_modules/gpii-iod/src/multiDataSource.js +++ b/gpii/node_modules/gpii-iod/src/multiDataSource.js @@ -47,32 +47,42 @@ fluid.defaults("gpii.iod.multiDataSource", { "onCreate.multiDataSource": { func: "{gpii.iod.multiDataSource}.addDataSource", args: ["{that}"] + }, + "onDestroy.removeSource": { + func: "{gpii.iod.multiDataSource}.removeDataSource", + args: ["{that}"] } } } } }, - events: { onNewDataSource: null // component name, path, priority }, - members: { sortedDataSources: [] }, - invokers: { getImpl: { funcName: "gpii.iod.multiDataSource.getImpl", - args: [ "{that}", "{arguments}.0", "{arguments}.1" ] + args: ["{that}", "{arguments}.0", "{arguments}.1"] }, addDataSource: { funcName: "gpii.iod.multiDataSource.addDataSource", - args: [ "{that}", "{arguments}.0"] + args: ["{that}", "{arguments}.0"] + }, + removeDataSource: { + funcName: "gpii.iod.multiDataSource.removeDataSource", + args: ["{that}", "{arguments}.0"] } } }); +/** + * Called when a new data source is to be added. + * @param {Component} that The gpii.iod.multiDataSource instance. + * @param {Component} dataSource The new data source. + */ gpii.iod.multiDataSource.addDataSource = function (that, dataSource) { // Insert the new one at the top of the array, so newer sources of the same priority are before the older ones. that.sortedDataSources.unshift(dataSource); @@ -82,13 +92,36 @@ gpii.iod.multiDataSource.addDataSource = function (that, dataSource) { }); }; +/** + * Called when an existing data source has gone. + * @param {Component} that The gpii.iod.multiDataSource instance. + * @param {Component} dataSource The removed data source. + */ +gpii.iod.multiDataSource.removeDataSource = function (that, dataSource) { + var index = that.sortedDataSources.indexOf(dataSource); + if (index === -1) { + fluid.log("multiDataSource: removed an unknown data source"); + } else { + that.sortedDataSources.splice(index, 1); + } +}; + +/** + * Data source getter. Attempts a get() on each root data source until the first success. + * @param {Component} that The gpii.iod.multiDataSource instance. + * @param {Object} options The options. + * @param {Object} directModel The request. + * @return {Promise} Resolves with the result, rejects with the first rejection if all sources reject. + */ gpii.iod.multiDataSource.getImpl = function (that, options, directModel) { var promise = fluid.promise(); - var firstReject; + // Copy the data source array, so it doesn't get modified while being enumerated. + var dataSources = that.sortedDataSources.slice(); + var next = function (index) { - var source = that.sortedDataSources[index]; + var source = dataSources[index]; if (source) { var result = source.get(directModel, options); result.then(promise.resolve, function (reason) { @@ -102,7 +135,7 @@ gpii.iod.multiDataSource.getImpl = function (that, options, directModel) { } }; - if (that.sortedDataSources.length) { + if (dataSources.length) { next(0); } else { promise.reject({ diff --git a/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js b/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js index 9c84d5bbd..ba9b5a3c9 100644 --- a/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js +++ b/gpii/node_modules/gpii-iod/test/multiDataSourceTests.js @@ -235,6 +235,36 @@ fluid.defaults("gpii.tests.multiDataSource.testCaseHolder", { "@expand:fluid.getMembers({multiDataSource}.sortedDataSources, options.path)" ] }] + }, { + name: "Removing data sources", + expect: 2, + sequence: [{ + func: "{multiDataSource}.sortedDataSources.2.destroy" + }, { + // Check the destroyed source isn't in the list. + func: "jqUnit.assertDeepEq", + args: [ + "Destroyed data source should have been removed", + [ + "zeroth", + "first", + // removed: "new-second", + "second", + "third" + ], + "@expand:fluid.getMembers({multiDataSource}.sortedDataSources, options.path)" + ] + }, { + // Check things still work as expected, and no results from the removed source. + task: "{multiDataSource}.get", + args: [{request: ["new-second", "second", "third"]}], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "result from second should be received for 'before-second or second', after removing before-second", + {from: "second"}, + "{arguments}.0" + ] + }] }] }] }); From 074360765c83f6a10d873a6911ff9e1722a9fd2a Mon Sep 17 00:00:00 2001 From: ste Date: Sat, 4 Jan 2020 20:10:45 +0000 Subject: [PATCH 51/56] GPII-2971: Able to take packages from a local path. --- .../gpii-iod/src/installOnDemand.js | 25 +- .../gpii-iod/src/multiDataSource.js | 3 +- .../gpii-iod/src/packageDataSource.js | 231 +++++- .../gpii-iod/src/packageInstaller.js | 85 +- gpii/node_modules/gpii-iod/src/packages.js | 101 +-- .../gpii-iod/test/installOnDemandTests.js | 31 +- .../gpii-iod/test/packageDataSourceTests.js | 779 +++++++++++++++--- .../gpii-iod/test/packageInstallerTests.js | 97 ++- .../gpii-iod/test/packagesTests.js | 712 +++++++++------- 9 files changed, 1450 insertions(+), 614 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index e91fc246e..c99d80ebb 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -34,12 +34,13 @@ fluid.registerNamespace("gpii.iod"); * @typedef {Object} Installation * @property {id} - Installation ID * @property {PackageData} packageData - Package data. - * @property {string} packageName - packageData.name + * @property {String} packageName - packageData.name * @property {Component} installer - The gpii.iod.installer instance. - * @property {boolean} failed - true if the installation had failed. - * @property {string} tmpDir - Temporary working directory. - * @property {string} installerFile - Path to the downloaded package file. - * @property {string[]} cleanupPaths - The directories to remove during cleanup. + * @property {Boolean} failed - true if the installation had failed. + * @property {String} tmpDir - Temporary working directory. + * @property {String} installerFile - Path to the downloaded package file. + * @property {String} installerFileHash - The hash of the downloaded installer. + * @property {String[]} cleanupPaths - The directories to remove during cleanup. * */ @@ -50,8 +51,11 @@ fluid.defaults("gpii.iod", { type: "gpii.iod.packages", options: { events: { - "onServerFound": "{gpii.iod}.events.onServerFound" - } + "onServerFound": "{gpii.iod}.events.onServerFound", + "onLocalPackagesFound": "{gpii.iod}.events.onLocalPackagesFound" + }, + // Map of recognised keys that sign the packages. + trustedKeys: "{gpii.iod}.options.config.trustedKeys" } } }, @@ -60,12 +64,13 @@ fluid.defaults("gpii.iod", { createOnEvent: "onInstallerLoad", type: "{arguments}.0", options: { - installation: "{arguments}.1" + installationID: "{arguments}.1" } } }, events: { onServerFound: null, // [ endpoint address ] + onLocalPackagesFound: null, // [ directory ] onInstallerLoad: null // [ packageInstaller grade name, installation ] }, listeners: { @@ -123,7 +128,7 @@ fluid.defaults("gpii.iod", { // The IoD server address endpoint: undefined, // Map of recognised keys that sign the packages. - allowedKeys: {}, + trustedKeys: {}, // Packages to install on startup autoInstall: [] }, @@ -363,7 +368,7 @@ gpii.iod.initialiseInstallation = function (that, packageData) { var installerGrade = that.options.installerGrades[packageData.packageType]; if (installerGrade) { // Load the installer. - that.events.onInstallerLoad.fire(installerGrade, installation); + that.events.onInstallerLoad.fire(installerGrade, installation.id); promise.resolve(installation); } else { promise.reject({ diff --git a/gpii/node_modules/gpii-iod/src/multiDataSource.js b/gpii/node_modules/gpii-iod/src/multiDataSource.js index f949fc230..217b5e591 100644 --- a/gpii/node_modules/gpii-iod/src/multiDataSource.js +++ b/gpii/node_modules/gpii-iod/src/multiDataSource.js @@ -41,8 +41,9 @@ fluid.defaults("gpii.iod.multiDataSource", { createOnEvent: "onNewDataSource", type: "{arguments}.0", options: { - path: "{arguments}.1", priority: "{arguments}.2", + // The "path" or "url" value for the data source. + address: "{arguments}.1", listeners: { "onCreate.multiDataSource": { func: "{gpii.iod.multiDataSource}.addDataSource", diff --git a/gpii/node_modules/gpii-iod/src/packageDataSource.js b/gpii/node_modules/gpii-iod/src/packageDataSource.js index b05c1a0a0..36fb1002c 100644 --- a/gpii/node_modules/gpii-iod/src/packageDataSource.js +++ b/gpii/node_modules/gpii-iod/src/packageDataSource.js @@ -19,39 +19,234 @@ "use strict"; var fluid = require("infusion"), - crypto = require("crypto"); + crypto = require("crypto"), + fs = require("fs"), + json5 = require("json5"), + path = require("path"), + url = require("url"); require("kettle"); var gpii = fluid.registerNamespace("gpii"); -fluid.registerNamespace("gpii.iod.packages"); +fluid.registerNamespace("gpii.iod.packageDataSource"); -fluid.defaults("gpii.iod.remotePackageDataSource", { - gradeNames: ["kettle.dataSource.URL"], - invokers: { - "checkPackageSignature": { - funcName: "gpii.iod.checkPackageSignature", - args: [ "{arguments}.0", "{gpii.iod}.options.config.allowedKeys" ] +fluid.defaults("gpii.iod.packageDataSource", { + gradeNames: ["fluid.component"], + readOnlyGrade: "gpii.iodServer.packageDataSource", + events: { + onLostDataSource: null + } +}); + +fluid.defaults("gpii.iod.packageDataSource.remote", { + gradeNames: ["kettle.dataSource.URL", "gpii.iod.packageDataSource"], + url: "@expand:gpii.iod.joinUrl({that}.options.address, {that}.options.urlPath)", + urlPath: "/packages/%packageName", + termMap: { + packageName: "%packageName" + } +}); + +fluid.defaults("gpii.iod.packageDataSource.local", { + gradeNames: ["kettle.dataSource", "gpii.iod.packageDataSource"], + + components: { + encoding: { + type: "kettle.dataSource.encoding.none" } }, - listeners: { - "onRead.checkSignature": { - func: "{that}.checkPackageSignature", - args: "{arguments}.0" // packageResponse + invokers: { + getImpl: { + funcName: "gpii.iod.packageDataSource.getLocal", + args: [ + "@expand:{that}.loadData()", + //"{arguments}.0", + "{arguments}.1", + "{that}.options.path" + ] + }, + loadData: { + funcName: "gpii.iod.packageDataSource.loadData", + args: ["{that}", "{that}.options.path", "{that}.options.dataFile"] } + }, + path: "{that}.options.address", + dataFile: ".morphic-packages", + members: { + data: undefined } }); +/** + * @typedef {PackageResponse} LocalPackage + */ + +/** + * Convenience function to concatenate two parts of a URL together, ensuring there's a single '/' between each part + * (like `path.join`, but performs no normalisation and always uses slash). + * + * @param {String} front The first part of the URL + * @param {String} end The final part of the URL + * @return {String} `font` and `end` combined. + */ +gpii.iod.joinUrl = function (front, end) { + return front.replace(/\/+$/, "") + "/" + end.replace(/^\/+/, ""); +}; + +/** + * Gets the package data from a local path. + * @param {Promise} loadPromise A promise that resolves when the local package data is available. + * @param {Object} packageRequest The package request. + * @param {Object} directory The directory where the local packages are located. + * @return {Promise} Resolves with the package response. + */ +gpii.iod.packageDataSource.getLocal = function (loadPromise, packageRequest, directory) { + var promise = fluid.promise(); + loadPromise.then(function (data) { + /** @type {LocalPackage} */ + var localPackage = fluid.copy(data[packageRequest.packageName]); + if (localPackage) { + var checkInstallerPromise; + + if (localPackage.installer) { + // Get the absolute path, and create the url for it. + var localPath = path.join(directory, localPackage.installer); + var installerURL = url.pathToFileURL(localPath); + if (localPackage.offset) { + installerURL.searchParams.set("offset", localPackage.offset); + } + localPackage.installer = installerURL.toString(); + + // Check if the installer file exists, as it would be pointless to return a response if the package + // can't be installed. + checkInstallerPromise = fluid.promise(); + fs.access(localPath, function (err) { + if (err) { + checkInstallerPromise.reject({ + isError: true, + message: "File for " + packageRequest.packageName + " not found", + localPath: localPath, + error: err + }); + } else { + checkInstallerPromise.resolve(); + } + }); + } + + fluid.toPromise(checkInstallerPromise).then(function () { + promise.resolve(localPackage); + }, promise.reject); + + } else { + promise.reject({ + message: "Package " + packageRequest.packageName + " not found" + }); + } + }, promise.reject); + + return promise; +}; + +/** + * Loads the package data from a local file. + * @param {Component} that The gpii.iod.packageDataSource.local instance. + * @param {String} directory The directory where the packages are kept. + * @param {String} file The local package data file (a child of `directory`) (".morphic-packages") + * @return {Promise>} Resolves with the packages from the local data file. + */ +gpii.iod.packageDataSource.loadData = function (that, directory, file) { + var promise = fluid.promise(); + var dataFile = path.join(directory, file); + + gpii.iod.packageDataSource.checkDataFile(that, dataFile).then(function (changed) { + if (changed || !that.data) { + fs.readFile(dataFile, "utf8", function (err, content) { + if (err) { + promise.reject({ + isError: true, + message: "IoD: unable to load the data file " + dataFile + ": " + err.message, + error: err + }); + } else { + try { + that.data = json5.parse(content); + if (!that.data.packages) { + that.data.packages = {}; + } + fluid.log("IoD: Loaded " + Object.keys(that.data.packages).length + " packages from " + dataFile); + promise.resolve(that.data.packages); + } catch (err) { + promise.reject({ + isError: true, + message: "IoD: unable to parse the data file " + dataFile + ": " + err.message, + error: err + }); + } + } + }); + } else { + promise.resolve(that.data.packages); + } + }, function (err) { + fluid.log("IoD: Unable to read from " + dataFile + ": ", err.message || err); + that.destroy(); + promise.reject(err); + }); + + return promise; +}; + +/** + * Checks if the data file exists, and has changed since the last time this was called. + * @param {Component} that The gpii.iod.packageDataSource.local instance. + * @param {String} dataFile Path to the .morphic-packages file. + * @return {Promise} Resolves with a boolean indicating if the file has changed (or the first call). Rejects + * if the file does not exist. + */ +gpii.iod.packageDataSource.checkDataFile = function (that, dataFile) { + var promise = fluid.promise(); + fs.stat(dataFile, function (err, stats) { + if (err) { + promise.reject(err); + } else { + var changed = stats.mtimeMs !== that.dataFileTime; + if (changed && that.dataFileTime) { + fluid.log("IoD: local package data file has changed - reloading"); + } + that.dataFileTime = stats.mtimeMs; + promise.resolve(changed); + } + }); + + return promise; +}; + +// fluid.defaults("gpii.iod.packageDataSource.local", { +// gradeNames: [ +// "kettle.dataSource.file.moduleTerms", "kettle.dataSource.encoding.JSON5", "gpii.iod.packageDataSource" +// ], +// path: "%gpii-universal/testData/installOnDemand/%packageName.json5", +// termMap: { +// "packageName": "%packageName" +// }, +// components: { +// encoding: { +// type: "kettle.dataSource.encoding.JSON5" +// } +// } +// }); + /** * Verifies the packageData JSON string against the packageDataSignature, returns the response with the packageData * field de-serialised. * @param {PackageResponse} packageResponse The response from the data source. - * @param {Object} allowedKeys Map of sha256 fingerprints (base64 encoded) of the public keys that are authorised. + * @param {Object} trustedKeys Map of sha256 fingerprints (base64 encoded) of the public keys that are authorised. * @return {Promise} Resolves with the response from the data source, with de-serialised packageData. */ -gpii.iod.checkPackageSignature = function (packageResponse, allowedKeys) { +gpii.iod.checkPackageSignature = function (packageResponse, trustedKeys) { var promise = fluid.promise(); - gpii.iod.verifySignedJSON(packageResponse.packageData, packageResponse.packageDataSignature, allowedKeys) + gpii.iod.verifySignedJSON(packageResponse.packageData, packageResponse.packageDataSignature, trustedKeys) .then(function (packageData) { var result = fluid.copy(packageResponse); result.packageData = packageData; @@ -68,12 +263,12 @@ gpii.iod.checkPackageSignature = function (packageResponse, allowedKeys) { * * @param {String} data The JSON data to verify (expects a `publicKey` field) * @param {String} signature The signature (base64). - * @param {Object} allowedKeys Map of sha256 fingerprints (base64 encoded) of the public keys that are authorised. + * @param {Object} trustedKeys Map of sha256 fingerprints (base64 encoded) of the public keys that are authorised. * * @return {Promise} Resolves, with the de-serialised object, when complete. Rejects if the signature doesn't validate, * or if one of the keys aren't in the given list. */ -gpii.iod.verifySignedJSON = function (data, signature, allowedKeys) { +gpii.iod.verifySignedJSON = function (data, signature, trustedKeys) { var promise = fluid.promise(); var failureMessage; var verified = false; @@ -83,7 +278,7 @@ gpii.iod.verifySignedJSON = function (data, signature, allowedKeys) { if (obj.publicKey) { // Get the sha256 fingerprint of the public key, and check if it's one of the allowed keys. var fingerprint = crypto.createHash("sha256").update(Buffer.from(obj.publicKey, "base64")).digest("base64"); - var keyName = fluid.keyForValue(allowedKeys, fingerprint); + var keyName = fluid.keyForValue(trustedKeys, fingerprint); var authorised = keyName !== undefined; fluid.log("IoD: Package signing key: " + fingerprint + " - ", (keyName || "unknown")); diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index 037d92d39..bc1d4a81b 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -22,6 +22,7 @@ var path = require("path"), fs = require("fs"), request = require("request"), crypto = require("crypto"), + URL = require("url").URL, child_process = require("child_process"); var fluid = require("infusion"); @@ -172,7 +173,10 @@ fluid.defaults("gpii.iod.packageInstaller", { * @param {Component} that The gpii.iod.installer instance. * @param {Component} iod The gpii.iod instance. */ -gpii.iod.installerCreated = function (that) { +gpii.iod.installerCreated = function (that, iod) { + if (that.options.installationID) { + that.installation = iod.installations[that.options.installationID]; + } if (that.installation) { that.installation.installer = that; that.packageData = that.installation.packageData; @@ -287,10 +291,13 @@ gpii.iod.downloadInstaller = function (that, installation, packageData) { installation.installerFile = path.join(installation.tempDir, packageData.installer); if (packageData.installerSource) { - if (/^https?:\/\//.test(packageData.installerSource)) { + + if (packageData.installerSource.indexOf("://") >= 0) { // Warning: Taking a url from an external source, downloading it, and then later executing it (as admin). var downloadPromise = gpii.iod.fileDownload(packageData.installerSource, installation.installerFile); - fluid.promise.follow(downloadPromise, promise); + downloadPromise.then(function (hash) { + installation.installerFileHash = hash; + }, promise.reject); } else { fs.copyFile(packageData.url, installation.installerFile, function (err) { if (err) { @@ -315,14 +322,14 @@ gpii.iod.downloadInstaller = function (that, installation, packageData) { /** * Downloads a file while generating its hash. * - * @param {String} url The remote uri. + * @param {String} address The remote uri. * @param {String} localPath Destination path. * @param {Object} options Options * @param {String} options.hash The hash algorithm (default: sha512) * @param {Function} options.process Callback for the progress, called with current and total. - * @return {Promise} Resolves with the hash when the download is complete. + * @return {Promise} Resolves with the hash (hex string) when the download is complete. */ -gpii.iod.fileDownload = function (url, localPath, options) { +gpii.iod.fileDownload = function (address, localPath, options) { options = Object.assign({ hash: "sha512" }, options); @@ -332,36 +339,53 @@ gpii.iod.fileDownload = function (url, localPath, options) { var output = fs.createWriteStream(localPath); var hash = crypto.createHash(options.hash); - hash.on("finish", function () { - promise.resolve(hash.digest()); + output.on("finish", function () { + promise.resolve(hash.digest("hex")); }); - var req = request.get({ - url: url + var url = new URL(address); + + var stream; + if (url.protocol === "file:") { + var offset = parseInt(url.searchParams.get("offset")) || 0; + stream = fs.createReadStream(url.pathname, { + start: offset + }); + + stream.on("open", function () { + stream.pipe(output); + }); + } else { + stream = request.get({ + url: address + }); + + stream.on("response", function (response) { + if (response.statusCode === 200) { + stream.pipe(output); + } else { + promise.reject({ + isError: true, + message: "Unable to download package: " + response.statusCode + " " + response.statusMessage, + address: address + }); + } + }); + } + + stream.on("data", function (data) { + hash.update(data); }); - req.on("error", function (err) { + stream.on("error", function (err) { promise.reject({ isError: true, message: "Unable to download package: " + err.message, - url: url, + address: address, error: err }); }); - req.on("response", function (response) { - if (response.statusCode === 200) { - response.pipe(output); - response.pipe(hash); - } else { - promise.reject({ - isError: true, - message: "Unable to download package: " + response.statusCode + " " + response.statusMessage, - url: url - }); - } - }); - return promise; }; @@ -381,7 +405,16 @@ gpii.iod.checkPackage = function (that, installation, packageData) { } else { // TODO: It shouldn't be checked here - another process may over-write it before the high privilege executes it. // Instead, take ownership then check the integrity in the same context as it's being ran. - promise = fluid.promise().resolve(); + var matches = packageData.installerHash === installation.installerFileHash; + promise = fluid.promise(); + if (matches) { + promise.resolve(); + } else { + promise.reject({ + isError: true, + message: "The downloaded installation file is wrong" + }); + } } return promise; }; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index 4b09762bc..f210d90bf 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -30,6 +30,7 @@ fluid.registerNamespace("gpii.iod.packages.resolvers"); fluid.require("%lifecycleManager"); require("./packageInstaller.js"); require("./iodSettingsHandler.js"); +require("./multiDataSource.js"); /** * Information about a package. @@ -81,32 +82,23 @@ gpii.iod.resolvers = {}; fluid.defaults("gpii.iod.packages", { gradeNames: ["fluid.component"], components: { - packageDataSource: { - type: "kettle.dataSource.file", + dataSource: { + type: "gpii.iod.multiDataSource", options: { - gradeNames: ["kettle.dataSource.file.moduleTerms", "kettle.dataSource.encoding.JSON5"], - path: "%gpii-universal/testData/installOnDemand/%packageName.json5", - termMap: { - "packageName": "%packageName" + invokers: { + "checkPackageSignature": { + funcName: "gpii.iod.checkPackageSignature", + args: ["{arguments}.0", "{gpii.iod.packages}.options.trustedKeys"] + } }, - components: { - encoding: { - type: "kettle.dataSource.encoding.JSON5" + listeners: { + "onRead.checkSignature": { + func: "{that}.checkPackageSignature", + args: "{arguments}.0" // packageResponse } } } }, - remotePackageDataSource: { - createOnEvent: "onServerFound", - type: "gpii.iod.remotePackageDataSource", - options: { - url: "@expand:gpii.iod.joinUrl({arguments}.0, {that}.options.urlPath)", - urlPath: "/packages/%packageName", - termMap: { - packageName: "%packageName" - } - } - }, variableResolver: { type: "gpii.lifecycleManager.variableResolver" } @@ -127,9 +119,13 @@ fluid.defaults("gpii.iod.packages", { }, listeners: { onCreate: "fluid.identity", - onServerFound: { - funcName: "fluid.set", - args: [ "{that}", "endpoint", "{arguments}.0"] + "onServerFound.dataSource": { + listener: "{that}.dataSource.events.onNewDataSource", + args: ["gpii.iod.packageDataSource.remote", "{arguments}.0", 20] // endpoint + }, + "onLocalPackagesFound.dataSource": { + listener: "{that}.dataSource.events.onNewDataSource", + args: ["gpii.iod.packageDataSource.local", "{arguments}.0", 10] // path } }, members: { @@ -151,21 +147,8 @@ fluid.defaults("gpii.iod.packages", { resolvers: { exists: "gpii.iod.existsResolver" } - }); -/** - * Convenience function to concatenate two parts of a URL together, ensuring there's a single '/' between each part - * (like `path.join`, but performs no normalisation and always uses slash). - * - * @param {String} front The first part of the URL - * @param {String} end The final part of the URL - * @return {String} `font` and `end` combined. - */ -gpii.iod.joinUrl = function (front, end) { - return front.replace(/\/+$/, "") + "/" + end.replace(/^\/+/, ""); -}; - /** * Resolver for ${{exists}.path}, determines if a filesystem path exists. The path can include environment variables, * named between two '%' symbols (like %this%), to work around the resolvers not supporting nested expressions. @@ -244,26 +227,21 @@ gpii.iod.getPackageData = function (that, packageRequest) { var promise = fluid.promise(); - var remote = !!that.remotePackageDataSource; - var dataSource = remote ? that.remotePackageDataSource : that.packageDataSource; - - if (dataSource) { - dataSource.get({ - packageName: packageRequest.packageName, - language: packageRequest.language, - version: packageRequest.version - }).then(function (packageResponse) { - fluid.log("IoD: Package response: ", packageResponse); - // Remote datasource wraps the packageData, local doesn't. - /** @type PackageData */ - var packageData = remote ? packageResponse.packageData : packageResponse; - - if (remote && packageResponse.installer) { - packageData.installerSource = gpii.iod.joinUrl(that.endpoint, packageResponse.installer); + that.dataSource.get({ + packageName: packageRequest.packageName, + language: packageRequest.language, + version: packageRequest.version + }).then(function (packageResponse) { + fluid.log("IoD: Package response: ", packageResponse); + /** @type {PackageData} */ + var packageData = packageResponse.packageData; + + if (packageData.name === packageRequest.packageName) { + if (packageResponse.installer) { + packageData.installerSource = packageResponse.installer; } - if (packageRequest.language && packageData.languages) - { + if (packageRequest.language && packageData.languages) { // Merge the language-specific info. var lang = gpii.iod.matchLanguage(Object.keys(packageData.languages), packageRequest.language); if (lang) { @@ -274,19 +252,20 @@ gpii.iod.getPackageData = function (that, packageRequest) { var resolvedPackageData = that.resolvePackage(packageData); promise.resolve(resolvedPackageData); - }, function (err) { + } else { promise.reject({ isError: true, - message: "Unknown package " + packageRequest.packageName, - error: err + message: "Unable to get package " + packageRequest.packageName + + ": Incorrect package name '" + packageData.name + "'" }); - }); - } else { + } + }, function (err) { promise.reject({ isError: true, - message: "No package data source for IoD" + message: "Unable to get package " + packageRequest.packageName + ": " + (err.message || "unknown error"), + error: err }); - } + }); return promise; }; diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 0abf0c2bd..87ee4b4af 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -32,6 +32,7 @@ var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.tests.iod"); +require("./common.js"); require("../index.js"); var teardowns = []; @@ -108,18 +109,18 @@ fluid.defaults("gpii.tests.iod", { listeners: { "onCreate.discoverServer": null, - "onCreate.readInstallations": null - }, - distributeOptions: { - packageDataSource: { - record: { - gradeNames: ["kettle.dataSource.file.moduleTerms"], - path: __dirname + "/packageData/%packageName.json5", - termMap: { - "packageName": "%packageName" - } - }, - target: "{that packages packageDataSource}.options" + "onCreate.readInstallations": null, + "onCreate.generateData": { + funcName: "gpii.tests.iod.generateLocalPackages", + args: [ + "local-packages.json5", + "localPackages/.morphic-packages", + "{tests}.options.keyPair" + ] + }, + "onCreate.addLocalPackages": { + func: "{packages}.events.onLocalPackagesFound.fire", + args: [path.join(__dirname, "localPackages")] } }, invokers: { @@ -137,6 +138,12 @@ fluid.defaults("gpii.tests.iod", { "testPackageType2a": "gpii.tests.iod.testInstaller2", "testPackageType2b": "gpii.tests.iod.testInstaller2", "testFailPackageType": "gpii.tests.iod.testInstallerFail" + }, + keyPair: "@expand:gpii.tests.iod.generateKeyPair()", + config: { + trustedKeys: { + packagesTest: "{that}.options.keyPair.fingerprint" + } } }); diff --git a/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js b/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js index 781e8ae43..3a2c4a9cd 100644 --- a/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js +++ b/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js @@ -22,162 +22,677 @@ var fluid = require("infusion"); var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); -var crypto = require("crypto"); +var path = require("path"), + fs = require("fs"), + url = require("url"), + json5 = require("json5"), + jqUnit = fluid.require("node-jqunit", require, "jqUnit"); -var jqUnit = fluid.require("node-jqunit"); var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.tests.iodPackageData"); +require("./common.js"); require("../index.js"); require("gpii-iodServer"); -gpii.tests.iodPackageData.verifySignedJSONTests = [ - { - id: "correctly signed", - input: { - data: { - testField: "testValue", - publicKey: "$key" +fluid.defaults("gpii.tests.iodPackageData.tests", { + gradeNames: ["fluid.test.testEnvironment"], + + components: { + packages: { + type: "gpii.iod.packages", + options: { + events: { + onServerFound: null, + onLocalPackagesFound: null + } + } + }, + iodServer: { + type: "kettle.server", + options: { + port: 51286, + components: { + packageServer: { + type: "kettle.app", + options: { + requestHandlers: { + packages: { + route: "/iod/packages/:packageName", + method: "get", + type: "gpii.tests.iodPackageData.packagesRequest" + } + } + } + } + } + } + }, + tester: { + type: "gpii.tests.iodPackageData.testCaseHolder", + options: { + testData: "{tests}.options.testData" + } + }, + localTester: { + type: "gpii.tests.iodPackageData.testCaseHolder.packageSource", + options: { + isLocal: true, + testData: "{tests}.options.testData", + installerSource: url.pathToFileURL(path.join(__dirname, "localPackages/packages/existing-file")).toString(), + invokers: { + "createDataSource": { + func: "{packages}.events.onLocalPackagesFound.fire", + args: [path.join(__dirname, "localPackages")] + } + } + } + }, + localTester2: { + type: "gpii.tests.iodPackageData.testCaseHolder.packageSource", + options: { + isRemote: true, + testData: "{tests}.options.testData", + expect: { + installerSource: "/installer/location-exists" + }, + invokers: { + "createDataSource": { + func: "{packages}.events.onServerFound.fire", + args: ["http://127.0.0.1:51286/iod"] + } + } + } + } + }, + testData: { + signing: { + keyPair: "@expand:gpii.tests.iod.generateKeyPair()", + testData: { + testField: "test value", + publicKey: "{that}.options.testData.signing.keyPair.publicKey" + }, + testDataNoKey: { + testField: "test value" + }, + signed: { + expander: { + funcName: "gpii.tests.iod.generateSignedData", + args: ["{that}.options.testData.signing.testData", "{that}.options.testData.signing.keyPair"] + } }, - signature: "$signature", - allowedKeys: "$fingerprint" + wrong: { + expander: { + funcName: "gpii.tests.iod.generateSignedData", + args: [{value: "wrong"}, "{that}.options.testData.signing.keyPair"] + } + } }, - expect: "resolve" + localPackages: { + packageA: { + packageData: { + name: "packageA" + } + }, + packageB: { + packageData: { + name: "packageB" + }, + installer: "packageDataSourceTests.js" + }, + packageC: { + packageData: { + name: "packageC" + }, + installer: "packageDataSourceTests.js", + offset: 42 + }, + packageD: { + packageData: { + name: "packageD" + }, + installer: "not/exist" + } + } } -]; - - +}); -gpii.tests.iodPackageData.generateKey = function (passphrase) { - var keyPair = crypto.generateKeyPairSync("rsa", { - modulusLength: 4096, - publicKeyEncoding: { - type: "spki", - format: "pem" - }, - privateKeyEncoding: { - type: "pkcs1", - format: "pem", - cipher: "aes-128-cbc", - passphrase: passphrase || "test" +fluid.defaults("gpii.tests.iodPackageData.packagesRequest", { + gradeNames: ["kettle.request.http"], + invokers: { + handleRequest: { + funcName: "gpii.tests.iodPackageData.handleRequest", + args: [ + "{packageServer}", "{request}", "{request}.req.params.packageName", "{request}.req.params.lang" + ] } - }); - - // Include the passphrase - keyPair.passphrase = passphrase; - // signPackageData expects the private key to be `key`. - keyPair.key = keyPair.privateKey; - delete keyPair.privateKey; - - // Get the key (without the PEM header+trailer) - var keyBinary = gpii.iodServer.packageFile.readPEM(keyPair.publicKey); - // Generate the finger print - keyPair.fingerprint = crypto.createHash("sha256").update(keyBinary).digest("base64"); - - return keyPair; -}; + } +}); +/** + * Handles /packages requests. Responds with a {PackageResponse} for the given package. + * + * @param {Component} packages The gpii.iodServer.packageServer instance. + * @param {Component} request The gpii.iodServer.packageServer.packagesRequest for this request. + * @param {String} packageName Name of the requested package. + */ +gpii.tests.iodPackageData.handleRequest = function (packages, request, packageName) { + fluid.log("package requested: " + packageName); + var localPackages; + try { + localPackages = json5.parse(fs.readFileSync(path.join(__dirname, "localPackages/.morphic-packages"))); + } catch (e) { + fluid.log("Error loading local package data (probably expected): ", e.message); + localPackages = {}; + } -gpii.tests.iodPackageData.createPackage = function (packageData, key, packageFile) { - return gpii.iodServer.packageFile.create(packageData, null, key, packageFile); + var result = fluid.copy(localPackages.packages[packageName]); + if (result) { + if (result.installer) { + result.installer = "/installer/" + packageName; + delete result.location; + } + request.events.onSuccess.fire(result); + } else { + request.events.onError.fire({ + message: "No such package", + statusCode: 404 + }); + } }; -gpii.tests.iodPackageData.assertReject = function (msg, promise) { - var promiseTogo = fluid.promise(); - promise.then(function () { - jqUnit.assertEquals(msg, "reject", "resolve"); - promiseTogo.resolve(); - }, function (reason) { - fluid.log("rejection result: ", reason); - jqUnit.assert("promise rejected"); - promiseTogo.resolve(); - }); - return promiseTogo; +gpii.tests.iodPackageData.assertContains = function (msg, expected, actual) { + var contains = (actual || "").toString().includes(expected); + if (contains) { + jqUnit.assert(msg); + } else { + jqUnit.assertEquals(msg, expected, actual); + } }; -jqUnit.asyncTest("test verifySignedJSON", function () { - - var pair = gpii.tests.iodPackageData.generateKey("test pass"); - - var data = { - testField: "test value", - publicKey: pair.publicKey - }; - - var signedData = gpii.iodServer.packageFile.signPackageData(data, pair); - var signedString = signedData.buffer.toString("utf8"); - - // The signature is well formed, but for the wrong thing - var wrongSignature = crypto.createSign("RSA-SHA512").update("something else").sign(pair); - // The signature isn't a signature - var badSignature = Buffer.from("bad signature"); - - - var promises = [ - gpii.iod.verifySignedJSON(signedString, signedData.signature, {key1: pair.fingerprint}).then(function (result) { - var expect = JSON.parse(signedString); - jqUnit.assertDeepEq("verifySignedJSON with valid data should resolve with the expected result", - expect, result); - }, function (e) { - fluid.log("reject reason: ", e); - jqUnit.fail("verifySignedJSON with valid data should resolve"); - }), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with invalid signedString should reject", - gpii.iod.verifySignedJSON(signedString.replace("test", "TEST"), signedData.signature, {key1: pair.fingerprint})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with the wrong signature should reject", - gpii.iod.verifySignedJSON(signedString, wrongSignature, {key1: pair.fingerprint})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with a bad signature should reject", - gpii.iod.verifySignedJSON(signedString, badSignature, {key1: pair.fingerprint})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with an empty signature should reject", - gpii.iod.verifySignedJSON(signedString, Buffer.from([]), {key1: pair.fingerprint})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with unknown fingerprint should reject", - gpii.iod.verifySignedJSON(signedString, signedData.signature, {key1: "wrong fingerprint"})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with no known fingerprints should reject", - gpii.iod.verifySignedJSON(signedString, signedData.signature, {})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with no public key in object should reject", - gpii.iod.verifySignedJSON("{\"testField\":123}", signedData.signature, {key1: pair.fingerprint})), - - gpii.tests.iodPackageData.assertReject("verifySignedJSON with invalid json should reject", - gpii.iod.verifySignedJSON("} invalid json:", signedData.signature, {key1: pair.fingerprint})), - - function () { - // test checkPackageSignature with valid data - var packageResponse = { - packageData: signedString, - packageDataSignature: signedData.signature - }; - // The packageData should return as an object. - var expect = { - packageData: JSON.parse(signedString), - packageDataSignature: signedData.signature - }; - - return gpii.iod.checkPackageSignature(packageResponse, {key1: pair.fingerprint}).then(function (result) { - jqUnit.assertDeepEq("checkPackageSignature with valid data should resolve with the expected result", - expect, result); - }, function (reason) { - jqUnit.assertEquals("checkPackageSignature with valid data should have resolved", "resolve", reason); - }); - }, +fluid.defaults("gpii.tests.iodPackageData.testCaseHolder", { + gradeNames: ["fluid.test.testCaseHolder"], + modules: [{ + name: "verifySignedJSON tests", + tests: [{ + expect: 1, + name: "valid data", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.signed.string", + "{that}.options.testData.signing.signed.signature", + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "verifySignedJSON with valid data should resolve with parsed data", + "@expand:JSON.parse({that}.options.testData.signing.signed.string)", + "{arguments}.0" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with invalid signed string should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.wrong.string", + "{that}.options.testData.signing.signed.signature", + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: Signature could not be verified.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with the wrong signature should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.signed.string", + "{that}.options.testData.signing.wrong.signature", + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: Signature could not be verified.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with a bad signature should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.signed.string", + Buffer.from("bad signature"), + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: Signature could not be verified.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with an empty signature should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.signed.string", + Buffer.from([]), + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: Signature could not be verified.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with unknown fingerprint should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.signed.string", + "{that}.options.testData.signing.signed.signature", + {key1: "another-fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: Signed by an unknown key.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with no known fingerprints should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "{that}.options.testData.signing.signed.string", + "{that}.options.testData.signing.signed.signature", + {} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: Signed by an unknown key.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with no public key in object should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "@expand:JSON.stringify({that}.options.testData.signing.testDataNoKey)", + "{that}.options.testData.signing.signed.signature", + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Signed JSON data failed verification: JSON object did not contain a publicKey field.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "verifySignedJSON with invalid JSON should reject", + sequence: [{ + task: "gpii.iod.verifySignedJSON", + args: [ + "} invalid json:", + "{that}.options.testData.signing.signed.signature", + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Reject reason should be the expected value", + "Error while verifying signed JSON data: Unexpected token } in JSON at position 0", + "{arguments}.0.message" + ] + }] + }] + }, { + name: "checkPackageSignature tests", + tests: [{ + expect: 1, + name: "valid data", + sequence: [{ + task: "gpii.iod.checkPackageSignature", + args: [ + { + packageData: "{that}.options.testData.signing.signed.string", + packageDataSignature: "{that}.options.testData.signing.signed.signature" + }, + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "checkPackageSignature with valid data should resolve with parsed packageData", + { + packageData: "@expand:JSON.parse({that}.options.testData.signing.signed.string)", + packageDataSignature: "{that}.options.testData.signing.signed.signature" + }, + "{arguments}.0" + ] + }] + }, { + expect: 1, + name: "invalid data", + sequence: [{ + task: "gpii.iod.checkPackageSignature", + args: [ + { + packageData: "{that}.options.testData.signing.signed.string", + packageDataSignature: "{that}.options.testData.signing.wrong.signature" + }, + {key1: "{that}.options.testData.signing.keyPair.fingerprint"} + ], + reject: "jqUnit.assertDeepEq", + rejectArgs: [ + "checkPackageSignature with invalid data should reject", + "Signed JSON data failed verification: Signature could not be verified.", + "{arguments}.0.message" + ] + }] + }] + }, { + name: "getLocal tests", + tests: [{ + expect: 5, + name: "local package source", + sequence: [{ + task: "gpii.iod.packageDataSource.getLocal", + args: [ + "@expand:fluid.toPromise({that}.options.testData.localPackages)", + { + packageName: "unknown-package" + }, + "" + ], + reject: "jqUnit.assertDeepEq", + rejectArgs: [ + "getLocal should reject with an unknown package", + "Package unknown-package not found", + "{arguments}.0.message" + ] + }, { + task: "gpii.iod.packageDataSource.getLocal", + args: [ + "@expand:fluid.toPromise({that}.options.testData.localPackages)", + { + packageName: "packageA" + }, + "" + ], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "getLocal should resolve with the requested package", + "{that}.options.testData.localPackages.packageA", + "{arguments}.0" + ] + }, { + task: "gpii.iod.packageDataSource.getLocal", + args: [ + "@expand:fluid.toPromise({that}.options.testData.localPackages)", + { + packageName: "packageB" + }, + __dirname + ], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "getLocal should resolve with the correct location", + { + packageData: { + name: "packageB" + }, + installer: "file://" + __filename + }, + "{arguments}.0" + ] + }, { + task: "gpii.iod.packageDataSource.getLocal", + args: [ + "@expand:fluid.toPromise({that}.options.testData.localPackages)", + { + packageName: "packageC" + }, + __dirname + ], + resolve: "jqUnit.assertDeepEq", + resolveArgs: [ + "getLocal should resolve with the correct location and offset", + { + packageData: { + name: "packageC" + }, + installer: "file://" + __filename + "?offset=42", + offset: 42 + }, + "{arguments}.0" + ] + }, { + task: "gpii.iod.packageDataSource.getLocal", + args: [ + "@expand:fluid.toPromise({that}.options.testData.localPackages)", + { + packageName: "packageD" + }, + __dirname + ], + reject: "jqUnit.assertDeepEq", + rejectArgs: [ + "getLocal should reject with a non-existing installer", + "File for packageD not found", + "{arguments}.0.message" + ] + }] + }] + }] +}); - gpii.tests.iodPackageData.assertReject("checkPackageSignature with invalid data should reject", - gpii.iod.checkPackageSignature( - {packageData: signedString, packageDataSignature: "wrong"}, {key1: pair.fingerprint})) +fluid.defaults("gpii.tests.iodPackageData.testCaseHolder.packageSource", { + gradeNames: ["fluid.test.testCaseHolder"], + modules: [{ + name: "packageSource tests", + tests: [{ + expect: 0, + name: "Generating .morphic-packages", + sequence: [{ + func: "gpii.tests.iod.generateLocalPackages", + args: [ + "local-packages.json5", + "localPackages/.morphic-packages", + "{that}.options.testData.signing.keyPair" + ] + }, { + func: "fluid.set", + args: [ + "{packages}.options", + "trustedKeys.testkey1", + "{that}.options.testData.signing.keyPair.fingerprint" + ] + }] + }, { + expect: 0, + name: "Add local package source", + sequence: [{ + func: "{that}.createDataSource", + args: [path.join(__dirname, "localPackages")] + }] + }] + }, { + name: "package tests", + tests: [{ + expect: 1, + name: "Testing successful package", + sequence: [{ + task: "{packages}.getPackageData", + args: [ + {packageName: "working"} + ], + resolve: "jqUnit.assertEquals", + resolveArgs: [ + "getPackageData should resolve with the requested package", + "working", + "{arguments}.0.name" + ] + }] + }, { + expect: 1, + name: "Testing unknown package", + sequence: [{ + task: "{packages}.getPackageData", + args: [ + {packageName: "unknown-package"} + ], + reject: "gpii.tests.iodPackageData.assertContains", + rejectArgs: [ + "getPackageData should reject for unknown package", + "Unable to get package unknown-package:", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "Testing wrong named package", + sequence: [{ + task: "{packages}.getPackageData", + args: [ + {packageName: "renamed"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "getPackageData should reject for renamed package", + "Unable to get package renamed: Incorrect package name 'real-name'", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "Testing untrusted package", + sequence: [{ + task: "{packages}.getPackageData", + args: [ + {packageName: "untrusted"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "getPackageData should reject for untrusted package", + "Unable to get package untrusted: Signed JSON data failed verification: Signed by an unknown key.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "Testing unsigned package", + sequence: [{ + task: "{packages}.getPackageData", + args: [ + {packageName: "unsigned"} + ], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "getPackageData should reject for unsigned package", + "Unable to get package unsigned: Signed JSON data failed verification: JSON object did not contain a publicKey field.", + "{arguments}.0.message" + ] + }] + }, { + expect: 1, + name: "Testing package with installer location", + sequence: [{ + task: "{packages}.getPackageData", + args: [ + {packageName: "location-exists"} + ], + resolve: "jqUnit.assertEquals", + resolveArgs: [ + "getPackageData should resolve location-exists, with the correct installerSource", + "{that}.options.expect.installerSource", + "{arguments}.0.installerSource" + ] + }] + }, { + expect: 2, + name: "Testing package with installer location, which doesn't exist", + sequence: [{ + task: "gpii.tests.iodPackageData.testInstallerNotExist", + args: [ "{packages}", {packageName: "location-not-exists"}, "{that}.options.isRemote" ], + resolve: "jqUnit.assert" + }] + }, { + expect: 1, + name: "Removing the packages file should disable the source", + sequence: [{ + func: "gpii.tests.iodPackageData.renamePackageFile", + args: ["localPackages/.morphic-packages", "localPackages/.morphic-packages-removed"] + }, { + // Request a package that did exist. + task: "{packages}.getPackageData", + args: [ + {packageName: "working"} + ], + reject: "jqUnit.assertTrue", + rejectArgs: [ + "getPackageData should reject for package after removing the local packages file", + "{arguments}.0.message" + ] + }, { + // Put the file back again. + func: "gpii.tests.iodPackageData.renamePackageFile", + args: ["localPackages/.morphic-packages-removed", "localPackages/.morphic-packages"] + }] + }] + }] +}); - ]; +/** + * Renames the packages file. + * @param {String} from The current name. + * @param {String} to The new name. + */ +gpii.tests.iodPackageData.renamePackageFile = function (from, to) { + fs.renameSync(path.resolve(__dirname, from), path.resolve(__dirname, to)); +}; - fluid.promise.sequence(promises).then(jqUnit.start, jqUnit.fail); +gpii.tests.iodPackageData.testInstallerNotExist = function (packages, packageRequest, isRemote) { + var result = packages.getPackageData(packageRequest); + var promise; + + if (isRemote) { + promise = result.then(jqUnit.assert); + } else { + promise = fluid.promise(); + result.then(promise.reject, function (reason) { + jqUnit.assertEquals("getPackageData should reject for location-not-exists package", + "Unable to get package location-not-exists: File for location-not-exists not found", reason.message); + promise.resolve(); + }); + } -}); + return promise; +}; +module.exports = kettle.test.bootstrap("gpii.tests.iodPackageData.tests"); diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index c35a247c0..a0a209c9f 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -379,20 +379,19 @@ jqUnit.asyncTest("test installation pipe-line", function () { }, jqUnit.fail); }); +gpii.tests.iodInstaller.sha512 = function (content) { + return crypto.createHash("sha512").update(content).digest("hex"); +}; + // Test the file download -jqUnit.asyncTest("test https download", function () { +jqUnit.asyncTest("test file download", function () { - if (process.env.GPII_QUICKTEST) { - fluid.log("Skipping download tests"); - jqUnit.assert(); - jqUnit.start(); - return; - } + var skipSSL = !!process.env.GPII_QUICKTEST; gpii.tests.iodInstaller.downloadTests = fluid.freezeRecursive([ { url: "https://raw.githubusercontent.com/GPII/universal/108be0f5f0377eaec9100c1926647e7550efc2ea/gpii.js", - expect: "8cb82683c931e15995b2573fda41c41eaacab59e" + expect: "969125ff55aac6237549f04d0f0307a54bbfbec1d9d9c742ff2129c16aef44f471a406c9ba8dcc28bf9d5855166384819c728d375ba0a03167c2eb45fbd9e3c0" }, { url: "https://gpii-test.invalid", @@ -440,7 +439,21 @@ jqUnit.asyncTest("test https download", function () { // Unopened port (hopefully) url: "https://127.0.0.1:51749", expect: "reject" + }, + // Local file tests + { + url: "file://" + __filename, + expect: gpii.tests.iodInstaller.sha512(fs.readFileSync(__filename)) + }, + { + url: "file://" + __filename + "?offset=20", + expect: gpii.tests.iodInstaller.sha512(fs.readFileSync(__filename, "utf8").substr(20)) + }, + { + url: "file://" + __dirname + "/no-such-file", + expect: "reject" } + ]); var filePrefix = path.join(os.tmpdir(), "gpii-test-download" + Math.random().toString(36) + "-"); @@ -457,9 +470,11 @@ jqUnit.asyncTest("test https download", function () { }); }); - var tests = gpii.tests.iodInstaller.downloadTests; - jqUnit.expect(tests.length * 3); + + + jqUnit.expect(tests.length * 4); + var testIndex = -1; var nextTest = function () { @@ -471,46 +486,40 @@ jqUnit.asyncTest("test https download", function () { var test = tests[testIndex]; var suffix = " - test " + testIndex + "(" + test.url + ")"; - var outFile = filePrefix + testIndex; - files.push(outFile); + if (skipSSL && test.url.indexOf("badssl.com") > -1) { + fluid.log("Skipping SSL test" + suffix); + jqUnit.expect(-4); + nextTest(); + } else { + + var outFile = filePrefix + testIndex; + files.push(outFile); - var p = gpii.iod.fileDownload(test.url, outFile); + var p = gpii.iod.fileDownload(test.url, outFile); - jqUnit.assertTrue("fileDownload must return a promise" + suffix, fluid.isPromise(p)); + jqUnit.assertTrue("fileDownload must return a promise" + suffix, fluid.isPromise(p)); - p.then(function () { - jqUnit.assertNotEquals("fileDownload must only succeed if expected" + suffix, test.expect, "reject"); + p.then(function (result) { + jqUnit.assertNotEquals("fileDownload must only succeed if expected" + suffix, test.expect, "reject"); - if (test.expect === "resolve") { - jqUnit.assert("resolved"); + if (test.expect === "resolve") { + jqUnit.assert("resolved"); + jqUnit.assert("resolved"); + } else if (test.expect !== "reject") { + var digest = gpii.tests.iodInstaller.sha512(fs.readFileSync(outFile)); + jqUnit.assertEquals("Hash in result must be correct", test.expect, result); + jqUnit.assertEquals("Hash of download must be correct", test.expect, digest); + } nextTest(); - } else if (test.expect !== "reject") { - var input = fs.createReadStream(outFile); - var hash = crypto.createHash("sha1"); - input.on("readable", function () { - var buffer = input.read(); - if (buffer) { - hash.update(buffer); - } else { - var digest = hash.digest("hex"); - jqUnit.assertEquals("Hash of download must be correct", test.expect, digest); - nextTest(); - } - }); - - input.on("error", function (err) { + }, function (err) { + jqUnit.assertEquals("fileDownload must only reject if expected" + suffix, test.expect, "reject"); + jqUnit.expect(-2); // the resolve block as 2 more asserts + if (test.expects !== "reject") { fluid.log(err); - jqUnit.fail(err); - }); - } - }, function (err) { - jqUnit.assertEquals("fileDownload must only reject if expected" + suffix, test.expect, "reject"); - jqUnit.assert("Balancing the expected assert count"); - if (test.expects !== "reject") { - fluid.log(err); - } - nextTest(); - }); + } + nextTest(); + }); + } }; diff --git a/gpii/node_modules/gpii-iod/test/packagesTests.js b/gpii/node_modules/gpii-iod/test/packagesTests.js index 4541f24d4..1acab4659 100644 --- a/gpii/node_modules/gpii-iod/test/packagesTests.js +++ b/gpii/node_modules/gpii-iod/test/packagesTests.js @@ -22,7 +22,7 @@ var fluid = require("infusion"); var kettle = fluid.require("kettle"); kettle.loadTestingSupport(); -var JSON5 = require("json5"), +var json5 = require("json5"), fs = require("fs"), path = require("path"), os = require("os"); @@ -32,168 +32,352 @@ var gpii = fluid.registerNamespace("gpii"); fluid.registerNamespace("gpii.tests.iodPackages"); +require("./common.js"); require("../index.js"); -var teardowns = []; - -jqUnit.module("gpii.tests.iodPackages", { - teardown: function () { - while (teardowns.length) { - teardowns.pop()(); - } - } -}); - -gpii.tests.iodPackages.getPackageDataTests = fluid.freezeRecursive([ - { - id: "No matching package", - request: { - packageName: "package-not-exists" - }, - expect: "reject" - }, - { - id: "variables resolved", - request: { - packageName: "env" - }, - expect: { - name: "env", - test: process.env.PATH, - packageType: "testPackageType1" - } - }, - { - id: "Single language package", - request: { - packageName: "package1" - }, - expect: JSON5.parse(fs.readFileSync(__dirname + "/packageData/package1.json5", "utf8")) - }, - { - id: "Single language package, with language specified", - request: { - packageName: "package1", - language: "fr-FR" - }, - expect: JSON5.parse(fs.readFileSync(__dirname + "/packageData/package1.json5", "utf8")) - }, - { - id: "Multi-language package, language not specified", - request: { - packageName: "languages" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "example.filename" - } - }, - { - id: "Multi-language package, unknown language specified", - request: { - packageName: "languages", - language: "xx-YY" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "example.filename" - } - }, - { - id: "Multi-language package, unknown language, no country specified", - request: { - packageName: "languages", - language: "xx" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "example.filename" - } - }, - { - id: "Multi-language package, full language specified", - request: { - packageName: "languages", - language: "es-ES" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es-es", - "language": "es-ES" - } - }, - { - id: "Multi-language package, full language specified 2", - request: { - packageName: "languages", - language: "es-MX" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es-mx", - "language": "es-MX" - } - }, - { - id: "Multi-language package, no country specified", - request: { - packageName: "languages", - language: "es" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es", - "language": "es" - } - }, - { - id: "Multi-language package, unknown country specified", - request: { - packageName: "languages", - language: "es-YY" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.es", - "language": "es" - } - }, - { - id: "Multi-language package, no country specified, no non-country package", - request: { - packageName: "languages", - language: "zh" +fluid.defaults("gpii.tests.iodPackages.tests", { + gradeNames: ["fluid.test.testEnvironment"], + + components: { + packages: { + type: "gpii.iod.packages", + options: { + events: { + onServerFound: null, + onLocalPackagesFound: null + }, + trustedKeys: { + packagesTest: "{tests}.options.keyPair.fingerprint" + }, + listeners: { + "onCreate.generateData": { + funcName: "gpii.tests.iod.generateLocalPackages", + args: [ + "local-packages.json5", + "localPackages/.morphic-packages", + "{tests}.options.keyPair" + ] + } + } + } }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.zh-cn", - "language": "zh-CN" + tester: { + type: "gpii.tests.iodPackages.testCaseHolder" } }, - { - id: "Multi-language package, unknown country specified, no non-country package", - request: { - packageName: "languages", - language: "zh-YY" - }, - expect: { - "name": "languages", - "packageType": "testPackageType1", - "filename": "file.zh-cn", - "language": "zh-CN" - } + keyPair: "@expand:gpii.tests.iod.generateKeyPair()" +}); + +fluid.defaults("gpii.tests.iodPackages.testCaseHolder", { + gradeNames: ["fluid.test.testCaseHolder"], + modules: [{ + name: "exists resolver tests", + tests: [{ + expect: 5, + name: "testExistsResolver", + sequence: [{ + func: "gpii.tests.iodPackages.testExistsResolver" + }] + }] + }, { + name: "resolvePackage() tests", + tests: [{ + name: "testing resolvePackage()", + sequence: [{ + func: "gpii.tests.iodPackages.testResolvePackage", + args: ["{packages}", "@expand:fluid.getGlobalValue(gpii.tests.iodPackages.resolvePackageTests)"] + }] + }] + }, { + name: "checkInstalled() tests", + tests: [{ + name: "testing checkInstalled()", + sequence: [{ + func: "gpii.tests.iodPackages.testCheckInstalled", + args: ["{packages}", "{that}.options.testData.checkInstalledTests"] + }] + }] + }, { + name: "PackageData tests", + tests: [{ + name: "testing with no data source", + expect: 1, + sequence: [{ + task: "{packages}.getPackageData", + args: [{ + packageName: "some-package" + }], + reject: "jqUnit.assertEquals", + rejectArgs: [ + "Should reject with the expected error", + "no root data sources", + "{arguments}.0.error.message" + ] + }] + }, { + name: "adding a data source", + expect: 1, + sequence: [{ + func: "{packages}.events.onLocalPackagesFound.fire", + args: [path.join(__dirname, "localPackages")] + }, { + func: "jqUnit.assertEquals", + args: [ + "Packages should have a data source", + 1, + "{packages}.dataSource.sortedDataSources.length" + ] + }] + }, { + name: "testing getPackageData()", + expect: 1, + sequence: [{ + task: "gpii.tests.iodPackages.testGetPackageData", + args: ["{packages}", "{that}.options.testData.getPackageDataTests"], + resolve: "jqUnit.assert" + }] + }] + }], + testData: { + checkInstalledTests: [ + { + name: "literal true", + isInstalled: true, + expect: true + }, + { + name: "literal false", + isInstalled: false, + expect: false + }, + { + name: "string true", + isInstalled: "true", + expect: true + }, + { + name: "string false", + isInstalled: "false", + expect: false + }, + { + name: "literal 1", + isInstalled: 1, + expect: true + }, + { + name: "literal 0", + isInstalled: 0, + expect: false + }, + { + name: "string 1", + isInstalled: "1", + expect: true + }, + { + name: "string 0", + isInstalled: "0", + expect: false + }, + { + name: "word string", + isInstalled: "hello", + expect: true + }, + { + name: "empty string", + isInstalled: "", + expect: false + }, + { + name: "null", + isInstalled: null, + expect: false + }, + { + name: "undefined", + isInstalled: undefined, + expect: false + }, + { + name: "no value", + expect: false + }, + { + name: "empty object", + isInstalled: {}, + expect: false + }, + { + name: "object", + isInstalled: {something: "hello"}, + expect: false + }, + { + name: "object containing isInstalled:true", + isInstalled: {isInstalled: true}, + expect: true + }, + { + name: "object containing isInstalled:0", + isInstalled: {isInstalled: "0"}, + expect: false + } + ], + getPackageDataTests: [ + { + id: "No matching package", + request: { + packageName: "package-not-exists" + }, + expect: "reject" + }, + { + id: "variables resolved", + request: { + packageName: "env" + }, + expect: { + name: "env", + test: process.env.PATH, + packageType: "testPackageType1" + } + }, + { + id: "Single language package", + request: { + packageName: "package1" + }, + expect: json5.parse(fs.readFileSync(__dirname + "/packageData/package1.json5", "utf8")) + }, + { + id: "Single language package, with language specified", + request: { + packageName: "package1", + language: "fr-FR" + }, + expect: json5.parse(fs.readFileSync(__dirname + "/packageData/package1.json5", "utf8")) + }, + { + id: "Multi-language package, language not specified", + request: { + packageName: "languages" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, unknown language specified", + request: { + packageName: "languages", + language: "xx-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, unknown language, no country specified", + request: { + packageName: "languages", + language: "xx" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "example.filename" + } + }, + { + id: "Multi-language package, full language specified", + request: { + packageName: "languages", + language: "es-ES" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es-es", + "language": "es-ES" + } + }, + { + id: "Multi-language package, full language specified 2", + request: { + packageName: "languages", + language: "es-MX" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es-mx", + "language": "es-MX" + } + }, + { + id: "Multi-language package, no country specified", + request: { + packageName: "languages", + language: "es" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es", + "language": "es" + } + }, + { + id: "Multi-language package, unknown country specified", + request: { + packageName: "languages", + language: "es-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.es", + "language": "es" + } + }, + { + id: "Multi-language package, no country specified, no non-country package", + request: { + packageName: "languages", + language: "zh" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.zh-cn", + "language": "zh-CN" + } + }, + { + id: "Multi-language package, unknown country specified, no non-country package", + request: { + packageName: "languages", + language: "zh-YY" + }, + expect: { + "name": "languages", + "packageType": "testPackageType1", + "filename": "file.zh-cn", + "language": "zh-CN" + } + } + ] } -]); +}); -gpii.tests.iodPackages.resolvePackageTests = fluid.freezeRecursive([ +// This test data needs to be declared outside a component, to avoid resolving the `${{values}}` don't get resolved. +gpii.tests.iodPackages.resolvePackageTests = [ // Resolver { name: "environment", @@ -406,116 +590,12 @@ gpii.tests.iodPackages.resolvePackageTests = fluid.freezeRecursive([ }, expect: "it works" } -]); +]; -gpii.tests.iodPackages.checkInstalledTests = fluid.freezeRecursive([ - { - name: "literal true", - isInstalled: true, - expect: true - }, - { - name: "literal false", - isInstalled: false, - expect: false - }, - { - name: "string true", - isInstalled: "true", - expect: true - }, - { - name: "string false", - isInstalled: "false", - expect: false - }, - { - name: "literal 1", - isInstalled: 1, - expect: true - }, - { - name: "literal 0", - isInstalled: 0, - expect: false - }, - { - name: "string 1", - isInstalled: "1", - expect: true - }, - { - name: "string 0", - isInstalled: "0", - expect: false - }, - { - name: "word string", - isInstalled: "hello", - expect: true - }, - { - name: "empty string", - isInstalled: "", - expect: false - }, - { - name: "null", - isInstalled: null, - expect: false - }, - { - name: "undefined", - isInstalled: undefined, - expect: false - }, - { - name: "no value", - expect: false - }, - { - name: "empty object", - isInstalled: {}, - expect: false - }, - { - name: "object", - isInstalled: {something: "hello"}, - expect: false - }, - { - name: "object containing isInstalled:true", - isInstalled: {isInstalled: true}, - expect: true - }, - { - name: "object containing isInstalled:0", - isInstalled: {isInstalled: "0"}, - expect: false - } -]); - -fluid.defaults("gpii.tests.iodPackages", { - gradeNames: [ "gpii.iod" ], - distributeOptions: { - packageDataSource: { - record: { - gradeNames: ["kettle.dataSource.file.moduleTerms"], - path: __dirname + "/packageData/%packageName.json5", - termMap: { - "packageName": "%packageName" - } - }, - target: "{that packages packageDataSource}.options" - } - }, - invokers: { - readInstallations: "fluid.identity", - writeInstallation: "fluid.identity" - } -}); - -jqUnit.test("test the 'exists' resolver function", function () { +/** + * Tests for existsResolver(). + */ +gpii.tests.iodPackages.testExistsResolver = function () { // Test it works on a non-existent file var result = gpii.iod.existsResolver(path.join(os.tmpdir(), "not-exist" + Math.random())); @@ -536,18 +616,21 @@ jqUnit.test("test the 'exists' resolver function", function () { process.env.GPII_TEST_RESOLVER2 = path.basename(__filename, "js"); var result5 = gpii.iod.existsResolver("%GPII_TEST_RESOLVER1%/%GPII_TEST_RESOLVER2%js"); jqUnit.assertTrue("existsResolver should return true, with multiple environment variables", result5); -}); - -jqUnit.test("test the package resolver", function () { - - var iod = gpii.tests.iodPackages(); +}; - fluid.each(gpii.tests.iodPackages.resolvePackageTests, function (test) { +/** + * Tests that the resovlers and tranformations within a packageData object get applied. + * @param {Component} packages The gpii.iod.packages instance. + * @param {Array} resolvePackageTests The tests. + */ +gpii.tests.iodPackages.testResolvePackage = function (packages, resolvePackageTests) { + jqUnit.expect(resolvePackageTests.length * 2 * 3); + fluid.each(resolvePackageTests, function (test) { var current = test; // Resolve the package more than once, to show it can be re-resolved. for (var i = 1; i <= 3; i++) { - var resolved = iod.packages.resolvePackage(current); + var resolved = packages.resolvePackage(current); var suffix = " - " + test.name + " (pass " + i + ")"; jqUnit.assertDeepEq("return of resolvePackage should contain the original package" + suffix, @@ -559,22 +642,53 @@ jqUnit.test("test the package resolver", function () { current = resolved; } }); +}; - iod.destroy(); -}); +/** + * Tests the checkInstalled() function. + * @param {Component} packages The gpii.iod.packages instance. + * @param {Array} checkInstalledTests The tests. + */ +gpii.tests.iodPackages.testCheckInstalled = function (packages, checkInstalledTests) { + // Run the canned tests. + jqUnit.expect(checkInstalledTests.length); + fluid.each(checkInstalledTests, function (test) { + var result = packages.checkInstalled(test); + jqUnit.assertEquals("checkInstalled should return the expected result - " + test.name, test.expect, result); + }); -// Test getPackageData returns correct information -jqUnit.asyncTest("test getPackageData", function () { + // Create a package which uses an environment variable to determine if it's installed. + var testEnv = "_gpii_test_checkInstalled"; + var testPackage = packages.resolvePackage({ + name: "checkInstalledTest", + isInstalled: "${{environment}." + testEnv + "}" + }); - var tests = gpii.tests.iodPackages.getPackageDataTests; - jqUnit.expect(tests.length * 2); + // Ensure the same package can return a different result - ie, the result is live. + var testValues = [ false, true, false, false, true, true, false ]; + jqUnit.expect(testValues.length); + fluid.each(testValues, function (value, index) { + // Change what isInstalled should resolve to. + process.env[testEnv] = value.toString(); + + var result = packages.checkInstalled(testPackage); + + jqUnit.assertEquals("checkInstalled should return the expected result - index=" + index, value, result); + }); + + delete process.env[testEnv]; +}; - var iod = gpii.tests.iodPackages(); +gpii.tests.iodPackages.testGetPackageData = function (packages, tests) { + + var promise = fluid.promise(); + + jqUnit.expect(tests.length * 2); var testIndex = -1; var nextTest = function () { if (++testIndex >= tests.length) { - jqUnit.start(); + promise.resolve(); return; } @@ -583,14 +697,18 @@ jqUnit.asyncTest("test getPackageData", function () { fluid.log("getPackage: " + test.request.packageName + ", " + test.request.language); - var p = iod.packages.getPackageData(test.request); + var p = packages.getPackageData(test.request); jqUnit.assertTrue("getPackageData must return a promise" + suffix, fluid.isPromise(p)); p.then(function (packageData) { + // Remove some fields that were added, so it can be compared directly with the file it came from. delete packageData.languages; delete packageData._original; + delete packageData.publicKey; + jqUnit.assertDeepEq("packageData must match expected" + suffix, test.expect, packageData); + nextTest(); }, function (e) { if (test.expect !== "reject") { @@ -603,33 +721,7 @@ jqUnit.asyncTest("test getPackageData", function () { }; nextTest(); -}); - -// Test checkInstalled works -jqUnit.test("test checkInstalled", function () { - var iod = gpii.tests.iodPackages(); - - fluid.each(gpii.tests.iodPackages.checkInstalledTests, function (test) { - var result = iod.packages.checkInstalled(test); - jqUnit.assertEquals("checkInstalled should return the expected result - " + test.name, test.expect, result); - }); - - // Ensure the same package can return a different result - ie, the result is live. - var testEnv = "_gpii_test_checkInstalled"; - var testPackage = iod.packages.resolvePackage({ - name: "checkInstalledTest", - isInstalled: "${{environment}." + testEnv + "}" - }); - - var testValues = [ false, true, false, false, true, true, false ]; - fluid.each(testValues, function (value, index) { - // Change what isInstalled resolves to. - process.env[testEnv] = value.toString(); + return promise; +}; - var result = iod.packages.checkInstalled(testPackage); - - jqUnit.assertEquals("checkInstalled should return the expected result - index=" + index, value, result); - }); - - delete process.env[testEnv]; -}); +module.exports = kettle.test.bootstrap("gpii.tests.iodPackages.tests"); From 0a24db844079f097828883e254ee261245de41cb Mon Sep 17 00:00:00 2001 From: ste Date: Sat, 4 Jan 2020 20:17:15 +0000 Subject: [PATCH 52/56] GPII-2971: Committing forgotten files. --- gpii/node_modules/gpii-iod/.gitignore | 2 + gpii/node_modules/gpii-iod/test/common.js | 131 ++++++++++++++++++ .../gpii-iod/test/local-packages.json5 | 34 +++++ 3 files changed, 167 insertions(+) create mode 100644 gpii/node_modules/gpii-iod/.gitignore create mode 100644 gpii/node_modules/gpii-iod/test/common.js create mode 100644 gpii/node_modules/gpii-iod/test/local-packages.json5 diff --git a/gpii/node_modules/gpii-iod/.gitignore b/gpii/node_modules/gpii-iod/.gitignore new file mode 100644 index 000000000..4e6302879 --- /dev/null +++ b/gpii/node_modules/gpii-iod/.gitignore @@ -0,0 +1,2 @@ +# Generated during testing: +test/localPackages diff --git a/gpii/node_modules/gpii-iod/test/common.js b/gpii/node_modules/gpii-iod/test/common.js new file mode 100644 index 000000000..38134203f --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/common.js @@ -0,0 +1,131 @@ +/* + * IoD Tests - package data source. + * + * Copyright 2020 Raising the Floor - International + * + * Licensed under the New BSD license. You may not use this file except in + * compliance with this License. + * + * The R&D leading to these results received funding from the + * Department of Education - Grant H421A150005 (GPII-APCP). However, + * these results do not necessarily represent the policy of the + * Department of Education, and you should not assume endorsement by the + * Federal Government. + * + * You may obtain a copy of the License at + * https://github.com/GPII/universal/blob/master/LICENSE.txt + */ + +"use strict"; + +var fluid = require("infusion"); +fluid.require("kettle"); + +var crypto = require("crypto"), + path = require("path"), + fs = require("fs"), + mkdirp = require("mkdirp"), + json5 = require("json5"); + +var gpii = fluid.registerNamespace("gpii"); +fluid.registerNamespace("gpii.tests.iod"); + +require("gpii-iodServer"); + +/** + * Signs `data` with the private key from `keyPair`. + * @param {Object} data The object to sign. + * @param {Object} keyPair The key to sign it with. + * @return {Object} The `string` and `signature` of the data. + */ +gpii.tests.iod.generateSignedData = function (data, keyPair) { + var signedData = gpii.iodServer.packageFile.signPackageData(data, keyPair); + + return { + string: signedData.buffer.toString("utf8"), + signature: signedData.signature + }; +}; + +gpii.tests.iod.keyPair = null; + +/** + * Generates a key pair. + * @param {String} passphrase [optional] The passphrase for the private key [default: "test"]. + * @return {Object} Object containing the private `key`, the corresponding `publicKey` and its `fingerprint`. + */ +gpii.tests.iod.generateKeyPair = function (passphrase) { + if (!gpii.tests.iod.keyPair) { + if (!passphrase) { + passphrase = "test"; + } + + var keyPair = crypto.generateKeyPairSync("rsa", { + modulusLength: 4096, + publicKeyEncoding: { + type: "spki", + format: "pem" + }, + privateKeyEncoding: { + type: "pkcs1", + format: "pem", + cipher: "aes-128-cbc", + passphrase: passphrase + } + }); + + // Include the passphrase + keyPair.passphrase = passphrase; + // signPackageData expects the private key to be `key`. + keyPair.key = keyPair.privateKey; + delete keyPair.privateKey; + + // Get the key (without the PEM header+trailer) + var keyBinary = gpii.iodServer.packageFile.readPEM(keyPair.publicKey); + // Generate the finger print + keyPair.fingerprint = crypto.createHash("sha256").update(keyBinary).digest("base64"); + keyPair.passphrase = passphrase; + gpii.tests.iod.keyPair = keyPair; + } + return gpii.tests.iod.keyPair; +}; + +/** + * Creates the .morphic-packages file, from local-packages.json5 and the contents of ./packageData/. + * + * @param {String} inputFile The local-packages.json5 file. + * @param {String} outputFile The output file, .morphic-packages. + * @param {Object} keyPair The private and public key object. + */ +gpii.tests.iod.generateLocalPackages = function (inputFile, outputFile, keyPair) { + var localPackages = json5.parse(fs.readFileSync(path.resolve(__dirname, inputFile))); + + // Add the files in the packageData directory + var dir = path.join(__dirname, "packageData"); + fluid.each(fs.readdirSync(dir), function (packageDataFile) { + var packageData = json5.parse(fs.readFileSync(path.join(dir, packageDataFile))); + localPackages.packages[packageData.name] = { + packageData: packageData + }; + }); + + // Replace the packageData objects with a serialised string with signature + fluid.each(localPackages.packages, function (localPackage) { + // If it's already a string, leave as-is + if (typeof(localPackage.packageData) !== "string") { + var signed = gpii.tests.iod.generateSignedData(localPackage.packageData, keyPair); + localPackage.packageData = signed.string; + if (!localPackage.packageDataSignature) { + localPackage.packageDataSignature = signed.signature.toString("base64"); + } + } + }); + + var outputPath = path.resolve(__dirname, outputFile); + fs.writeFileSync(outputPath, json5.stringify(localPackages, null, " ")); + + // Create a file that exists. + var packagesDir = path.join(path.dirname(outputPath), "packages"); + mkdirp.sync(packagesDir); + fs.writeFileSync(path.join(packagesDir, "existing-file"), "this file exists"); +}; diff --git a/gpii/node_modules/gpii-iod/test/local-packages.json5 b/gpii/node_modules/gpii-iod/test/local-packages.json5 new file mode 100644 index 000000000..23aeb6b61 --- /dev/null +++ b/gpii/node_modules/gpii-iod/test/local-packages.json5 @@ -0,0 +1,34 @@ +{ + packages: { + working: { + packageData: { + name: "working" + } + }, + renamed: { + packageData: { + name: "real-name" + } + }, + untrusted: { + // signature is valid, but the publicKey has not been trusted + packageData: '{"name":"untrusted","publicKey":"MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA9jz6Ls29OC1Nviy6r7BtHFXhNl3SBTF5kspJKHMylAZbZBGQEt/fhnSBIrsHYEG3nYP8y2Ghg2W6Yev3Q2191MIzDTSi1x838hFVxTwlunZJFXamktp8GCNh/MOW0/Db/eqsDlJ/l9AEZ8M3/4Nq0ON7k5cr6Ifh9qoK6HAhdOzQzp/GVGY3sf0mZCY2p0YJSa8zpgFwuNNTt3/ExhvXwEMIUwsBnZ4R/wERpAcG6C9GIxEzdFlZpbPASDDANBjW2kyFyqrPisKv6ZB9z6fDJ3SJBW3ZKlwdLWQf1G/DzxX9iRk2jTusuekDlhGTf+m7Vim+MmvJDQ7odA/CkwKi/ior+46lAk8H8kaXx01g1jjFr/x+LUIcmm72mQg3MLDKNQ4pIJ2Q1f9smR0Fac9l3/2cLbXaG/uASJkDJ6DoWpHwJehuEI57HCCug11i0WjrWMP4GdFpjiFA+mvC6dWEA7dDwGIRnm9X916o/L8217RzitH8VqNSu+vqVdYDT0E/vMDOPKD0jtMlDSfTUZqo9k7Bz2ugUTqBr2JATvmoFCevKjTXixs3g1sj42j20WAibzzMi//F492IsibnU4/0KR/6FuwKZ87sYsE+0sKEZNmdF/PlVEg0K6WiYkxIV+UaI4gvefCMYYQLix4RS17ZA4qZKDeAEvGZKeyFGc4ShNcCAwEAAQ=="}', + packageDataSignature: 'nVJJfxKOZm1M9PyBMqGuLuG3aq9EqaMkvvpHJ26bctEO470L4Nm1JcCWzc2G9Er8W8g/CbYSRzOaCv1gQxtJTnSiHMS18irYF/gaz6p9xPJa298iUmFh84AmksCpQbdK3/mLrrdcnJQIFdu2aP8231N8KxeuZi+SNT7knGTYSHfmTqnzQJ1RuloM73YZjpgDi4MMRKT9m/f5LVfw+hXw747phIxBtjHF3Ut/FVktTNSopnlrn/r5gNK8eRo0pO1WH3PjSEzKoDbDdbm0D2y/eBvwDzyflRkZClnYEf4503kE9umKWmKk6nYPDGF6/QLv8hVso1gyK2igCIQdhcBFhVQYSfVJEUi9jC8/uia0auhFqtOPBUVYdq0+nSA6hkLZsMJ9YVquFVx89U59842G/dn/asBv2WfYXMaOJbHVtUG4dTf7mi1z2zZKgRAFzcLuCWNEHeblNsKRbXQ35o5IyltftFhjb+haamS+224ULjMtLaKIBufc8/Ep1xuANqNtsL1I4wfbROzIbWAgOCGGrm343cHDgxS7pnWH4bQrBm4Zl/rBR5Lz81pILA0wvYY1zaPikj7IIS0FXjRU7XQVoVhDGtrHkHKjwL+5sfYXLrE/pz/7T6YPPePiP/X8VVM1zwJgDMpw4jiYKsREW1g2eGlIw3VKH69g8bUp6rnpSzw=', + }, + unsigned: { + packageData: '{"name":"unsigned"}', + }, + "location-exists": { + packageData: { + name: "location-exists", + }, + installer: "packages/existing-file" + }, + "location-not-exists": { + packageData: { + name: "location-not-exists", + }, + installer: "packages/none-existing-file" + } + } +} From 7fd09e4234f5fd786dae7576c7f48c1549434c0b Mon Sep 17 00:00:00 2001 From: ste Date: Sun, 26 Jan 2020 15:33:39 +0000 Subject: [PATCH 53/56] GPII-2971: Multiple package sources specified in config work. --- .../gpii-iod/src/installOnDemand.js | 93 ++++++++-- .../gpii-iod/src/multiDataSource.js | 7 +- .../gpii-iod/src/packageDataSource.js | 3 + .../gpii-iod/src/packageInstaller.js | 19 ++- gpii/node_modules/gpii-iod/src/packages.js | 6 +- .../gpii-iod/test/installOnDemandTests.js | 58 ++----- .../gpii-iod/test/packageInstallerTests.js | 160 ++++++++++++++++++ 7 files changed, 284 insertions(+), 62 deletions(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index c99d80ebb..0c7683097 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -24,6 +24,7 @@ var path = require("path"), os = require("os"), fs = require("fs"), request = require("request"), + url = require("url"), glob = require("glob"); var gpii = fluid.registerNamespace("gpii"); @@ -45,7 +46,18 @@ fluid.registerNamespace("gpii.iod"); */ fluid.defaults("gpii.iod", { - gradeNames: ["fluid.component", "fluid.modelComponent"], + gradeNames: ["fluid.component", "fluid.modelComponent", "fluid.contextAware"], + + contextAwareness: { + platform: { + checks: { + windows: { + contextValue: "{gpii.contexts.windows}", + gradeNames: ["gpii.iod.windows"] + } + } + } + }, components: { packages: { type: "gpii.iod.packages", @@ -74,21 +86,25 @@ fluid.defaults("gpii.iod", { onInstallerLoad: null // [ packageInstaller grade name, installation ] }, listeners: { - "onCreate.discoverServer": "{that}.discoverServer", - "onCreate.readInstallations": "{that}.readInstallations", - "onServerFound.setEndpoint": { - funcName: "fluid.set", - args: [ "{that}", "endpoint", "{arguments}.0"] + "onCreate.discoverPackageSources": { + func: "{that}.discoverPackageSources", + args: [ "{that}.options.config.packageSources", false ] }, + "onCreate.readInstallations": "{that}.readInstallations", "onServerFound.autoInstall": { funcName: "gpii.iod.autoInstall", args: ["{that}", "{that}.options.config.autoInstall"] } }, invokers: { - discoverServer: { - funcName: "gpii.iod.discoverServer", - args: ["{that}"] + discoverPackageSources: { + funcName: "gpii.iod.discoverPackageSources", + args: [ + "{that}.events.onLocalPackagesFound", + "{that}.events.onServerFound", + "{arguments}.0", // address(es) + "{arguments}.1" // check before adding? + ] }, requirePackage: { funcName: "gpii.iod.requirePackage", @@ -498,15 +514,60 @@ gpii.iod.uninstallPackage = function (that, installation) { }; /** - * Discovers the IoD server. + * Adds the given packages sources, after optionally checking if it is valid. * - * @param {Component} that The gpii.iod instance. + * @param {Event} onLocalPackagesFound The event to fire when a local package source is to be added. + * @param {Event} onServerFound The event to fire when a remote package source is to be added. + * @param {Array|String} addresses The package source address(es) to add. + * @param {Boolean} check True to check if the source is usable before adding it. + * @return {Promise} Resolves when complete. */ -gpii.iod.discoverServer = function (that) { - var addr = process.env.GPII_IOD_ENDPOINT || that.options.config.endpoint; - if (addr) { - gpii.iod.checkService(addr).then(that.events.onServerFound.fire); - } +gpii.iod.discoverPackageSources = function (onLocalPackagesFound, onServerFound, addresses, check) { + var promises = []; + + fluid.each(fluid.makeArray(addresses), function (address) { + var localPath; + if (address.includes("://")) { + try { + localPath = url.fileURLToPath(address); + } catch (e) { + // ignore + } + } else { + localPath = address; + } + + var foundPromise; + if (localPath) { + if (check) { + var dataFile = path.join(localPath, ".morphic-packages"); + fs.access(dataFile, function (err) { + if (err) { + fluid.log("IoD: Not using local package source '" + localPath + "': ", err.message); + foundPromise.reject(); + } else { + foundPromise.resolve(localPath); + } + }); + } else { + foundPromise = fluid.toPromise(localPath); + } + + foundPromise.then(onLocalPackagesFound.fire); + } else { + foundPromise = check ? gpii.iod.checkService(address) : fluid.toPromise(address); + foundPromise.then(onServerFound.fire); + } + + // Ignore any rejections + var p = fluid.promise(); + foundPromise.then(p.resolve, function () { + p.resolve(); + }); + promises.push(p); + }); + + return fluid.promise.sequence(promises); }; /** diff --git a/gpii/node_modules/gpii-iod/src/multiDataSource.js b/gpii/node_modules/gpii-iod/src/multiDataSource.js index 217b5e591..a990c18f7 100644 --- a/gpii/node_modules/gpii-iod/src/multiDataSource.js +++ b/gpii/node_modules/gpii-iod/src/multiDataSource.js @@ -125,7 +125,12 @@ gpii.iod.multiDataSource.getImpl = function (that, options, directModel) { var source = dataSources[index]; if (source) { var result = source.get(directModel, options); - result.then(promise.resolve, function (reason) { + result.then(function (result) { + if (source.options.appendData) { + result = Object.assign({}, source.options.appendData, result); + } + promise.resolve(result); + }, function (reason) { if (!firstReject) { firstReject = reason; } diff --git a/gpii/node_modules/gpii-iod/src/packageDataSource.js b/gpii/node_modules/gpii-iod/src/packageDataSource.js index 36fb1002c..fdc25d6d2 100644 --- a/gpii/node_modules/gpii-iod/src/packageDataSource.js +++ b/gpii/node_modules/gpii-iod/src/packageDataSource.js @@ -44,6 +44,9 @@ fluid.defaults("gpii.iod.packageDataSource.remote", { urlPath: "/packages/%packageName", termMap: { packageName: "%packageName" + }, + appendData: { + baseUrl: "{that}.options.address" } }); diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index bc1d4a81b..e2dbd1ae2 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -48,12 +48,12 @@ fluid.defaults("gpii.iod.packageInstaller", { executeCommand: { funcName: "gpii.iod.executeCommand", // PackageCommand, command, args - args: ["{that}", "{iod}", "{arguments}.0", "{arguments}.1", "{arguments}.2"] + args: ["{that}", "{arguments}.0", "{arguments}.1", "{arguments}.2"] }, startProcess: { funcName: "gpii.iod.startProcess", // command, args - args: ["{that}", "{iod}", "{arguments}.0", "{arguments}.1"] + args: ["{arguments}.0", "{arguments}.1"] }, // Remaining invokers are part of the installation pipe-line. Each one is passed the installation and returns @@ -282,13 +282,14 @@ gpii.iod.downloadInstaller = function (that, installation, packageData) { fluid.log("IoD: Downloading installer " + packageData.installerSource); + installation.installerFile = path.join(installation.tempDir, packageData.installer); + var promise = fluid.promise(); if (packageData.installCommands.download) { promise = that.executeCommand(packageData.installCommands.initialise); } else { promise = fluid.promise(); - installation.installerFile = path.join(installation.tempDir, packageData.installer); if (packageData.installerSource) { @@ -297,9 +298,10 @@ gpii.iod.downloadInstaller = function (that, installation, packageData) { var downloadPromise = gpii.iod.fileDownload(packageData.installerSource, installation.installerFile); downloadPromise.then(function (hash) { installation.installerFileHash = hash; + promise.resolve(); }, promise.reject); } else { - fs.copyFile(packageData.url, installation.installerFile, function (err) { + fs.copyFile(packageData.installerSource, installation.installerFile, function (err) { if (err) { promise.reject({ isError: true, @@ -433,6 +435,15 @@ gpii.iod.prepareInstall = function (that, installation, packageData) { if (packageData.installCommands.prepareInstall) { promise = that.executeCommand(packageData.installCommands.prepareInstall); } else { + // TODO: remove + if (installation.packageData.elevate) { + packageData.installerArgs = Object.assign({ + elevate: true + }, packageData.installerArgs); + packageData.uninstallerArgs = Object.assign({ + elevate: true + }, packageData.uninstallerArgs); + } promise = fluid.promise().resolve(); } return promise; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index f210d90bf..d04b6e939 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -238,7 +238,11 @@ gpii.iod.getPackageData = function (that, packageRequest) { if (packageData.name === packageRequest.packageName) { if (packageResponse.installer) { - packageData.installerSource = packageResponse.installer; + if (packageResponse.baseUrl && !packageResponse.installer.includes("://")) { + packageData.installerSource = gpii.iod.joinUrl(packageResponse.baseUrl, packageResponse.installer); + } else { + packageData.installerSource = packageResponse.installer; + } } if (packageRequest.language && packageData.languages) { diff --git a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js index 87ee4b4af..b925b533c 100644 --- a/gpii/node_modules/gpii-iod/test/installOnDemandTests.js +++ b/gpii/node_modules/gpii-iod/test/installOnDemandTests.js @@ -46,14 +46,6 @@ jqUnit.module("gpii.tests.iod", { }); gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ - { - packageRequest: "no-such-package", - expect: "reject" - }, - { - packageRequest: "unknownType", - expect: "reject" - }, { packageRequest: "package1", expect: { @@ -62,6 +54,14 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ }, resolveValue: true }, + { + packageRequest: "no-such-package", + expect: "reject" + }, + { + packageRequest: "unknownType", + expect: "reject" + }, { packageRequest: { packageName: "package1" @@ -105,22 +105,18 @@ gpii.tests.iod.startInstallerTests = fluid.freezeRecursive([ ]); fluid.defaults("gpii.tests.iod", { - gradeNames: [ "gpii.iod", "gpii.lifecycleManager" ], + gradeNames: [ "gpii.iod", "gpii.lifecycleManager", "gpii.userListeners.usb" ], listeners: { - "onCreate.discoverServer": null, + //"onCreate.discoverPackageSources": null, "onCreate.readInstallations": null, "onCreate.generateData": { funcName: "gpii.tests.iod.generateLocalPackages", args: [ "local-packages.json5", "localPackages/.morphic-packages", - "{tests}.options.keyPair" + "{that}.options.keyPair" ] - }, - "onCreate.addLocalPackages": { - func: "{packages}.events.onLocalPackagesFound.fire", - args: [path.join(__dirname, "localPackages")] } }, invokers: { @@ -143,7 +139,10 @@ fluid.defaults("gpii.tests.iod", { config: { trustedKeys: { packagesTest: "{that}.options.keyPair.fingerprint" - } + }, + packageSources: [ + path.join(__dirname, "localPackages") + ] } }); @@ -540,7 +539,7 @@ jqUnit.asyncTest("test uninstallation after restart", function () { jqUnit.asyncTest("test service discovery", function () { - jqUnit.expect(4); + jqUnit.expect(3); var server = require("http").createServer(); server.listen(0, "127.0.0.1"); @@ -555,22 +554,6 @@ jqUnit.asyncTest("test service discovery", function () { var localUrl = "http://" + server.address().address + ":" + server.address().port + "/"; fluid.log("listening: ", localUrl); - var timeout = setTimeout(function () { - jqUnit.fail("Timeout waiting for endpoint request/reply"); - }, 5000); - - var iodOptions = { - listeners: { - onServerFound: function () { - clearTimeout(timeout); - jqUnit.start(); - } - }, - config: { - endpoint: localUrl - } - }; - // try checkService directly var successPromise = gpii.iod.checkService(localUrl).then(function () { jqUnit.assert("checkService should resolve"); @@ -590,12 +573,7 @@ jqUnit.asyncTest("test service discovery", function () { fluid.promise.sequence([ failPromise, - successPromise, - function () { - // Check that discoverServer fires the event - var iod = gpii.tests.iod(iodOptions); - iod.discoverServer(); - } - ]); + successPromise + ]).then(jqUnit.start); }); }); diff --git a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js index a0a209c9f..b6b6cec86 100644 --- a/gpii/node_modules/gpii-iod/test/packageInstallerTests.js +++ b/gpii/node_modules/gpii-iod/test/packageInstallerTests.js @@ -21,6 +21,8 @@ var os = require("os"), fs = require("fs"), path = require("path"), + rimraf = require("rimraf"), + mkdirp = require("mkdirp"), crypto = require("crypto"); var fluid = require("infusion"); @@ -526,6 +528,164 @@ jqUnit.asyncTest("test file download", function () { nextTest(); }); +jqUnit.asyncTest("test downloadInstaller", function () { + + var tempDir = path.join(os.tmpdir(), "gpii-downloadInstaller-tests" + Math.random()); + mkdirp.sync(tempDir); + gpii.tests.iodInstaller.teardowns.push(function () { + rimraf.sync(tempDir); + }); + + var tests = [ + { + id: "successful local file URL", + installation: { + tempDir: "tempDir" + }, + packageData: { + installerSource: "file://installer-source", + installer: "installer.msi" + }, + fileDownload: "the hash", + expect: { + disposition: "resolve", + fileDownload: true, + installation: { + tempDir: "tempDir", + installerFile: "tempDir/installer.msi", + installerFileHash: "the hash" + } + } + }, { + id: "successful remote file", + installation: { + tempDir: "tempDir" + }, + packageData: { + installerSource: "https://installer-source", + installer: "installer.msi" + }, + fileDownload: "the hash", + expect: { + disposition: "resolve", + fileDownload: true, + installation: { + tempDir: "tempDir", + installerFile: "tempDir/installer.msi", + installerFileHash: "the hash" + } + } + }, { + id: "unsuccessful local file", + installation: { + tempDir: "tempDir" + }, + packageData: { + installerSource: "file://installer-source-reject", + installer: "installer.msi" + }, + expect: { + disposition: "reject", + fileDownload: true + } + }, { + id: "no file", + installation: { + tempDir: "tempDir" + }, + packageData: { + installer: "installer.msi" + }, + expect: { + disposition: "resolve", + fileDownload: false, + installation: { + tempDir: "tempDir", + installerFile: "tempDir/installer.msi" + } + } + } + ]; + + var installer; + var currentTest; + var suffix; + + // Mock fileDownload + var fileDownloadOrig = gpii.iod.fileDownload; + gpii.tests.iodInstaller.teardowns.push(function () { + gpii.iod.fileDownload = fileDownloadOrig; + }); + gpii.iod.fileDownload = function (address) { + var promise = fluid.promise(); + if (!currentTest.expect.fileDownload) { + jqUnit.fail("Call to fileDownload was unexpected" + suffix); + } + + jqUnit.assertEquals("fileDownload must be called with the correct address", + currentTest.packageData.installerSource, address); + + if (address.endsWith("reject")) { + promise.reject(); + } else { + promise.resolve(currentTest.fileDownload); + } + + return promise; + }; + + + var nextTest = function (testIndex) { + currentTest = tests[testIndex]; + if (installer) { + installer.destroy(); + } + if (!currentTest) { + jqUnit.start(); + return; + } + + suffix = " - test " + testIndex + "(" + currentTest.id + ")"; + + var installation = fluid.copy(currentTest.installation) || {}; + if (!installation.tempDir) { + installation.tempDir = tempDir; + } + var packageData = fluid.copy(currentTest.packageData); + if (!packageData.installCommands) { + packageData.installCommands = {}; + } + + + installer = gpii.tests.iodInstaller.loggingInstaller(); + + var p = gpii.iod.downloadInstaller(installer, installation, packageData); + jqUnit.assertTrue("fileDownload must return a promise" + suffix, fluid.isPromise(p)); + + p.then(function () { + jqUnit.assertEquals("downloadInstaller must only resolve if expected" + suffix, + currentTest.expect.disposition, "resolve"); + + jqUnit.assertDeepEq("downloadInstaller must only resolve if expected" + suffix, + currentTest.expect.installation, installation); + + nextTest(testIndex + 1); + }, function (err) { + if (currentTest.expect.disposition !== "reject") { + fluid.log(err); + } + jqUnit.assertEquals("downloadInstaller must only reject if expected" + suffix, + currentTest.expect.disposition, "reject"); + + nextTest(testIndex + 1); + }); + + }; + + nextTest(0); + +}); + jqUnit.asyncTest("test executeCommand", function () { var tests = gpii.tests.iodInstaller.executeCommandTests; From 809704341ee8f5ab523c6aba3f7a4aad7bb1c3da Mon Sep 17 00:00:00 2001 From: ste Date: Sun, 26 Jan 2020 15:45:30 +0000 Subject: [PATCH 54/56] GPII-2971: Corrected test expectation. --- gpii/node_modules/gpii-iod/test/packageDataSourceTests.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js b/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js index 3a2c4a9cd..0e56b22f8 100644 --- a/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js +++ b/gpii/node_modules/gpii-iod/test/packageDataSourceTests.js @@ -96,7 +96,7 @@ fluid.defaults("gpii.tests.iodPackageData.tests", { isRemote: true, testData: "{tests}.options.testData", expect: { - installerSource: "/installer/location-exists" + installerSource: "http://127.0.0.1:51286/iod/installer/location-exists" }, invokers: { "createDataSource": { From 9c0668b105239763ca12cc6a7f9941f594604093 Mon Sep 17 00:00:00 2001 From: ste Date: Mon, 27 Jan 2020 17:46:29 +0000 Subject: [PATCH 55/56] GPII-2971: Detecting mounted drives for IoD --- gpii/node_modules/gpii-iod/src/installOnDemand.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 0c7683097..7eb7360a1 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -86,10 +86,15 @@ fluid.defaults("gpii.iod", { onInstallerLoad: null // [ packageInstaller grade name, installation ] }, listeners: { - "onCreate.discoverPackageSources": { + "onCreate.configuredPackageSources": { func: "{that}.discoverPackageSources", args: [ "{that}.options.config.packageSources", false ] }, + "onCreate.localPackageSources": { + priority: "after:configuredPackageSources", + func: "{that}.discoverPackageSources", + args: [ "@expand:{that}.getMountedVolumes()", true ] + }, "onCreate.readInstallations": "{that}.readInstallations", "onServerFound.autoInstall": { funcName: "gpii.iod.autoInstall", @@ -97,6 +102,7 @@ fluid.defaults("gpii.iod", { } }, invokers: { + getMountedVolumes: "fluid.identity", discoverPackageSources: { funcName: "gpii.iod.discoverPackageSources", args: [ @@ -165,6 +171,10 @@ fluid.defaults("gpii.iodLifeCycleManager", { "{lifecycleManager}.events.onSessionStop": { func: "{that}.autoRemove", args: [false] + }, + "{userListeners}.usb.events.onMount": { + func: "{that}.discoverPackageSources", + args: ["{arguments}.1", true] } }, @@ -541,6 +551,7 @@ gpii.iod.discoverPackageSources = function (onLocalPackagesFound, onServerFound, if (localPath) { if (check) { var dataFile = path.join(localPath, ".morphic-packages"); + foundPromise = fluid.promise(); fs.access(dataFile, function (err) { if (err) { fluid.log("IoD: Not using local package source '" + localPath + "': ", err.message); From 479816a77f9b9dd0c3a92011743220fe6b98bc63 Mon Sep 17 00:00:00 2001 From: ste Date: Wed, 29 Jan 2020 23:46:34 +0000 Subject: [PATCH 56/56] GPII-2971: Using exit codes for installer commands --- .../gpii-iod/src/installOnDemand.js | 35 ++++++++ .../gpii-iod/src/packageDataSource.js | 15 ++-- .../gpii-iod/src/packageInstaller.js | 84 ++++++++++++++----- gpii/node_modules/gpii-iod/src/packages.js | 10 ++- testData/preferences/iod_jaws.json5 | 15 ++++ 5 files changed, 134 insertions(+), 25 deletions(-) create mode 100644 testData/preferences/iod_jaws.json5 diff --git a/gpii/node_modules/gpii-iod/src/installOnDemand.js b/gpii/node_modules/gpii-iod/src/installOnDemand.js index 7eb7360a1..ae68d553d 100644 --- a/gpii/node_modules/gpii-iod/src/installOnDemand.js +++ b/gpii/node_modules/gpii-iod/src/installOnDemand.js @@ -547,6 +547,7 @@ gpii.iod.discoverPackageSources = function (onLocalPackagesFound, onServerFound, localPath = address; } + var foundPromise; if (localPath) { if (check) { @@ -557,6 +558,7 @@ gpii.iod.discoverPackageSources = function (onLocalPackagesFound, onServerFound, fluid.log("IoD: Not using local package source '" + localPath + "': ", err.message); foundPromise.reject(); } else { + fluid.log("IoD: Using local package source '" + localPath + "':"); foundPromise.resolve(localPath); } }); @@ -618,3 +620,36 @@ gpii.iod.autoInstall = function (that, packages) { }, 1000); } }; + +// For node < 10.12.0 +if (!url.pathToFileURL) { + url.pathToFileURL = function (localPath) { + var togo = new url.URL("file://"); + togo.pathname = path.resolve(localPath); + return togo; + }; +} +if (!url.fileURLToPath) { + url.fileURLToPath = function (fileUrl) { + var u = new url.URL(fileUrl); + var pathTogo; + if (u.protocol === "file:") { + pathTogo = decodeURIComponent(u.pathname); + if (process.platform === "win32") { + pathTogo = pathTogo.replace(/\//g, "\\"); + if (u.hostname) { + // UNC path + pathTogo = "\\\\" + u.hostname + u.pathname; + } else if (pathTogo[2] === ":") { + // X:\ path + pathTogo = pathTogo.substr(1); + } else { + throw new Error("File url has no drive or host. " + fileUrl); + } + } + } else { + throw new Error("File url must be a file: url. " + fileUrl); + } + return pathTogo; + }; +} diff --git a/gpii/node_modules/gpii-iod/src/packageDataSource.js b/gpii/node_modules/gpii-iod/src/packageDataSource.js index fdc25d6d2..21d830c79 100644 --- a/gpii/node_modules/gpii-iod/src/packageDataSource.js +++ b/gpii/node_modules/gpii-iod/src/packageDataSource.js @@ -172,19 +172,24 @@ gpii.iod.packageDataSource.loadData = function (that, directory, file) { error: err }); } else { + var failed; try { that.data = json5.parse(content); - if (!that.data.packages) { - that.data.packages = {}; - } - fluid.log("IoD: Loaded " + Object.keys(that.data.packages).length + " packages from " + dataFile); - promise.resolve(that.data.packages); } catch (err) { + failed = true; promise.reject({ isError: true, message: "IoD: unable to parse the data file " + dataFile + ": " + err.message, error: err }); + + } + if (!failed) { + if (!that.data.packages) { + that.data.packages = {}; + } + fluid.log("IoD: Loaded " + Object.keys(that.data.packages).length + " packages from " + dataFile); + promise.resolve(that.data.packages); } } }); diff --git a/gpii/node_modules/gpii-iod/src/packageInstaller.js b/gpii/node_modules/gpii-iod/src/packageInstaller.js index e2dbd1ae2..d6c3b2bd3 100644 --- a/gpii/node_modules/gpii-iod/src/packageInstaller.js +++ b/gpii/node_modules/gpii-iod/src/packageInstaller.js @@ -22,7 +22,7 @@ var path = require("path"), fs = require("fs"), request = require("request"), crypto = require("crypto"), - URL = require("url").URL, + url = require("url"), child_process = require("child_process"); var fluid = require("infusion"); @@ -81,7 +81,7 @@ fluid.defaults("gpii.iod.packageInstaller", { }, installComplete: { funcName: "gpii.iod.installComplete", - args: ["{that}", "{that}.installation", "{that}.installation.packageData"] + args: ["{that}", "{iod}", "{that}.installation", "{that}.installation.packageData"] }, startApplication: { funcName: "gpii.iod.startApplication", @@ -94,7 +94,7 @@ fluid.defaults("gpii.iod.packageInstaller", { uninstallPackage: "fluid.notImplemented", uninstallComplete: { funcName: "gpii.iod.installComplete", - args: ["{that}", "{that}.installation", "{that}.installation.packageData"] + args: ["{that}", "{iod}", "{that}.installation", "{that}.installation.packageData"] } }, events: { @@ -141,15 +141,15 @@ fluid.defaults("gpii.iod.packageInstaller", { func: "{that}.stopApplication", priority: "first" }, - "onRemovePackage.uninstallPackage": { + "onRemovePackage.uninstall": { func: "{that}.uninstallPackage", priority: "after:stopApplication" }, "onRemovePackage.cleanup": { func: "{that}.cleanup", - priority: "after:uninstallPackage" + priority: "after:uninstall" }, - "onRemovePackage.uninstallComplete": { + "onRemovePackage.complete": { func: "{that}.uninstallComplete", priority: "after:cleanup" } @@ -247,7 +247,12 @@ gpii.iod.customCommand = function (that, when) { var command = commands && commands[that.installation.currentStage + ":" + when]; var togo; if (command) { - togo = that.executeCommand(command); + var promises = fluid.transform(fluid.makeArray(command), function (c) { + return function () { + return that.executeCommand(c); + }; + }); + togo = fluid.promise.sequence(promises); } return togo; }; @@ -345,12 +350,13 @@ gpii.iod.fileDownload = function (address, localPath, options) { promise.resolve(hash.digest("hex")); }); - var url = new URL(address); + var downloadUrl = new url.URL(address); var stream; - if (url.protocol === "file:") { - var offset = parseInt(url.searchParams.get("offset")) || 0; - stream = fs.createReadStream(url.pathname, { + if (downloadUrl.protocol === "file:") { + var offset = parseInt(downloadUrl.searchParams.get("offset")) || 0; + var file = url.fileURLToPath(address); + stream = fs.createReadStream(file, { start: offset }); @@ -478,17 +484,31 @@ gpii.iod.cleanup = function (that, installation, packageData) { /** * Called when the installation has completed. * @param {Component} that The gpii.iod.installer instance. + * @param {Component} iod The gpii.iod instance. * @param {Installation} installation The installation state. * @param {PackageData} packageData The package data. * @return {Promise} Resolves when complete. */ -gpii.iod.installComplete = function (that, installation, packageData) { +gpii.iod.installComplete = function (that, iod, installation, packageData) { var promise; fluid.log("IoD: Completed installation of " + packageData.name); if (packageData.installCommands.complete) { promise = that.executeCommand(packageData.installCommands.complete); } else { - promise = fluid.promise().resolve(); + // Check if the application is detected, to see if the (un)installation process really worked. + var installed = iod.packages.checkInstalled(packageData); + var installing = (that.currentAction === "install"); + promise = fluid.promise(); + + if (installed === installing) { + promise.resolve(); + } else { + promise.reject({ + isError: true, + message: packageData.name + + (installing ? " was not detected after installing" : " was still detected after uninstalling") + }); + } } return promise; }; @@ -634,7 +654,9 @@ gpii.iod.startProcess = function (command, args) { }); } } else { - promise.resolve(); + promise.resolve({ + exitCode: code + }); } }); return promise; @@ -660,21 +682,45 @@ gpii.iod.executeCommand = function (that, execOptions, command, args) { execOptions.args = fluid.makeArray(args || execOptions.args); execOptions = gpii.iod.expand(execOptions, that.installation); - var promise; + var processPromise; if (!execOptions.command) { - promise = fluid.promise().reject({ + processPromise = fluid.promise().reject({ isError: true, message: "executeCommand called without a command" }); } else if (execOptions.elevate && that.startElevatedProcess) { - promise = that.startElevatedProcess(execOptions.command, execOptions.args, {desktop: execOptions.desktop}); + processPromise = that.startElevatedProcess(execOptions.command, execOptions.args, {desktop: execOptions.desktop}); } else { if (execOptions.elevate) { fluid.log(fluid.logLevel.WARN, "Running elevated commands is not supported on this system."); } - promise = that.startProcess(execOptions.command, execOptions.args); + processPromise = that.startProcess(execOptions.command, execOptions.args); } - return promise; + var promiseTogo = fluid.promise(); + processPromise.then(function (result) { + var success; + // Resolve or reject based on the exit code. + if (fluid.isValue(execOptions.success)) { + success = fluid.makeArray(execOptions.success).includes(result.exitCode); + } else if (fluid.isValue(execOptions.failure)) { + success = !fluid.makeArray(execOptions.failure).includes(result.exitCode); + } else { + success = true; + } + + if (success) { + promiseTogo.resolve(result); + } else { + promiseTogo.reject({ + message: "Command returned " + (execOptions.success ? "non-success" : "failure") + + " exit code " + result.exitCode, + execOptions: execOptions, + result: result + }); + } + }, promiseTogo.reject); + + return promiseTogo; }; diff --git a/gpii/node_modules/gpii-iod/src/packages.js b/gpii/node_modules/gpii-iod/src/packages.js index d04b6e939..68e9b61ea 100644 --- a/gpii/node_modules/gpii-iod/src/packages.js +++ b/gpii/node_modules/gpii-iod/src/packages.js @@ -66,6 +66,12 @@ require("./multiDataSource.js"); * @property {PackageCommand} installCommands.cleanup The cleanup command. * @property {PackageCommand} installCommands.complete The installation is complete. * + * @property {Object} uninstallCommands Commands to execute at certain points in the installation, rather than + * perform the default action (if any). + * @property {PackageCommand} uninstallCommands.uninstall The initialise command. + * @property {PackageCommand} uninstallCommands.cleanup The download command. + * @property {PackageCommand} uninstallCommands.complete The check command. + * */ /** @@ -73,6 +79,8 @@ require("./multiDataSource.js"); * @typedef {Object} PackageCommand * @property {String} command The command to invoke. * @property {String|Array} args arguments passed to the command. + * @property {Number|Array} success Exit code(s) to assume success (mutually exclusive with failure). + * @property {Number|Array} failure Exit code(s) to assume failure (mutually exclusive with success). * @property {Boolean} elevate true to run as administrator. * @property {Boolean} desktop true to run in the context of the desktop, if elevate is true. */ @@ -327,5 +335,5 @@ gpii.iod.checkInstalled = function (that, packageData) { isInstalled = isInstalled.isInstalled || isInstalled.value; } - return !!fluid.coerceToPrimitive(isInstalled); + return isInstalled === undefined || !!fluid.coerceToPrimitive(isInstalled); }; diff --git a/testData/preferences/iod_jaws.json5 b/testData/preferences/iod_jaws.json5 new file mode 100644 index 000000000..1da86a90c --- /dev/null +++ b/testData/preferences/iod_jaws.json5 @@ -0,0 +1,15 @@ + +{ + "flat": { + "contexts": { + "gpii-default": { + "name": "Default preferences", + "preferences": { + "http://registry.gpii.net/applications/net.gpii.test.iod": { + "jaws": {} + } + } + } + } + } +}